mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-15 20:24:07 +00:00
Merge branch 'master' into vdimir/join_select_inner_table
This commit is contained in:
vdimir
commit
9b68b61cbd
3
.github/workflows/create_release.yml
vendored
3
.github/workflows/create_release.yml
vendored
@ -3,6 +3,9 @@ name: CreateRelease
|
||||
concurrency:
|
||||
group: release
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
|
||||
4
.github/workflows/docker_test_images.yml
vendored
4
.github/workflows/docker_test_images.yml
vendored
@ -1,4 +1,5 @@
|
||||
name: Build docker images
|
||||
|
||||
'on':
|
||||
workflow_call:
|
||||
inputs:
|
||||
@ -12,6 +13,9 @@ name: Build docker images
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
jobs:
|
||||
DockerBuildAarch64:
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
|
||||
4
.github/workflows/reusable_build_stage.yml
vendored
4
.github/workflows/reusable_build_stage.yml
vendored
@ -1,7 +1,11 @@
|
||||
### FIXME: merge reusable_test.yml and reusable_build.yml as they are almost identical
|
||||
# and then merge reusable_build_stage.yml and reusable_test_stage.yml
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
name: BuildStageWF
|
||||
|
||||
'on':
|
||||
workflow_call:
|
||||
inputs:
|
||||
|
||||
4
.github/workflows/reusable_test_stage.yml
vendored
4
.github/workflows/reusable_test_stage.yml
vendored
@ -1,4 +1,8 @@
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
name: StageWF
|
||||
|
||||
'on':
|
||||
workflow_call:
|
||||
inputs:
|
||||
|
||||
@ -14,9 +14,10 @@ The following versions of ClickHouse server are currently supported with securit
|
||||
|
||||
| Version | Supported |
|
||||
|:-|:-|
|
||||
| 24.9 | ✔️ |
|
||||
| 24.8 | ✔️ |
|
||||
| 24.7 | ✔️ |
|
||||
| 24.6 | ✔️ |
|
||||
| 24.6 | ❌ |
|
||||
| 24.5 | ❌ |
|
||||
| 24.4 | ❌ |
|
||||
| 24.3 | ✔️ |
|
||||
|
||||
@ -69,6 +69,11 @@ set (SRCS_PROGRAM_OPTIONS
|
||||
"${LIBRARY_DIR}/libs/program_options/src/winmain.cpp"
|
||||
)
|
||||
|
||||
# Always compile this file with the highest possible level of optimizations, even in Debug builds.
|
||||
# Otherwise the client takes too long to start and SQL stateless tests (many options to parse)
|
||||
# https://github.com/ClickHouse/ClickHouse/issues/65745
|
||||
set_source_files_properties(${SRCS_PROGRAM_OPTIONS} PROPERTIES COMPILE_FLAGS "-O3")
|
||||
|
||||
add_library (_boost_program_options ${SRCS_PROGRAM_OPTIONS})
|
||||
add_library (boost::program_options ALIAS _boost_program_options)
|
||||
target_include_directories (_boost_program_options SYSTEM BEFORE PUBLIC ${LIBRARY_DIR})
|
||||
|
||||
@ -164,15 +164,6 @@ target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
|
||||
# Because our coverage callbacks call malloc, and recursive call of malloc could not work.
|
||||
target_compile_options(_jemalloc PRIVATE ${WITHOUT_COVERAGE_FLAGS_LIST})
|
||||
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||
target_compile_definitions(_jemalloc PRIVATE
|
||||
-DJEMALLOC_DEBUG=1
|
||||
# Usage examples:
|
||||
# - MALLOC_CONF=log:.
|
||||
# - MALLOC_CONF='log:core.malloc.exit|core.sallocx.entry|core.sdallocx.entry'
|
||||
-DJEMALLOC_LOG=1)
|
||||
endif ()
|
||||
|
||||
target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1)
|
||||
|
||||
# jemalloc provides support two unwind flavors:
|
||||
|
||||
@ -50,7 +50,11 @@ if(NOT MARIADB_UNIX_ADDR)
|
||||
set(MARIADB_UNIX_ADDR "/tmp/mysql.sock")
|
||||
endif()
|
||||
|
||||
set(HAVE_ALLOCA_H 1)
|
||||
if (OS_FREEBSD)
|
||||
set(HAVE_ALLOCA_H 0)
|
||||
else()
|
||||
set(HAVE_ALLOCA_H 1)
|
||||
endif()
|
||||
set(HAVE_ARPA_INET_H 1)
|
||||
set(HAVE_DLFCN_H 1)
|
||||
set(HAVE_FCNTL_H 1)
|
||||
|
||||
@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.8.4.13"
|
||||
ARG VERSION="24.9.1.3278"
|
||||
ARG PACKAGES="clickhouse-keeper"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.8.4.13"
|
||||
ARG VERSION="24.9.1.3278"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="24.8.4.13"
|
||||
ARG VERSION="24.9.1.3278"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
#docker-official-library:off
|
||||
|
||||
32
docs/changelogs/v24.7.5.37-stable.md
Normal file
32
docs/changelogs/v24.7.5.37-stable.md
Normal file
@ -0,0 +1,32 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.7.5.37-stable (f2533ca97be) FIXME as compared to v24.7.4.51-stable (70fe2f6fa52)
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#68971](https://github.com/ClickHouse/ClickHouse/issues/68971): Fix the upper bound of the function `fromModifiedJulianDay`. It was supposed to be `9999-12-31` but was mistakenly set to `9999-01-01`. [#67583](https://github.com/ClickHouse/ClickHouse/pull/67583) ([PHO](https://github.com/depressed-pho)).
|
||||
* Backported in [#68816](https://github.com/ClickHouse/ClickHouse/issues/68816): Fixed crash in Parquet filtering when data types in the file substantially differ from requested types (e.g. `... FROM file('a.parquet', Parquet, 'x String')`, but the file has `x Int64`). Without this fix, use `input_format_parquet_filter_push_down = 0` as a workaround. [#68131](https://github.com/ClickHouse/ClickHouse/pull/68131) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Backported in [#69006](https://github.com/ClickHouse/ClickHouse/issues/69006): After https://github.com/ClickHouse/ClickHouse/pull/61984 `schema_inference_make_columns_nullable=0` still can make columns `Nullable` in Parquet/Arrow formats. The change was backward incompatible and users noticed the changes in the behaviour. This PR makes `schema_inference_make_columns_nullable=0` to work as before (no Nullable columns will be inferred) and introduces new value `auto` for this setting that will make columns `Nullable` only if data has information about nullability. [#68298](https://github.com/ClickHouse/ClickHouse/pull/68298) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#68884](https://github.com/ClickHouse/ClickHouse/issues/68884): Fixes [#50868](https://github.com/ClickHouse/ClickHouse/issues/50868). Small DateTime64 constant values returned by a nested subquery inside a distributed query were wrongly transformed to Nulls, thus causing errors and possible incorrect query results. [#68323](https://github.com/ClickHouse/ClickHouse/pull/68323) ([Shankar](https://github.com/shiyer7474)).
|
||||
* Backported in [#69027](https://github.com/ClickHouse/ClickHouse/issues/69027): Added back virtual columns ` _table` and `_database` to distributed tables. They were available until version 24.3. [#68672](https://github.com/ClickHouse/ClickHouse/pull/68672) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#68862](https://github.com/ClickHouse/ClickHouse/issues/68862): Fix possible error `Size of permutation (0) is less than required (...)` during Variant column permutation. [#68681](https://github.com/ClickHouse/ClickHouse/pull/68681) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#68788](https://github.com/ClickHouse/ClickHouse/issues/68788): Fix issue with materialized constant keys when hashing maps with arrays as keys in functions `sipHash(64/128)Keyed`. [#68731](https://github.com/ClickHouse/ClickHouse/pull/68731) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
|
||||
* Backported in [#68909](https://github.com/ClickHouse/ClickHouse/issues/68909): Fix complex types metadata parsing in DeltaLake. Closes [#68739](https://github.com/ClickHouse/ClickHouse/issues/68739). [#68836](https://github.com/ClickHouse/ClickHouse/pull/68836) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#69158](https://github.com/ClickHouse/ClickHouse/issues/69158): Fix possible wrong result during anyHeavy state merge. [#68950](https://github.com/ClickHouse/ClickHouse/pull/68950) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69118](https://github.com/ClickHouse/ClickHouse/issues/69118): Fix logical error when we have empty async insert. [#69080](https://github.com/ClickHouse/ClickHouse/pull/69080) ([Han Fei](https://github.com/hanfei1991)).
|
||||
|
||||
#### NO CL CATEGORY
|
||||
|
||||
* Backported in [#68944](https://github.com/ClickHouse/ClickHouse/issues/68944):. [#68897](https://github.com/ClickHouse/ClickHouse/pull/68897) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#68832](https://github.com/ClickHouse/ClickHouse/issues/68832): Turn off fault injection for insert in `01396_inactive_replica_cleanup_nodes_zookeeper`. [#68715](https://github.com/ClickHouse/ClickHouse/pull/68715) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#68779](https://github.com/ClickHouse/ClickHouse/issues/68779): Fix flaky test 00989_parallel_parts_loading. [#68737](https://github.com/ClickHouse/ClickHouse/pull/68737) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#68960](https://github.com/ClickHouse/ClickHouse/issues/68960): Fix 2477 timeout. [#68752](https://github.com/ClickHouse/ClickHouse/pull/68752) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Backported in [#69050](https://github.com/ClickHouse/ClickHouse/issues/69050): Fix 01114_database_atomic flakiness. [#68930](https://github.com/ClickHouse/ClickHouse/pull/68930) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
500
docs/changelogs/v24.9.1.3278-stable.md
Normal file
500
docs/changelogs/v24.9.1.3278-stable.md
Normal file
@ -0,0 +1,500 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.9.1.3278-stable (6d058d82a8e) FIXME as compared to v24.9.1.1-new (e02b434d2fc)
|
||||
|
||||
#### Backward Incompatible Change
|
||||
* Allow to write `SETTINGS` before `FORMAT` in a chain of queries with `UNION` when subqueries are inside parentheses. This closes [#39712](https://github.com/ClickHouse/ClickHouse/issues/39712). Change the behavior when a query has the SETTINGS clause specified twice in a sequence. The closest SETTINGS clause will have a preference for the corresponding subquery. In the previous versions, the outermost SETTINGS clause could take a preference over the inner one. [#60197](https://github.com/ClickHouse/ClickHouse/pull/60197) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Do not allow explicitly specifying UUID when creating a table in Replicated database. Also, do not allow explicitly specifying ZooKeeper path and replica name for *MergeTree tables in Replicated databases. [#66104](https://github.com/ClickHouse/ClickHouse/pull/66104) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Reimplement Dynamic type. Now when the limit of dynamic data types is reached new types are not casted to String but stored in a special data structure in binary format with binary encoded data type. Now any type ever inserted into Dynamic column can be read from it as subcolumn. [#68132](https://github.com/ClickHouse/ClickHouse/pull/68132) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Expressions like `a[b].c` are supported for named tuples, as well as named subscripts from arbitrary expressions, e.g., `expr().name`. This is useful for processing JSON. This closes [#54965](https://github.com/ClickHouse/ClickHouse/issues/54965). In previous versions, an expression of form `expr().name` was parsed as `tupleElement(expr(), name)`, and the query analyzer was searching for a column `name` rather than for the corresponding tuple element; while in the new version, it is changed to `tupleElement(expr(), 'name')`. In most cases, the previous version was not working, but it is possible to imagine a very unusual scenario when this change could lead to incompatibility: if you stored names of tuple elements in a column or an alias, that was named differently than the tuple element's name: `SELECT 'b' AS a, CAST([tuple(123)] AS 'Array(Tuple(b UInt8))') AS t, t[1].a`. It is very unlikely that you used such queries, but we still have to mark this change as potentially backward incompatible. [#68435](https://github.com/ClickHouse/ClickHouse/pull/68435) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* When the setting `print_pretty_type_names` is enabled, it will print `Tuple` data type in a pretty form in `SHOW CREATE TABLE` statements, `formatQuery` function, and in the interactive mode in `clickhouse-client` and `clickhouse-local`. In previous versions, this setting was only applied to `DESCRIBE` queries and `toTypeName`. This closes [#65753](https://github.com/ClickHouse/ClickHouse/issues/65753). [#68492](https://github.com/ClickHouse/ClickHouse/pull/68492) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### New Feature
|
||||
* Function `toStartOfInterval()` now has a new overload which emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function. ([#55619](https://github.com/ClickHouse/ClickHouse/issues/55619)). It allows to align date or timestamp values to multiples of a given interval from an *arbitrary* origin (instead of 0000-01-01 00:00:00.000 as *fixed* origin). For example, `SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));` returns `2023-01-01 14:44:30` which is a multiple of 1 minute intervals, starting from origin `2023-01-01 14:35:30`. [#56738](https://github.com/ClickHouse/ClickHouse/pull/56738) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Add support for `ATTACH PARTITION ALL FROM`. [#61987](https://github.com/ClickHouse/ClickHouse/pull/61987) ([Kirill Nikiforov](https://github.com/allmazz)).
|
||||
* Adds a setting `input_format_try_infer_variants` which allows Variant type to be inferred during schema inference for text formats when there is more than one possible type for column/array elements. [#63798](https://github.com/ClickHouse/ClickHouse/pull/63798) ([Shaun Struwig](https://github.com/Blargian)).
|
||||
* Introduced JSONCompactWithProgress format where ClickHouse outputs each row as a newline-delimited JSON object, including metadata, data, progress, totals, and statistics. [#66205](https://github.com/ClickHouse/ClickHouse/pull/66205) ([Alexey Korepanov](https://github.com/alexkorep)).
|
||||
* Implement new JSON data type. [#66444](https://github.com/ClickHouse/ClickHouse/pull/66444) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Add the `input_format_json_empty_as_default` setting which, when enabled, treats empty fields in JSON inputs as default values. Closes [#59339](https://github.com/ClickHouse/ClickHouse/issues/59339). [#66782](https://github.com/ClickHouse/ClickHouse/pull/66782) ([Alexis Arnaud](https://github.com/a-a-f)).
|
||||
* Added functions `overlay` and `overlayUTF8` which replace parts of a string by another string. Example: `SELECT overlay('Hello New York', 'Jersey', 11)` returns `Hello New Jersey`. [#66933](https://github.com/ClickHouse/ClickHouse/pull/66933) ([李扬](https://github.com/taiyang-li)).
|
||||
* Add new Command, Lightweight Delete In Partition ``` DELETE FROM [db.]table [ON CLUSTER cluster] [IN PARTITION partition_expr] WHERE expr; ``` ``` VM-114-29-tos :) select * from ads_app_poster_ip_source_channel_di_replicated_local;. [#67805](https://github.com/ClickHouse/ClickHouse/pull/67805) ([sunny](https://github.com/sunny19930321)).
|
||||
* Implemented comparison for `Interval` data type values so they are converting now to the least supertype. [#68057](https://github.com/ClickHouse/ClickHouse/pull/68057) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Add create_if_not_exists setting to default to IF NOT EXISTS behavior during CREATE statements. [#68164](https://github.com/ClickHouse/ClickHouse/pull/68164) ([Peter Nguyen](https://github.com/petern48)).
|
||||
* Makes possible to read Iceberg tables in Azure and locally. [#68210](https://github.com/ClickHouse/ClickHouse/pull/68210) ([Daniil Ivanik](https://github.com/divanik)).
|
||||
* Add aggregate functions distinctDynamicTypes/distinctJSONPaths/distinctJSONPathsAndTypes for better introspection of JSON column type content. [#68463](https://github.com/ClickHouse/ClickHouse/pull/68463) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Query cache entries can now be dropped by tag. For example, the query cache entry created by `SELECT 1 SETTINGS use_query_cache = true, query_cache_tag = 'abc'` can now be dropped by `SYSTEM DROP QUERY CACHE TAG 'abc'` (or of course just: `SYSTEM DROP QUERY CACHE` which will clear the entire query cache). [#68477](https://github.com/ClickHouse/ClickHouse/pull/68477) ([Michał Tabaszewski](https://github.com/pinsvin00)).
|
||||
* Add storage encryption for named collections. [#68615](https://github.com/ClickHouse/ClickHouse/pull/68615) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Added `ripeMD160` function, which computes the RIPEMD-160 cryptographic hash of a string. Example: `SELECT hex(ripeMD160('The quick brown fox jumps over the lazy dog'))` returns `37F332F68DB77BD9D7EDD4969571AD671CF9DD3B`. [#68639](https://github.com/ClickHouse/ClickHouse/pull/68639) ([Dergousov Maxim](https://github.com/m7kss1)).
|
||||
* Add virtual column _headers for url table engine. Closes [#65026](https://github.com/ClickHouse/ClickHouse/issues/65026). [#68867](https://github.com/ClickHouse/ClickHouse/pull/68867) ([flynn](https://github.com/ucasfl)).
|
||||
* Adding `system.projections` table to track available projections. [#68901](https://github.com/ClickHouse/ClickHouse/pull/68901) ([Jordi Villar](https://github.com/jrdi)).
|
||||
* Add new function `arrayZipUnaligned` for spark compatiablity(arrays_zip), which allowed unaligned arrays based on original `arrayZip`. ``` sql SELECT arrayZipUnaligned([1], [1, 2, 3]). [#69030](https://github.com/ClickHouse/ClickHouse/pull/69030) ([李扬](https://github.com/taiyang-li)).
|
||||
* Adding `RealTimeMicroseconds` metric in the HTTP Header `X-ClickHouse-Summary`. This way we can know the CPU time of a request without having to go to check it in `system.query_log`. [#69032](https://github.com/ClickHouse/ClickHouse/pull/69032) ([Alejandro](https://github.com/alexon1234)).
|
||||
* Added cp/mv commands for keeper client which atomically copies/moves node. [#69034](https://github.com/ClickHouse/ClickHouse/pull/69034) ([Mikhail Artemenko](https://github.com/Michicosun)).
|
||||
* Adds argument `scale` (default: `true`) to function `arrayAUC` which allows to skip the normalization step (issue [#69609](https://github.com/ClickHouse/ClickHouse/issues/69609)). [#69717](https://github.com/ClickHouse/ClickHouse/pull/69717) ([gabrielmcg44](https://github.com/gabrielmcg44)).
|
||||
|
||||
#### Performance Improvement
|
||||
* Improve the join performance by rerange the right table by keys while the table keys are dense in left or inner hash join. [#60341](https://github.com/ClickHouse/ClickHouse/pull/60341) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Improve all join perfromance by append `RowRefList` or `RowRef` to AddedColumns for lazy output, while buildOutput, we use `RowRefList`/`RowRef` for output, and remove `is_join_get` condition from `buildOutput` for loop. [#63677](https://github.com/ClickHouse/ClickHouse/pull/63677) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Load filesystem cache metadata asynchronously to boot process, in order to make restarts faster. [#65736](https://github.com/ClickHouse/ClickHouse/pull/65736) ([Daniel Pozo Escalona](https://github.com/danipozo)).
|
||||
* Functions `array` and `map` were optimized to process certain common cases much faster. [#67707](https://github.com/ClickHouse/ClickHouse/pull/67707) ([李扬](https://github.com/taiyang-li)).
|
||||
* Trivial optimize on orc string reading especially when column contains no NULLs. [#67794](https://github.com/ClickHouse/ClickHouse/pull/67794) ([李扬](https://github.com/taiyang-li)).
|
||||
* Improved overall performance of merges by reducing the overhead of scheduling steps of merges. [#68016](https://github.com/ClickHouse/ClickHouse/pull/68016) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Speed up requests to S3 when a profile is not set, credentials are not set, and IMDS is not available (for example, when you are querying a public bucket on a machine outside of a cloud). This closes [#52771](https://github.com/ClickHouse/ClickHouse/issues/52771). [#68082](https://github.com/ClickHouse/ClickHouse/pull/68082) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* New algorithm to determine the unit of marks distribution between replicas by consistent hash. Different numbers of marks chosen for different read patterns to improve performance. [#68424](https://github.com/ClickHouse/ClickHouse/pull/68424) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Try to devirtualize format reader in RowInputFormatWithNamesAndTypes, and see if it could bring some performance improvement. [#68437](https://github.com/ClickHouse/ClickHouse/pull/68437) ([李扬](https://github.com/taiyang-li)).
|
||||
* Add the parallel merge with key implementation to maximize the CPU utilization. [#68441](https://github.com/ClickHouse/ClickHouse/pull/68441) ([Jiebin Sun](https://github.com/jiebinn)).
|
||||
* Add settings `output_format_orc_dictionary_key_size_threshold` to allow user to enable dict encoding for string column in ORC output format. It helps reduce the output orc file size and improve reading performance significantly. [#68591](https://github.com/ClickHouse/ClickHouse/pull/68591) ([李扬](https://github.com/taiyang-li)).
|
||||
* Implemented reading of required files only during hive partitioning. [#68963](https://github.com/ClickHouse/ClickHouse/pull/68963) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Introduce new Keeper request RemoveRecursive which removes node with all it's subtree. [#69332](https://github.com/ClickHouse/ClickHouse/pull/69332) ([Mikhail Artemenko](https://github.com/Michicosun)).
|
||||
* Speedup insert performance with vector similarity index by adding data to vector index parallel. [#69493](https://github.com/ClickHouse/ClickHouse/pull/69493) ([flynn](https://github.com/ucasfl)).
|
||||
* Previously the algorithmic complexity of part deduplication logic in parallel replica announcement handling was O(n^2) which could take noticeable time for tables with many part (or partitions). This change makes the complexity O(n*log(n)). [#69596](https://github.com/ClickHouse/ClickHouse/pull/69596) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
|
||||
#### Improvement
|
||||
* Hardened parts of the codebase related to parsing of small entities. The following (minor) bugs were found and fixed: - if a `DeltaLake` table is partitioned by Bool, the partition value is always interpreted as false; - `ExternalDistributed` table was using only a single shard in the provided addresses; the value of `max_threads` setting and similar were printed as `'auto(N)'` instead of `auto(N)`. [#52503](https://github.com/ClickHouse/ClickHouse/pull/52503) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Refreshable materialized view improvements: append mode (`... REFRESH EVERY 1 MINUTE APPEND ...`) to add rows to existing table instead of overwriting the whole table, retries (disabled by default, configured in SETTINGS section of the query), `SYSTEM WAIT VIEW <name>` query that waits for the currently running refresh, some fixes. [#58934](https://github.com/ClickHouse/ClickHouse/pull/58934) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Use cgroup-specific metrics for CPU usage accounting instead of system-wide metrics. [#62003](https://github.com/ClickHouse/ClickHouse/pull/62003) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* IO scheduling for remote S3 disks is now done on the level of HTTP socket streams (instead of the whole S3 requests) to resolve `bandwidth_limit` throttling issues. [#65182](https://github.com/ClickHouse/ClickHouse/pull/65182) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Allow a user to have multiple authentication methods instead of only one. Allow authentication methods to be reset to most recently added method. If you want to run instances on 24.8 and one on 24.9 for some time, it's better to set `max_authentication_methods_per_user = 1` for that period to avoid potential errors. [#65277](https://github.com/ClickHouse/ClickHouse/pull/65277) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Functions `upperUTF8` and `lowerUTF8` were previously only able to uppercase / lowercase Cyrillic characters. This limitation is now removed and characters in arbitrary languages are uppercased/lowercased. Example: `SELECT upperUTF8('Süden')` now returns `SÜDEN`. [#65761](https://github.com/ClickHouse/ClickHouse/pull/65761) ([李扬](https://github.com/taiyang-li)).
|
||||
* When lightweight delete happens on a table with projection(s), despite users have choices either throw an exception (by default) or drop the projection lightweight delete would happen, now the third option is to still have lightweight delete and then rebuild projection(s). [#66169](https://github.com/ClickHouse/ClickHouse/pull/66169) ([Shichao](https://github.com/jsc0218)).
|
||||
* Two options (`dns_allow_resolve_names_to_ipv4` and `dns_allow_resolve_names_to_ipv6`) have been added, to allow block connections ip family. [#66895](https://github.com/ClickHouse/ClickHouse/pull/66895) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Added `min_max`as a new type of (experimental) statistics. It supports estimating range predicates over numeric columns, e.g. `x < 100`. [#67013](https://github.com/ClickHouse/ClickHouse/pull/67013) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Make C-z ignorance configurable (ignore_shell_suspend) in clickhouse-client. [#67134](https://github.com/ClickHouse/ClickHouse/pull/67134) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Improve castOrDefault from Variant/Dynamic columns so it works when inner types are not convertable at all. [#67150](https://github.com/ClickHouse/ClickHouse/pull/67150) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Improve unicode encoding in JSON output formats. Ensures that valid JSON is generated in the case of certain byte sequences in the result data. [#67938](https://github.com/ClickHouse/ClickHouse/pull/67938) ([mwoenker](https://github.com/mwoenker)).
|
||||
* Added profile events for merges and mutations for better introspection. [#68015](https://github.com/ClickHouse/ClickHouse/pull/68015) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Odbc: get http_max_tries from server configuration. [#68128](https://github.com/ClickHouse/ClickHouse/pull/68128) ([Rodolphe Dugé de Bernonville](https://github.com/RodolpheDuge)).
|
||||
* Add wildcard support for user identification in x509 SubjectAltName extension. [#68236](https://github.com/ClickHouse/ClickHouse/pull/68236) ([Marco Vilas Boas](https://github.com/marco-vb)).
|
||||
* Improve schema inference of date times. Now DateTime64 used only when date time has fractional part, otherwise regular DateTime is used. Inference of Date/DateTime is more strict now, especially when `date_time_input_format='best_effort'` to avoid inferring date times from strings in corner cases. [#68382](https://github.com/ClickHouse/ClickHouse/pull/68382) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Delete old code of named collections from dictionaries and substitute it to the new, which allows to use DDL created named collections in dictionaries. Closes [#60936](https://github.com/ClickHouse/ClickHouse/issues/60936), closes [#36890](https://github.com/ClickHouse/ClickHouse/issues/36890). [#68412](https://github.com/ClickHouse/ClickHouse/pull/68412) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Use HTTP/1.1 instead of HTTP/1.0 (set by default) for external HTTP authentication. [#68456](https://github.com/ClickHouse/ClickHouse/pull/68456) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Functions `upperUTF8` and `lowerUTF8` were previously only able to uppercase / lowercase Cyrillic characters. This limitation is now removed and arbitrary characters are uppercased/lowercased. Example: `SELECT upperUTF8('Süden')` now returns `SÜDEN`. [#68523](https://github.com/ClickHouse/ClickHouse/pull/68523) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Added a new set of metrics for Thread Pool introspection, providing deeper insights into thread pool performance and behavior. [#68674](https://github.com/ClickHouse/ClickHouse/pull/68674) ([filimonov](https://github.com/filimonov)).
|
||||
* Support query parameters in async inserts with format `Values`. [#68741](https://github.com/ClickHouse/ClickHouse/pull/68741) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Support Date32 on dateTrunc and toStartOfInterval. [#68874](https://github.com/ClickHouse/ClickHouse/pull/68874) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Add a new setting: `output_format_always_quote_identifiers` to always quote identifiers if it is enabled. - Add a new setting: `output_format_identifier_quoting_style` to set the identifier quoting style. Applicable values: `'None'`, `'Backticks'` (default), `'DoubleQuotes'`, `'BackticksMySQL'`. [#68896](https://github.com/ClickHouse/ClickHouse/pull/68896) ([tuanpach](https://github.com/tuanpach)).
|
||||
* Add `plan_step_name` and `plan_step_description` columns to `system.processors_profile_log`. [#68954](https://github.com/ClickHouse/ClickHouse/pull/68954) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Support for the Spanish language in the embedded dictionaries. [#69035](https://github.com/ClickHouse/ClickHouse/pull/69035) ([Vasily Okunev](https://github.com/VOkunev)).
|
||||
* Add CPU arch to short fault info. [#69037](https://github.com/ClickHouse/ClickHouse/pull/69037) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* CREATE TABLE AS copy PRIMARY KEY, ORDER BY, and similar clauses. Now it supports only for MergeTree family of table engines. [#69076](https://github.com/ClickHouse/ClickHouse/pull/69076) ([sakulali](https://github.com/sakulali)).
|
||||
* Replication of subset of columns is now available through MaterializedPostgreSQL. Closes [#33748](https://github.com/ClickHouse/ClickHouse/issues/33748). [#69092](https://github.com/ClickHouse/ClickHouse/pull/69092) ([Kruglov Kirill](https://github.com/1on)).
|
||||
* Add `std::string getHost(const std::string & key) const;` method to "base/poco/Util/include/Poco/Util/AbstractConfiguration.h" which does the same as `Poco::Util::AbstractConfiguration::getString` method but additionally does validity check to make sure value is a correct IP address or domain name. [#69130](https://github.com/ClickHouse/ClickHouse/pull/69130) ([Maxim](https://github.com/m4xxx1m)).
|
||||
* Do not block retries when establishing a new keeper connection. [#69148](https://github.com/ClickHouse/ClickHouse/pull/69148) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Update DatabaseFactory so it would be possible for user defined database engines to have arguments, settings and table overrides (similar to StorageFactory). [#69201](https://github.com/ClickHouse/ClickHouse/pull/69201) ([Барыкин Никита Олегович](https://github.com/NikBarykin)).
|
||||
* Restore mode that replaces all external table engines and functions to Null (`restore_replace_external_engines_to_null`, `restore_replace_external_table_functions_to_null` settings) was failing if table had SETTINGS. Now it removes settings from table definition in this case and allows to restore such tables. [#69253](https://github.com/ClickHouse/ClickHouse/pull/69253) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Reduce memory usage of inserts to JSON by using adaptive write buffer size. A lot of files created by JSON column in wide part contains small amount of data and it doesn't make sense to allocate 1MB buffer for them. [#69272](https://github.com/ClickHouse/ClickHouse/pull/69272) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
|
||||
* Added user-level settings `min_free_disk_bytes_to_throw_insert` and `min_free_disk_ratio_to_throw_insert` to prevent insertions on disks that are almost full. [#69376](https://github.com/ClickHouse/ClickHouse/pull/69376) ([Marco Vilas Boas](https://github.com/marco-vb)).
|
||||
* Not retaining thread in concurrent hash join threadpool to avoid query excessively spawn threads. [#69406](https://github.com/ClickHouse/ClickHouse/pull/69406) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Allow empty arguments for arrayZip/arrayZipUnaligned, as concat did in https://github.com/ClickHouse/ClickHouse/pull/65887. It is for spark compatiability in Gluten CH Backend. [#69576](https://github.com/ClickHouse/ClickHouse/pull/69576) ([李扬](https://github.com/taiyang-li)).
|
||||
* Support more advanced SSL options for Keeper's internal communication (e.g. private keys with passphrase). [#69582](https://github.com/ClickHouse/ClickHouse/pull/69582) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Sleep for 10ms before retrying to acquire a lock in `databasereplicatedddlworker::enqueuequeryimpl`. [#69588](https://github.com/ClickHouse/ClickHouse/pull/69588) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Index analysis can take noticeable time for big tables with many parts or partitions. This change should enable killing a heavy query at that stage. [#69606](https://github.com/ClickHouse/ClickHouse/pull/69606) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Masking sensitive info in `gcs()` table function. [#69611](https://github.com/ClickHouse/ClickHouse/pull/69611) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Sync changes to `interpreterdropquery::executetodatabaseimpl` from the private fork. [#69670](https://github.com/ClickHouse/ClickHouse/pull/69670) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#69905](https://github.com/ClickHouse/ClickHouse/issues/69905): Change the join to sort settings type to unsigned int. [#69886](https://github.com/ClickHouse/ClickHouse/pull/69886) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Rebuild projection for merges that reduce number of rows. [#62364](https://github.com/ClickHouse/ClickHouse/pull/62364) ([cangyin](https://github.com/cangyin)).
|
||||
* Fix attaching table when pg dbname contains "-" in MaterializedPostgreSQL. [#62730](https://github.com/ClickHouse/ClickHouse/pull/62730) ([takakawa](https://github.com/takakawa)).
|
||||
* Storage Join support for nullable columns in left table, close [#61247](https://github.com/ClickHouse/ClickHouse/issues/61247). [#66926](https://github.com/ClickHouse/ClickHouse/pull/66926) ([vdimir](https://github.com/vdimir)).
|
||||
* Incorrect query result with parallel replicas (distribute queries as well) when `IN` operator contains conversion to Decimal(). The bug was introduced with the new analyzer. [#67234](https://github.com/ClickHouse/ClickHouse/pull/67234) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix the problem that alter modfiy order by causes inconsistent metadata. [#67436](https://github.com/ClickHouse/ClickHouse/pull/67436) ([iceFireser](https://github.com/iceFireser)).
|
||||
* Fix the upper bound of the function `fromModifiedJulianDay`. It was supposed to be `9999-12-31` but was mistakenly set to `9999-01-01`. [#67583](https://github.com/ClickHouse/ClickHouse/pull/67583) ([PHO](https://github.com/depressed-pho)).
|
||||
* Fix when the index is not at the beginning of the tuple during `IN` query. [#67626](https://github.com/ClickHouse/ClickHouse/pull/67626) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fixed error on generated columns in MaterializedPostgreSQL when adnum ordering is broken [#63161](https://github.com/ClickHouse/ClickHouse/issues/63161). Fixed error on id column with nextval expression as default MaterializedPostgreSQL when there are generated columns in table. Fixed error on dropping publication with symbols except [a-z1-9-]. [#67664](https://github.com/ClickHouse/ClickHouse/pull/67664) ([Kruglov Kirill](https://github.com/1on)).
|
||||
* Fix expiration in `RoleCache`. [#67748](https://github.com/ClickHouse/ClickHouse/pull/67748) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix window view missing blocks due to slow flush to view. [#67983](https://github.com/ClickHouse/ClickHouse/pull/67983) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix MSAN issue caused by incorrect date format. [#68105](https://github.com/ClickHouse/ClickHouse/pull/68105) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Fixed crash in Parquet filtering when data types in the file substantially differ from requested types (e.g. `... FROM file('a.parquet', Parquet, 'x String')`, but the file has `x Int64`). Without this fix, use `input_format_parquet_filter_push_down = 0` as a workaround. [#68131](https://github.com/ClickHouse/ClickHouse/pull/68131) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix crash in `lag`/`lead` which is introduced in [#67091](https://github.com/ClickHouse/ClickHouse/issues/67091). [#68262](https://github.com/ClickHouse/ClickHouse/pull/68262) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* Try fix postgres crash when query is cancelled. [#68288](https://github.com/ClickHouse/ClickHouse/pull/68288) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* After https://github.com/ClickHouse/ClickHouse/pull/61984 `schema_inference_make_columns_nullable=0` still can make columns `Nullable` in Parquet/Arrow formats. The change was backward incompatible and users noticed the changes in the behaviour. This PR makes `schema_inference_make_columns_nullable=0` to work as before (no Nullable columns will be inferred) and introduces new value `auto` for this setting that will make columns `Nullable` only if data has information about nullability. [#68298](https://github.com/ClickHouse/ClickHouse/pull/68298) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fixes [#50868](https://github.com/ClickHouse/ClickHouse/issues/50868). Small DateTime64 constant values returned by a nested subquery inside a distributed query were wrongly transformed to Nulls, thus causing errors and possible incorrect query results. [#68323](https://github.com/ClickHouse/ClickHouse/pull/68323) ([Shankar](https://github.com/shiyer7474)).
|
||||
* Fix missing sync replica mode in query `SYSTEM SYNC REPLICA`. [#68326](https://github.com/ClickHouse/ClickHouse/pull/68326) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Fix bug in key condition. [#68354](https://github.com/ClickHouse/ClickHouse/pull/68354) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fix crash on drop or rename a role that is used in LDAP external user directory. [#68355](https://github.com/ClickHouse/ClickHouse/pull/68355) ([Andrey Zvonov](https://github.com/zvonand)).
|
||||
* Fix Progress column value of system.view_refreshes greater than 1 [#68377](https://github.com/ClickHouse/ClickHouse/issues/68377). [#68378](https://github.com/ClickHouse/ClickHouse/pull/68378) ([megao](https://github.com/jetgm)).
|
||||
* Process regexp flags correctly. [#68389](https://github.com/ClickHouse/ClickHouse/pull/68389) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* PostgreSQL-style cast operator (`::`) works correctly even for SQL-style hex and binary string literals (e.g., `SELECT x'414243'::String`). This closes [#68324](https://github.com/ClickHouse/ClickHouse/issues/68324). [#68482](https://github.com/ClickHouse/ClickHouse/pull/68482) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Minor patch for https://github.com/ClickHouse/ClickHouse/pull/68131. [#68494](https://github.com/ClickHouse/ClickHouse/pull/68494) ([Chang chen](https://github.com/baibaichen)).
|
||||
* Fix [#68239](https://github.com/ClickHouse/ClickHouse/issues/68239) SAMPLE n where n is an integer. [#68499](https://github.com/ClickHouse/ClickHouse/pull/68499) ([Denis Hananein](https://github.com/denis-hananein)).
|
||||
* Fix bug in mann-whitney-utest when the size of two districutions are not equal. [#68556](https://github.com/ClickHouse/ClickHouse/pull/68556) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* After unexpected restart, fail to start replication of ReplicatedMergeTree due to abnormal handling of covered-by-broken part. [#68584](https://github.com/ClickHouse/ClickHouse/pull/68584) ([baolin](https://github.com/baolinhuang)).
|
||||
* Fix `LOGICAL_ERROR`s when functions `sipHash64Keyed`, `sipHash128Keyed`, or `sipHash128ReferenceKeyed` are applied to empty arrays or tuples. [#68630](https://github.com/ClickHouse/ClickHouse/pull/68630) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Full text index may filter out wrong columns when index multiple columns, it didn't reset row_id between different columns, the reproduce procedure is in tests/queries/0_stateless/03228_full_text_with_multi_col.sql. Without this. [#68644](https://github.com/ClickHouse/ClickHouse/pull/68644) ([siyuan](https://github.com/linkwk7)).
|
||||
* Fix invalid character '\t' and '\n' in replica_name when creating a Replicated table, which causes incorrect parsing of 'source replica' in LogEntry. Mentioned in issue [#68640](https://github.com/ClickHouse/ClickHouse/issues/68640). [#68645](https://github.com/ClickHouse/ClickHouse/pull/68645) ([Zhigao Hong](https://github.com/zghong)).
|
||||
* Added back virtual columns ` _table` and `_database` to distributed tables. They were available until version 24.3. [#68672](https://github.com/ClickHouse/ClickHouse/pull/68672) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix possible error `Size of permutation (0) is less than required (...)` during Variant column permutation. [#68681](https://github.com/ClickHouse/ClickHouse/pull/68681) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix possible error `DB::Exception: Block structure mismatch in joined block stream: different columns:` with new JSON column. [#68686](https://github.com/ClickHouse/ClickHouse/pull/68686) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix issue with materialized constant keys when hashing maps with arrays as keys in functions `sipHash(64/128)Keyed`. [#68731](https://github.com/ClickHouse/ClickHouse/pull/68731) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
|
||||
* Make `ColumnsDescription::toString` format each column using the same `IAST::FormatState object`. This results in uniform columns metadata being written to disk and ZooKeeper. [#68733](https://github.com/ClickHouse/ClickHouse/pull/68733) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Fix merging of aggregated data for grouping sets. [#68744](https://github.com/ClickHouse/ClickHouse/pull/68744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix logical error, when we create a replicated merge tree, alter a column and then execute modify statistics. [#68820](https://github.com/ClickHouse/ClickHouse/pull/68820) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fix resolving dynamic subcolumns from subqueries in analyzer. [#68824](https://github.com/ClickHouse/ClickHouse/pull/68824) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix complex types metadata parsing in DeltaLake. Closes [#68739](https://github.com/ClickHouse/ClickHouse/issues/68739). [#68836](https://github.com/ClickHouse/ClickHouse/pull/68836) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fixed asynchronous inserts in case when metadata of table is changed (by `ALTER ADD/MODIFY COLUMN` queries) after insert but before flush to the table. [#68837](https://github.com/ClickHouse/ClickHouse/pull/68837) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix unexpected exception when passing empty tuple in array. This fixes [#68618](https://github.com/ClickHouse/ClickHouse/issues/68618). [#68848](https://github.com/ClickHouse/ClickHouse/pull/68848) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix parsing pure metadata mutations commands. [#68935](https://github.com/ClickHouse/ClickHouse/pull/68935) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Fix possible wrong result during anyHeavy state merge. [#68950](https://github.com/ClickHouse/ClickHouse/pull/68950) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fixed writing to Materialized Views with enabled setting `optimize_functions_to_subcolumns`. [#68951](https://github.com/ClickHouse/ClickHouse/pull/68951) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Don't use serializations cache in const Dynamic column methods. It could let to use-of-unitialized value or even race condition during aggregations. [#68953](https://github.com/ClickHouse/ClickHouse/pull/68953) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix parsing error when null should be inserted as default in some cases during JSON type parsing. [#68955](https://github.com/ClickHouse/ClickHouse/pull/68955) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* There were cases when path was concatenated incorrectly and had the `//` part in it, solving this problem using path normalization. [#69066](https://github.com/ClickHouse/ClickHouse/pull/69066) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fix logical error when we have empty async insert. [#69080](https://github.com/ClickHouse/ClickHouse/pull/69080) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fixed data race of progress indication in clickhouse-client during query canceling. [#69081](https://github.com/ClickHouse/ClickHouse/pull/69081) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Fix a bug that the vector similarity index (currently experimental) was not utilized when used with cosine distance as distance function. [#69090](https://github.com/ClickHouse/ClickHouse/pull/69090) ([flynn](https://github.com/ucasfl)).
|
||||
* This change addresses an issue where attempting to create a Replicated database again after a server failure during the initial creation process could result in error. [#69102](https://github.com/ClickHouse/ClickHouse/pull/69102) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Don't infer Bool type from String in CSV when `input_format_csv_try_infer_numbers_from_strings = 1` because we don't allow reading bool values from strings. [#69109](https://github.com/ClickHouse/ClickHouse/pull/69109) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix explain ast insert queries parsing errors on client when `--multiquery` is enabled. [#69123](https://github.com/ClickHouse/ClickHouse/pull/69123) ([wxybear](https://github.com/wxybear)).
|
||||
* `UNION` clause in subqueries wasn't handled correctly in queries with parallel replicas and lead to LOGICAL_ERROR `Duplicate announcement received for replica`. [#69146](https://github.com/ClickHouse/ClickHouse/pull/69146) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix propogating structure argument in s3Cluster. Previously the `DEFAULT` expression of the column could be lost when sending the query to the replicas in s3Cluster. [#69147](https://github.com/ClickHouse/ClickHouse/pull/69147) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Respect format settings in Values format during conversion from expression to the destination type. [#69149](https://github.com/ClickHouse/ClickHouse/pull/69149) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix `clickhouse-client --queries-file` for readonly users (previously fails with `Cannot modify 'log_comment' setting in readonly mode`). [#69175](https://github.com/ClickHouse/ClickHouse/pull/69175) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix data race in clickhouse-client when it's piped to a process that terminated early. [#69186](https://github.com/ClickHouse/ClickHouse/pull/69186) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix incorrect results of Fix uniq and GROUP BY for JSON/Dynamic types. [#69203](https://github.com/ClickHouse/ClickHouse/pull/69203) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix the INFILE format detection for asynchronous inserts. If the format is not explicitly defined in the FORMAT clause, it can be detected from the INFILE file extension. [#69237](https://github.com/ClickHouse/ClickHouse/pull/69237) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* After [this issue](https://github.com/ClickHouse/ClickHouse/pull/59946#issuecomment-1943653197) there are quite a few table replicas in production such that their `metadata_version` node value is both equal to `0` and is different from the respective table's `metadata` node version. This leads to `alter` queries failing on such replicas. [#69274](https://github.com/ClickHouse/ClickHouse/pull/69274) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#69986](https://github.com/ClickHouse/ClickHouse/issues/69986): Fix inf loop after `restore replica` in the replicated merge tree with zero copy. [#69293](https://github.com/ClickHouse/ClickHouse/pull/69293) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Mark Dynamic type as not safe primary key type to avoid issues with Fields. [#69311](https://github.com/ClickHouse/ClickHouse/pull/69311) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Improve restoring of access entities' dependencies. [#69346](https://github.com/ClickHouse/ClickHouse/pull/69346) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix undefined behavior when all connection attempts fail getting a connection for insertions. [#69390](https://github.com/ClickHouse/ClickHouse/pull/69390) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Close [#69135](https://github.com/ClickHouse/ClickHouse/issues/69135). If we try to reuse joined data for `cross` join, but this could not happen in ClickHouse at present. It's better to keep `have_compressed` in `reuseJoinedData`. [#69404](https://github.com/ClickHouse/ClickHouse/pull/69404) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* Make `materialize()` function return full column when parameter is a sparse column. [#69429](https://github.com/ClickHouse/ClickHouse/pull/69429) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Fixed a `LOGICAL_ERROR` with function `sqidDecode` ([#69450](https://github.com/ClickHouse/ClickHouse/issues/69450)). [#69451](https://github.com/ClickHouse/ClickHouse/pull/69451) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Quick fix for s3queue problem on 24.6 or create query with database replicated. [#69454](https://github.com/ClickHouse/ClickHouse/pull/69454) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fixed case when memory consumption was too high because of the squashing in `INSERT INTO ... SELECT` or `CREATE TABLE AS SELECT` queries. [#69469](https://github.com/ClickHouse/ClickHouse/pull/69469) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Statements `SHOW COLUMNS` and `SHOW INDEX` now work properly if the table has dots in its name. [#69514](https://github.com/ClickHouse/ClickHouse/pull/69514) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
|
||||
* Usage of the query cache for queries with an overflow mode != 'throw' is now disallowed. This prevents situations where potentially truncated and incorrect query results could be stored in the query cache. (issue [#67476](https://github.com/ClickHouse/ClickHouse/issues/67476)). [#69549](https://github.com/ClickHouse/ClickHouse/pull/69549) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Keep original order of conditions during move to prewhere. Previously the order could change and it could lead to failing queries when the order is important. [#69560](https://github.com/ClickHouse/ClickHouse/pull/69560) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix Keeper multi-request preprocessing after ZNOAUTH error. [#69627](https://github.com/ClickHouse/ClickHouse/pull/69627) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix METADATA_MISMATCH that might have happened due to TTL with a WHERE clause in DatabaseReplicated when creating a new replica. [#69736](https://github.com/ClickHouse/ClickHouse/pull/69736) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fix `StorageS3(Azure)Queue` settings `tracked_file_ttl_sec`. We wrote it to keeper with key `tracked_file_ttl_sec`, but read as `tracked_files_ttl_sec`, which was a typo. [#69742](https://github.com/ClickHouse/ClickHouse/pull/69742) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Make getHyperrectangleForRowGroup not throw an exception when the data type in parquet file is not convertable into the requested data type. Solved the user's problem when the Parquet file had Decimal64 data type and the column data type was DateTime. [#69745](https://github.com/ClickHouse/ClickHouse/pull/69745) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#70021](https://github.com/ClickHouse/ClickHouse/issues/70021): Fix analyzer default with old compatibility value. [#69895](https://github.com/ClickHouse/ClickHouse/pull/69895) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69943](https://github.com/ClickHouse/ClickHouse/issues/69943): Don't check dependencies during CREATE OR REPLACE VIEW during DROP of old table. Previously CREATE OR REPLACE query failed when there are dependent tables of the recreated view. [#69907](https://github.com/ClickHouse/ClickHouse/pull/69907) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70003](https://github.com/ClickHouse/ClickHouse/issues/70003): Now SQL security will work with parameterized views correctly. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Allow to specify min and max for random settings in the test after tags in a form `setting=(min, max)`. For some tests really low value of a setting can lead to timeout (for example `index_granularity=1`) but we still want to randomize this setting in such tests. To avoid timeouts we can specify custom minimum value for this setting. [#67875](https://github.com/ClickHouse/ClickHouse/pull/67875) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Integration tests flaky check now runs each module(file) as a separate group. It allows to apply timeout per module. Previously slow tests could use all the time and other tests were not run in flaky check with successful result. [#68380](https://github.com/ClickHouse/ClickHouse/pull/68380) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Backported in [#69980](https://github.com/ClickHouse/ClickHouse/issues/69980): Makes dbms independent from clickhouse_functions. [#69914](https://github.com/ClickHouse/ClickHouse/pull/69914) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### NO CL CATEGORY
|
||||
|
||||
* Backported in [#69995](https://github.com/ClickHouse/ClickHouse/issues/69995):. [#69989](https://github.com/ClickHouse/ClickHouse/pull/69989) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### NO CL ENTRY
|
||||
|
||||
* NO CL ENTRY: 'Revert "Revert "Fix AWS ECS""'. [#65362](https://github.com/ClickHouse/ClickHouse/pull/65362) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* NO CL ENTRY: 'Revert "[RFC] Fix settings/current_database in system.processes for async BACKUP/RESTORE"'. [#68386](https://github.com/ClickHouse/ClickHouse/pull/68386) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* NO CL ENTRY: 'Revert "Improve compatibility of `upper/lowerUTF8` with Spark"'. [#68510](https://github.com/ClickHouse/ClickHouse/pull/68510) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* NO CL ENTRY: 'Revert "Fix unexpected behavior with `FORMAT` and `SETTINGS` parsing"'. [#68608](https://github.com/ClickHouse/ClickHouse/pull/68608) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* NO CL ENTRY: 'Revert "Fix prewhere without columns and without adaptive index granularity (almost w/o anything)"'. [#68897](https://github.com/ClickHouse/ClickHouse/pull/68897) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* NO CL ENTRY: 'Revert "Speed up some Kafka tests with multiprocessing"'. [#69356](https://github.com/ClickHouse/ClickHouse/pull/69356) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* NO CL ENTRY: 'Revert "Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. V"'. [#69393](https://github.com/ClickHouse/ClickHouse/pull/69393) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* NO CL ENTRY: 'Revert "Add user-level settings min_free_diskspace_bytes_to_throw_insert and min_free_diskspace_ratio_to_throw_insert"'. [#69705](https://github.com/ClickHouse/ClickHouse/pull/69705) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* NO CL ENTRY: 'Revert "Support more oss endpoints"'. [#69779](https://github.com/ClickHouse/ClickHouse/pull/69779) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Followup for [#56996](https://github.com/ClickHouse/ClickHouse/issues/56996). [#58289](https://github.com/ClickHouse/ClickHouse/pull/58289) ([vdimir](https://github.com/vdimir)).
|
||||
* Add chunked wrapper to native protocol. [#63781](https://github.com/ClickHouse/ClickHouse/pull/63781) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Local plan for parallel replicas. [#64448](https://github.com/ClickHouse/ClickHouse/pull/64448) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Disallow creating refreshable MV on Linux < 3.15. [#64953](https://github.com/ClickHouse/ClickHouse/pull/64953) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Allow query profiler period to be longer than 4 seconds. [#65255](https://github.com/ClickHouse/ClickHouse/pull/65255) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Improved getting of alter conversions for queries. [#65832](https://github.com/ClickHouse/ClickHouse/pull/65832) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Simplify some memory tracking parts. [#66648](https://github.com/ClickHouse/ClickHouse/pull/66648) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* What if we tighten limits for functional tests?. [#66837](https://github.com/ClickHouse/ClickHouse/pull/66837) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix flaky `test_delayed_replica_failover`. [#67541](https://github.com/ClickHouse/ClickHouse/pull/67541) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Fix(asan): access destroyed shared context from handleCrash(). [#67700](https://github.com/ClickHouse/ClickHouse/pull/67700) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. II. [#67749](https://github.com/ClickHouse/ClickHouse/pull/67749) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Make `ColumnLowCardinality::getName()` consistent with other columns. [#67789](https://github.com/ClickHouse/ClickHouse/pull/67789) ([李扬](https://github.com/taiyang-li)).
|
||||
* Catch exception in destructor of LocalFileHolder. [#67891](https://github.com/ClickHouse/ClickHouse/pull/67891) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Add keeper error description to the message. [#67935](https://github.com/ClickHouse/ClickHouse/pull/67935) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Timeout handling for functional and integration tests, store artifacts and report if timed out - sets 2h default timeout for all jobs. [#67944](https://github.com/ClickHouse/ClickHouse/pull/67944) ([Max K.](https://github.com/maxknv)).
|
||||
* Add test cases to 03217_datetime64_constant_to_ast. [#67966](https://github.com/ClickHouse/ClickHouse/pull/67966) ([vdimir](https://github.com/vdimir)).
|
||||
* Remove some no-parallel tags from tests (Part 6). [#68048](https://github.com/ClickHouse/ClickHouse/pull/68048) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Keeper improvements package. [#68108](https://github.com/ClickHouse/ClickHouse/pull/68108) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* CI: Docker images clean from test scripts. [#68117](https://github.com/ClickHouse/ClickHouse/pull/68117) ([Max K.](https://github.com/maxknv)).
|
||||
* Output an operation error for ZK Multi request failed operation into log. [#68127](https://github.com/ClickHouse/ClickHouse/pull/68127) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Better test for Not-ready Set is passed in system.* tables. [#68139](https://github.com/ClickHouse/ClickHouse/pull/68139) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* ci: add more logs in the functional tests reports. [#68153](https://github.com/ClickHouse/ClickHouse/pull/68153) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix fundamentally broken `test_throttling`. [#68180](https://github.com/ClickHouse/ClickHouse/pull/68180) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `00600_replace_running_query`. [#68184](https://github.com/ClickHouse/ClickHouse/pull/68184) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix non-deterministic result order in `test_storage_mysql.test_mysql_distributed`. [#68188](https://github.com/ClickHouse/ClickHouse/pull/68188) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Try to unflaky `test_storage_s3_queue/test.py::test_shards_distributed`. [#68233](https://github.com/ClickHouse/ClickHouse/pull/68233) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Bump usearch to v2.13.2. [#68248](https://github.com/ClickHouse/ClickHouse/pull/68248) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix 'Refresh set entry already exists'. [#68249](https://github.com/ClickHouse/ClickHouse/pull/68249) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* tests: make 01600_parts_states_metrics_long better. [#68265](https://github.com/ClickHouse/ClickHouse/pull/68265) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Avoid ignoring errors of execute_process() (set COMMAND_ERROR_IS_FATAL=ANY). [#68267](https://github.com/ClickHouse/ClickHouse/pull/68267) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix FullSortingJoinTest.AsofGreaterGeneratedTestData with empty data. [#68280](https://github.com/ClickHouse/ClickHouse/pull/68280) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix min/max time columns in TimeSeries table. [#68282](https://github.com/ClickHouse/ClickHouse/pull/68282) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Update fuzzer dictionary. [#68284](https://github.com/ClickHouse/ClickHouse/pull/68284) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* [Green CI] Test 00652_mergetree_mutations is flaky. [#68286](https://github.com/ClickHouse/ClickHouse/pull/68286) ([Daniil Ivanik](https://github.com/divanik)).
|
||||
* Fix test storage_join_direct_join. [#68287](https://github.com/ClickHouse/ClickHouse/pull/68287) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Add execution status to PipelineExecutor. Avoid LOGICAL_ERROR if the pushing pipeline was canceled by timeout. Replace `Pipeline for PushingPipelineExecutor was finished before all data was inserted. (LOGICAL_ERROR)` to `QUERY_WAS_CANCELLED`. Closes [#63979](https://github.com/ClickHouse/ClickHouse/issues/63979). [#68291](https://github.com/ClickHouse/ClickHouse/pull/68291) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* CI: Create new release branch workflow updates. [#68294](https://github.com/ClickHouse/ClickHouse/pull/68294) ([Max K.](https://github.com/maxknv)).
|
||||
* Update version after release. [#68306](https://github.com/ClickHouse/ClickHouse/pull/68306) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Various vector similarity index related fixes. [#68308](https://github.com/ClickHouse/ClickHouse/pull/68308) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix: min marks to read overflow with parallel replicas. [#68309](https://github.com/ClickHouse/ClickHouse/pull/68309) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Apply libunwind changes needed for musl. [#68312](https://github.com/ClickHouse/ClickHouse/pull/68312) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Update musl to 1.2.5 with unwind info. [#68313](https://github.com/ClickHouse/ClickHouse/pull/68313) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix data race on SampleKey. [#68321](https://github.com/ClickHouse/ClickHouse/pull/68321) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix part name in 00961_check_table. [#68325](https://github.com/ClickHouse/ClickHouse/pull/68325) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix 02995_index_10 timeout. [#68329](https://github.com/ClickHouse/ClickHouse/pull/68329) ([vdimir](https://github.com/vdimir)).
|
||||
* CI: Fix for change log critical bug fix regex. [#68332](https://github.com/ClickHouse/ClickHouse/pull/68332) ([Max K.](https://github.com/maxknv)).
|
||||
* Optionally re-enable compilation with `-O0`. [#68352](https://github.com/ClickHouse/ClickHouse/pull/68352) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add debug info for `00180_no_seek_avoiding_when_reading_from_cache`. [#68353](https://github.com/ClickHouse/ClickHouse/pull/68353) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix data race in `DynamicResourceManager::updateConfiguration`. [#68356](https://github.com/ClickHouse/ClickHouse/pull/68356) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Update version_date.tsv and changelog after v24.3.7.30-lts. [#68357](https://github.com/ClickHouse/ClickHouse/pull/68357) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* More improvements in integration tests. [#68358](https://github.com/ClickHouse/ClickHouse/pull/68358) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Rename: S3DiskNoKeyErrors -> DiskS3NoSuchKeyErrors. [#68361](https://github.com/ClickHouse/ClickHouse/pull/68361) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* CI: Minor fixes for changelog and release exceptions. [#68362](https://github.com/ClickHouse/ClickHouse/pull/68362) ([Max K.](https://github.com/maxknv)).
|
||||
* Check that setProcessListElement() is not called on global context. [#68364](https://github.com/ClickHouse/ClickHouse/pull/68364) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix off-by-one inline function info in stack traces. [#68365](https://github.com/ClickHouse/ClickHouse/pull/68365) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Check that merge entries are valid. [#68366](https://github.com/ClickHouse/ClickHouse/pull/68366) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* performance comparison test for output_format_parquet_write_page_index. [#68367](https://github.com/ClickHouse/ClickHouse/pull/68367) ([max-vostrikov](https://github.com/max-vostrikov)).
|
||||
* Add a test for [#57324](https://github.com/ClickHouse/ClickHouse/issues/57324). [#68376](https://github.com/ClickHouse/ClickHouse/pull/68376) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* CI: Auto release workflow. [#68402](https://github.com/ClickHouse/ClickHouse/pull/68402) ([Max K.](https://github.com/maxknv)).
|
||||
* Update delta lake test. [#68404](https://github.com/ClickHouse/ClickHouse/pull/68404) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Add a test for [#59118](https://github.com/ClickHouse/ClickHouse/issues/59118). [#68406](https://github.com/ClickHouse/ClickHouse/pull/68406) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. IV. [#68407](https://github.com/ClickHouse/ClickHouse/pull/68407) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Try to fix test 03221_mutation_analyzer_skip_part. [#68409](https://github.com/ClickHouse/ClickHouse/pull/68409) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Add `NotStarted` status to not-prepared processors. This is for better diagnostics in the case of PipelineStuck. [#68413](https://github.com/ClickHouse/ClickHouse/pull/68413) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Update log message. [#68426](https://github.com/ClickHouse/ClickHouse/pull/68426) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix 01119_session_log flakiness. [#68433](https://github.com/ClickHouse/ClickHouse/pull/68433) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Try to mitigate 02818_memory_profiler_sample_min_max_allocation_size flakiness. [#68434](https://github.com/ClickHouse/ClickHouse/pull/68434) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Update analyzer_tech_debt.txt. [#68443](https://github.com/ClickHouse/ClickHouse/pull/68443) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove obsolete `-n` / `--multiquery` from tests. [#68447](https://github.com/ClickHouse/ClickHouse/pull/68447) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Check for invalid regexp in JSON SKIP REGEXP section. [#68451](https://github.com/ClickHouse/ClickHouse/pull/68451) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Better inference of date times 2. [#68452](https://github.com/ClickHouse/ClickHouse/pull/68452) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* CI: Native build for package_aarch64. [#68457](https://github.com/ClickHouse/ClickHouse/pull/68457) ([Max K.](https://github.com/maxknv)).
|
||||
* Minor update in Dynamic/JSON serializations. [#68459](https://github.com/ClickHouse/ClickHouse/pull/68459) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix test `02122_join_group_by_timeout`. [#68462](https://github.com/ClickHouse/ClickHouse/pull/68462) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add check for `min_number_of_marks = 0` in parallel replicas requests. [#68476](https://github.com/ClickHouse/ClickHouse/pull/68476) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix `Broken pipe` error for `03149_numbers_max_block_size_zero.sh`. [#68478](https://github.com/ClickHouse/ClickHouse/pull/68478) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Use temporary tables for input and output in `clickhouse-local`. [#68483](https://github.com/ClickHouse/ClickHouse/pull/68483) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Pass-through RENAME and UUID-related operations in Overlay database to underlying databases. [#68486](https://github.com/ClickHouse/ClickHouse/pull/68486) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix output of clickhouse-test in case of tests timeouts. [#68487](https://github.com/ClickHouse/ClickHouse/pull/68487) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Part of [#68024](https://github.com/ClickHouse/ClickHouse/issues/68024). [#68488](https://github.com/ClickHouse/ClickHouse/pull/68488) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Miscellaneous changes in BaseDaemon. [#68489](https://github.com/ClickHouse/ClickHouse/pull/68489) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Miscellaneous changes from [#66999](https://github.com/ClickHouse/ClickHouse/issues/66999) (2). [#68490](https://github.com/ClickHouse/ClickHouse/pull/68490) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Miscellaneous. [#68504](https://github.com/ClickHouse/ClickHouse/pull/68504) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Miscellanous changes from [#66999](https://github.com/ClickHouse/ClickHouse/issues/66999). [#68507](https://github.com/ClickHouse/ClickHouse/pull/68507) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `01017_uniqCombined_memory_usage`. [#68508](https://github.com/ClickHouse/ClickHouse/pull/68508) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix race condition in MergeTreeRestartingThread. [#68513](https://github.com/ClickHouse/ClickHouse/pull/68513) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `01079_bad_alters_zookeeper_long`. [#68515](https://github.com/ClickHouse/ClickHouse/pull/68515) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bad exception messages. [#68520](https://github.com/ClickHouse/ClickHouse/pull/68520) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* CI: Auto Releases in prod. [#68525](https://github.com/ClickHouse/ClickHouse/pull/68525) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix build with `-DENABLE_LIBRARIES=0`. [#68527](https://github.com/ClickHouse/ClickHouse/pull/68527) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Removed the use of the context during function execution, as it was causing errors when the context was expired. [#68534](https://github.com/ClickHouse/ClickHouse/pull/68534) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* CI: Minor release workflow fix. [#68536](https://github.com/ClickHouse/ClickHouse/pull/68536) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix after [#68291](https://github.com/ClickHouse/ClickHouse/issues/68291). [#68548](https://github.com/ClickHouse/ClickHouse/pull/68548) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Stateless tests: increase hung check timeout. [#68552](https://github.com/ClickHouse/ClickHouse/pull/68552) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* CI: Tidy build timeout from 2h to 3h. [#68567](https://github.com/ClickHouse/ClickHouse/pull/68567) ([Max K.](https://github.com/maxknv)).
|
||||
* Reduce memory consumption in ghdata JSON tests. [#68571](https://github.com/ClickHouse/ClickHouse/pull/68571) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Move enabling experimental settings to a separate file. [#68577](https://github.com/ClickHouse/ClickHouse/pull/68577) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Minor logging fixes. [#68578](https://github.com/ClickHouse/ClickHouse/pull/68578) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix enumerating dynamic subcolumns. [#68582](https://github.com/ClickHouse/ClickHouse/pull/68582) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Update version_date.tsv and changelog after v23.8.16.16-lts. [#68593](https://github.com/ClickHouse/ClickHouse/pull/68593) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update 02995_index_7.sh. [#68594](https://github.com/ClickHouse/ClickHouse/pull/68594) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix style in Functions/printf.cpp. [#68595](https://github.com/ClickHouse/ClickHouse/pull/68595) ([vdimir](https://github.com/vdimir)).
|
||||
* Update version_date.tsv and changelog after v24.3.8.13-lts. [#68599](https://github.com/ClickHouse/ClickHouse/pull/68599) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Prioritizing of virtual columns in hive partitioning. [#68606](https://github.com/ClickHouse/ClickHouse/pull/68606) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* materialized_view_deduplication performance comparison test. [#68607](https://github.com/ClickHouse/ClickHouse/pull/68607) ([max-vostrikov](https://github.com/max-vostrikov)).
|
||||
* Update README.md. [#68610](https://github.com/ClickHouse/ClickHouse/pull/68610) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* patch: fix reference to sorting key in primary key docs. [#68612](https://github.com/ClickHouse/ClickHouse/pull/68612) ([Leon Kozlowski](https://github.com/leonkozlowski)).
|
||||
* Do not fuzz 02835 drop user during session. [#68620](https://github.com/ClickHouse/ClickHouse/pull/68620) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix 03221_s3_imds_decent_timeout. [#68627](https://github.com/ClickHouse/ClickHouse/pull/68627) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix test `01079_bad_alters_zookeeper_long`. [#68629](https://github.com/ClickHouse/ClickHouse/pull/68629) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Disable min_bytes_to_use_direct_io in some tests with Dynamic/JSON subcolumns because it's broken. [#68632](https://github.com/ClickHouse/ClickHouse/pull/68632) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix false "Killed by signal (output files)" in stress_tests.lib. [#68638](https://github.com/ClickHouse/ClickHouse/pull/68638) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Remove wrong release version. [#68646](https://github.com/ClickHouse/ClickHouse/pull/68646) ([Max K.](https://github.com/maxknv)).
|
||||
* Increase connectTimeoutMs IMDS connection timeout to 50ms to avoid failures in CI. [#68653](https://github.com/ClickHouse/ClickHouse/pull/68653) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* CI: Disable SQLLogic job. [#68654](https://github.com/ClickHouse/ClickHouse/pull/68654) ([Max K.](https://github.com/maxknv)).
|
||||
* Update version_date.tsv and changelog after v24.8.1.2684-lts. [#68664](https://github.com/ClickHouse/ClickHouse/pull/68664) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix flaky test test_distributed_replica_max_ignored_errors. [#68665](https://github.com/ClickHouse/ClickHouse/pull/68665) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Improve `02293_http_header_full_summary_without_progress` logging. [#68666](https://github.com/ClickHouse/ClickHouse/pull/68666) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Fix flaky check when all tests are skipped. [#68673](https://github.com/ClickHouse/ClickHouse/pull/68673) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Vector similarity index: make `bf16` the default quantization. [#68678](https://github.com/ClickHouse/ClickHouse/pull/68678) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* 3h is not enough. [#68683](https://github.com/ClickHouse/ClickHouse/pull/68683) ([Max K.](https://github.com/maxknv)).
|
||||
* Un-flake 01278_random_string_utf8. [#68684](https://github.com/ClickHouse/ClickHouse/pull/68684) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* CI: Integration tests timeout to 3h. [#68685](https://github.com/ClickHouse/ClickHouse/pull/68685) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix Upgrade Check: move some settings to 24.9 section. [#68688](https://github.com/ClickHouse/ClickHouse/pull/68688) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Update README.md. [#68690](https://github.com/ClickHouse/ClickHouse/pull/68690) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Use proper ErrorCodes, replace NETWORK_ERROR by HDFS_ERROR. [#68696](https://github.com/ClickHouse/ClickHouse/pull/68696) ([flynn](https://github.com/ucasfl)).
|
||||
* Fix for failing test in CI. [#68701](https://github.com/ClickHouse/ClickHouse/pull/68701) ([Pedro Ferreira](https://github.com/PedroTadim)).
|
||||
* CI: Stress test fix. [#68712](https://github.com/ClickHouse/ClickHouse/pull/68712) ([Max K.](https://github.com/maxknv)).
|
||||
* Speedup test 02150_index_hypothesis_race_long. [#68713](https://github.com/ClickHouse/ClickHouse/pull/68713) ([vdimir](https://github.com/vdimir)).
|
||||
* Turn off fault injection for insert in `01396_inactive_replica_cleanup_nodes_zookeeper`. [#68715](https://github.com/ClickHouse/ClickHouse/pull/68715) ([alesapin](https://github.com/alesapin)).
|
||||
* Update README.md - Meetups update. [#68723](https://github.com/ClickHouse/ClickHouse/pull/68723) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Fix flaky check. [#68725](https://github.com/ClickHouse/ClickHouse/pull/68725) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* fix shutdown for PeriodicLog. [#68728](https://github.com/ClickHouse/ClickHouse/pull/68728) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Update version_date.tsv and changelog after v24.5.5.41-stable. [#68729](https://github.com/ClickHouse/ClickHouse/pull/68729) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Bump Replxx to support custom descriptors. [#68730](https://github.com/ClickHouse/ClickHouse/pull/68730) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Update version_date.tsv and changelog after v24.6.3.38-stable. [#68732](https://github.com/ClickHouse/ClickHouse/pull/68732) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fixes `00080_show_tables_and_system_tables` that was a bit flaky. [#68734](https://github.com/ClickHouse/ClickHouse/pull/68734) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Update version_date.tsv and changelog after v24.7.3.47-stable. [#68735](https://github.com/ClickHouse/ClickHouse/pull/68735) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix flaky test 00989_parallel_parts_loading. [#68737](https://github.com/ClickHouse/ClickHouse/pull/68737) ([alesapin](https://github.com/alesapin)).
|
||||
* Update README.md. [#68738](https://github.com/ClickHouse/ClickHouse/pull/68738) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* Update version_date.tsv and changelog after v24.8.2.3-lts. [#68740](https://github.com/ClickHouse/ClickHouse/pull/68740) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.5.5.41-stable. [#68745](https://github.com/ClickHouse/ClickHouse/pull/68745) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* To make patch release possible from every commit on release branch, package_debug build is required and must not be skipped. [#68750](https://github.com/ClickHouse/ClickHouse/pull/68750) ([Max K.](https://github.com/maxknv)).
|
||||
* Try to disable rerun check if job triggered manually. [#68751](https://github.com/ClickHouse/ClickHouse/pull/68751) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix 2477 timeout. [#68752](https://github.com/ClickHouse/ClickHouse/pull/68752) ([Shichao](https://github.com/jsc0218)).
|
||||
* Update README.md. [#68764](https://github.com/ClickHouse/ClickHouse/pull/68764) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Update version_date.tsv and changelog after v24.5.6.45-stable. [#68766](https://github.com/ClickHouse/ClickHouse/pull/68766) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.6.4.42-stable. [#68767](https://github.com/ClickHouse/ClickHouse/pull/68767) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.7.4.51-stable. [#68768](https://github.com/ClickHouse/ClickHouse/pull/68768) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Split test case and reduce number of random runs to reduce the time necessary to run the test. [#68772](https://github.com/ClickHouse/ClickHouse/pull/68772) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Check setting use_json_alias_for_old_object_type in runtime. [#68793](https://github.com/ClickHouse/ClickHouse/pull/68793) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Make dynamic structure selection more consistent. [#68802](https://github.com/ClickHouse/ClickHouse/pull/68802) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix zero copy bug with encrypted disk and UNFREEZE. [#68821](https://github.com/ClickHouse/ClickHouse/pull/68821) ([Joe Lynch](https://github.com/joelynch)).
|
||||
* Speed up functions `lowerUTF8`/`upperUTF8`. [#68822](https://github.com/ClickHouse/ClickHouse/pull/68822) ([李扬](https://github.com/taiyang-li)).
|
||||
* Convert integration test `test_incorrect_datetime_format` to stateless. [#68823](https://github.com/ClickHouse/ClickHouse/pull/68823) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fix Function Typo. [#68835](https://github.com/ClickHouse/ClickHouse/pull/68835) ([Shichao](https://github.com/jsc0218)).
|
||||
* Fix test `03228_virtual_column_merge_dist`. [#68841](https://github.com/ClickHouse/ClickHouse/pull/68841) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix test `03221_mutation_analyzer_skip_part`. [#68842](https://github.com/ClickHouse/ClickHouse/pull/68842) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix flaky `02932_analyzer_rewrite_sum_column_and_constant `. [#68843](https://github.com/ClickHouse/ClickHouse/pull/68843) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Update README.md - Add Austin meetup. [#68845](https://github.com/ClickHouse/ClickHouse/pull/68845) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Fix ssl handshake error processing. [#68866](https://github.com/ClickHouse/ClickHouse/pull/68866) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Speedup test 00653_verification_monotonic_data_load. [#68878](https://github.com/ClickHouse/ClickHouse/pull/68878) ([vdimir](https://github.com/vdimir)).
|
||||
* ci: add IPv6 support to `NetworkManager`. [#68900](https://github.com/ClickHouse/ClickHouse/pull/68900) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* ci: add checks for `iptables-nft` in integration test runner. [#68902](https://github.com/ClickHouse/ClickHouse/pull/68902) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Fix `02378_part_log_profile_events` flakiness. [#68917](https://github.com/ClickHouse/ClickHouse/pull/68917) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Make long_parquet* tests less long. [#68918](https://github.com/ClickHouse/ClickHouse/pull/68918) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix 01114_database_atomic flakiness. [#68930](https://github.com/ClickHouse/ClickHouse/pull/68930) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* CI: Fix job rerun check. [#68931](https://github.com/ClickHouse/ClickHouse/pull/68931) ([Max K.](https://github.com/maxknv)).
|
||||
* perf tests set cgroups_memory_usage_observer_wait_time to zero. [#68957](https://github.com/ClickHouse/ClickHouse/pull/68957) ([vdimir](https://github.com/vdimir)).
|
||||
* Revert un-quoting `auto` settings values. Fixes [#68748](https://github.com/ClickHouse/ClickHouse/issues/68748). [#68979](https://github.com/ClickHouse/ClickHouse/pull/68979) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Actually fix false "Killed by signal (output files)" in stress_tests.lib. [#68983](https://github.com/ClickHouse/ClickHouse/pull/68983) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* no-tsan in 02735_parquet_encoder. [#68984](https://github.com/ClickHouse/ClickHouse/pull/68984) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Update CHANGELOG.md. [#68994](https://github.com/ClickHouse/ClickHouse/pull/68994) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test_role & test_keeper_s3_snapshot integration tests. [#69013](https://github.com/ClickHouse/ClickHouse/pull/69013) ([Shankar](https://github.com/shiyer7474)).
|
||||
* fix false leak detect in libfiu. [#69018](https://github.com/ClickHouse/ClickHouse/pull/69018) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Update README.md. [#69041](https://github.com/ClickHouse/ClickHouse/pull/69041) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* Update parametric-functions.md. [#69055](https://github.com/ClickHouse/ClickHouse/pull/69055) ([Samuel Warfield](https://github.com/Warfields)).
|
||||
* Disallow `ALTER TABLE ADD VECTOR SIMILARITY INDEX` if corresponding setting is not enabled. [#69065](https://github.com/ClickHouse/ClickHouse/pull/69065) ([flynn](https://github.com/ucasfl)).
|
||||
* Remove stale moving parts without zookeeper. [#69075](https://github.com/ClickHouse/ClickHouse/pull/69075) ([Kirill](https://github.com/kirillgarbar)).
|
||||
* Update README.md. [#69077](https://github.com/ClickHouse/ClickHouse/pull/69077) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* Fix typo in ActionsDag. [#69086](https://github.com/ClickHouse/ClickHouse/pull/69086) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Follow-up for https://github.com/ClickHouse/ClickHouse/pull/68332. [#69099](https://github.com/ClickHouse/ClickHouse/pull/69099) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix subnet in docker_compose_net.yml. [#69121](https://github.com/ClickHouse/ClickHouse/pull/69121) ([Ilya Golshtein](https://github.com/ilejn)).
|
||||
* Fix: expression description in plan after lift up union optimization. [#69124](https://github.com/ClickHouse/ClickHouse/pull/69124) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix locking in UserDefinedSQLObjectsZooKeeperStorage.cpp. [#69132](https://github.com/ClickHouse/ClickHouse/pull/69132) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Do not use docker pause for Kerberos KDC container in integration tests. [#69136](https://github.com/ClickHouse/ClickHouse/pull/69136) ([Ilya Golshtein](https://github.com/ilejn)).
|
||||
* Don't create Object type if use_json_alias_for_old_object_type=1 but allow_experimental_object_type=0. [#69150](https://github.com/ClickHouse/ClickHouse/pull/69150) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Use QueryPlan for merge. [#69167](https://github.com/ClickHouse/ClickHouse/pull/69167) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Update version_date.tsv and changelog after v24.3.10.33-lts. [#69177](https://github.com/ClickHouse/ClickHouse/pull/69177) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* CI: Rerun check: do not check if manual rerun. [#69181](https://github.com/ClickHouse/ClickHouse/pull/69181) ([Max K.](https://github.com/maxknv)).
|
||||
* Update version_date.tsv and changelog after v24.5.7.31-stable. [#69182](https://github.com/ClickHouse/ClickHouse/pull/69182) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.6.5.30-stable. [#69184](https://github.com/ClickHouse/ClickHouse/pull/69184) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix flaky 02915_move_partition_inactive_replica. [#69190](https://github.com/ClickHouse/ClickHouse/pull/69190) ([alesapin](https://github.com/alesapin)).
|
||||
* Update version_date.tsv and changelog after v24.8.3.59-lts. [#69191](https://github.com/ClickHouse/ClickHouse/pull/69191) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Disable memory test with sanitizer. [#69193](https://github.com/ClickHouse/ClickHouse/pull/69193) ([alesapin](https://github.com/alesapin)).
|
||||
* Disable perf-like test with sanitizers. [#69194](https://github.com/ClickHouse/ClickHouse/pull/69194) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix jepsen for aarch64. [#69199](https://github.com/ClickHouse/ClickHouse/pull/69199) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Test for parallel replicas, reverse in order reading mode. [#69202](https://github.com/ClickHouse/ClickHouse/pull/69202) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Collect sanitizer report from client to `client_log`. [#69204](https://github.com/ClickHouse/ClickHouse/pull/69204) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix flaky 00157_cache_dictionary. [#69205](https://github.com/ClickHouse/ClickHouse/pull/69205) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Hide Settings implementation. [#69213](https://github.com/ClickHouse/ClickHouse/pull/69213) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix progress bar when reading from Memory tables. [#69234](https://github.com/ClickHouse/ClickHouse/pull/69234) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* CMake: Update ICU build description. [#69240](https://github.com/ClickHouse/ClickHouse/pull/69240) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add testcase for ANN index usage with subquery. [#69241](https://github.com/ClickHouse/ClickHouse/pull/69241) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backports should read tags from its repo only. [#69252](https://github.com/ClickHouse/ClickHouse/pull/69252) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* 01114_database_atomic: Increase time frames to reduce flakiness. [#69255](https://github.com/ClickHouse/ClickHouse/pull/69255) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix dropping of file cache in CHECK query in case of enabled transactions. [#69256](https://github.com/ClickHouse/ClickHouse/pull/69256) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* CI: Merge stress and func runners type. [#69257](https://github.com/ClickHouse/ClickHouse/pull/69257) ([Max K.](https://github.com/maxknv)).
|
||||
* Make infrastructure related scripts private. [#69260](https://github.com/ClickHouse/ClickHouse/pull/69260) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Fix: Not-ready Set with parallel replicas. [#69264](https://github.com/ClickHouse/ClickHouse/pull/69264) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Revert "CREATE TABLE AS copy PRIMARY KEY, ORDER BY, and similar clauses.". [#69268](https://github.com/ClickHouse/ClickHouse/pull/69268) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Use CANNOT_DECOMPRESS error code for the compressed external data in CompressedReadBufferBase for checksum validation and decompression. [#69269](https://github.com/ClickHouse/ClickHouse/pull/69269) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* Fix the error in loading client suggestions and select from `system.function` when `allow_experimental_nlp_functions` setting is enabled and no lemmatizers are specified in the configuration. [#69271](https://github.com/ClickHouse/ClickHouse/pull/69271) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Improve logical error trace for TryResult. [#69297](https://github.com/ClickHouse/ClickHouse/pull/69297) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Just a small refactoring. [#69298](https://github.com/ClickHouse/ClickHouse/pull/69298) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Don't run 01287_max_execution_speed with slow builds. [#69299](https://github.com/ClickHouse/ClickHouse/pull/69299) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Add checks against segfault in DeltaLakeMetadata. [#69305](https://github.com/ClickHouse/ClickHouse/pull/69305) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Update StackTrace.cpp. [#69306](https://github.com/ClickHouse/ClickHouse/pull/69306) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Bump libarchive from v3.7.0 to v3.7.1. [#69318](https://github.com/ClickHouse/ClickHouse/pull/69318) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Bump curl to v8.9.1. [#69321](https://github.com/ClickHouse/ClickHouse/pull/69321) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* These tests take long time to execute with slow builds (either 700 or 400 seconds). [#69322](https://github.com/ClickHouse/ClickHouse/pull/69322) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Bump grpc to v1.59.5. [#69323](https://github.com/ClickHouse/ClickHouse/pull/69323) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Speed up some Kafka tests with multiprocessing. [#69324](https://github.com/ClickHouse/ClickHouse/pull/69324) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Minor follow-up for [#66933](https://github.com/ClickHouse/ClickHouse/issues/66933). [#69326](https://github.com/ClickHouse/ClickHouse/pull/69326) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Randomize integration tests settings. [#69328](https://github.com/ClickHouse/ClickHouse/pull/69328) ([vdimir](https://github.com/vdimir)).
|
||||
* Update version_date.tsv and changelog after v24.3.11.7-lts. [#69330](https://github.com/ClickHouse/ClickHouse/pull/69330) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.5.8.10-stable. [#69333](https://github.com/ClickHouse/ClickHouse/pull/69333) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Add CITATION.cff. [#69334](https://github.com/ClickHouse/ClickHouse/pull/69334) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update version_date.tsv and changelog after v24.6.6.6-stable. [#69335](https://github.com/ClickHouse/ClickHouse/pull/69335) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix a leak of a few bytes in function `lower/upperUTF8`. [#69336](https://github.com/ClickHouse/ClickHouse/pull/69336) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update version_date.tsv and changelog after v24.7.6.8-stable. [#69337](https://github.com/ClickHouse/ClickHouse/pull/69337) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.8.4.13-lts. [#69340](https://github.com/ClickHouse/ClickHouse/pull/69340) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Bump OpenSSL to v3.2.3. [#69341](https://github.com/ClickHouse/ClickHouse/pull/69341) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. V. [#69344](https://github.com/ClickHouse/ClickHouse/pull/69344) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Bump libuv to v1.48.0. [#69345](https://github.com/ClickHouse/ClickHouse/pull/69345) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update README.md - Meetups. [#69359](https://github.com/ClickHouse/ClickHouse/pull/69359) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. VI. [#69361](https://github.com/ClickHouse/ClickHouse/pull/69361) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Bump libarchive to v3.7.4. [#69367](https://github.com/ClickHouse/ClickHouse/pull/69367) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Rename `count_min` statistics to `countmin`. [#69377](https://github.com/ClickHouse/ClickHouse/pull/69377) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Refactor temporary file usage in MergeTask with QueryPlan. [#69383](https://github.com/ClickHouse/ClickHouse/pull/69383) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Fix type mismatch. [#69385](https://github.com/ClickHouse/ClickHouse/pull/69385) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix 24.8 setting compatibility `rows_before_aggregation`. [#69394](https://github.com/ClickHouse/ClickHouse/pull/69394) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Support more oss endpoints. [#69400](https://github.com/ClickHouse/ClickHouse/pull/69400) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix 01603_read_with_backoff_bug. [#69413](https://github.com/ClickHouse/ClickHouse/pull/69413) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* See if it would address strange issue with `Resource temporarily unavailable`. [#69416](https://github.com/ClickHouse/ClickHouse/pull/69416) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Added a server setting `database_replicated_allow_detach_permanently` that disallows `DETACH TABLE PERMANENTLY` queries in Replicated databases (allowed by default). [#69422](https://github.com/ClickHouse/ClickHouse/pull/69422) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* See https://s3.amazonaws.com/clickhouse-test-reports/69093/d9b1fb7a7cb0813dd12d8f73423662e02a458056/fast_test.html. [#69430](https://github.com/ClickHouse/ClickHouse/pull/69430) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fix test_disks_app_func. [#69449](https://github.com/ClickHouse/ClickHouse/pull/69449) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fixed consistency for `groupConcat` with `arrayStringConcat` in case of `FixedString` as argument. [#69455](https://github.com/ClickHouse/ClickHouse/pull/69455) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* The fix from [#69416](https://github.com/ClickHouse/ClickHouse/issues/69416) was incomplete. Reduce the size of printed statement to 4k max. [#69483](https://github.com/ClickHouse/ClickHouse/pull/69483) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* CMake: Add comment about ICU data files. [#69511](https://github.com/ClickHouse/ClickHouse/pull/69511) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* S3Queue: small refactoring. [#69513](https://github.com/ClickHouse/ClickHouse/pull/69513) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backport MergeTask changes from private to minimize merge conflicts. [#69525](https://github.com/ClickHouse/ClickHouse/pull/69525) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Fix getting the latest synced commit for long PRs. [#69538](https://github.com/ClickHouse/ClickHouse/pull/69538) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* When trying to ATTACH ReplicatedMergeTree tables that were previously PERMANENTLY DETACHED, we will fail due to explicitly specified parameters. Reference https://github.com/ClickHouse/ClickHouse/pull/66104. [#69539](https://github.com/ClickHouse/ClickHouse/pull/69539) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fix ci upgrade check failure for `join_to_sort` settings in pr https://github.com/ClickHouse/ClickHouse/pull/60341. [#69554](https://github.com/ClickHouse/ClickHouse/pull/69554) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Add function `kill_ci_runner`. Kill runner when pre-pull failed. [#69557](https://github.com/ClickHouse/ClickHouse/pull/69557) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Replace libpq code dump by postgresql fork + bump to v14.3. [#69564](https://github.com/ClickHouse/ClickHouse/pull/69564) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Mask azure connection string sensitive info. [#69570](https://github.com/ClickHouse/ClickHouse/pull/69570) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* Don't leave an empty znode when replicated table is dropped. [#69574](https://github.com/ClickHouse/ClickHouse/pull/69574) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Enable removerecursive in ci. [#69578](https://github.com/ClickHouse/ClickHouse/pull/69578) ([Mikhail Artemenko](https://github.com/Michicosun)).
|
||||
* Bump libpqxx to v7.7.5. [#69580](https://github.com/ClickHouse/ClickHouse/pull/69580) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove superfluous `--multiquery/-n`, pt. V. [#69581](https://github.com/ClickHouse/ClickHouse/pull/69581) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* A better fix for [#67476](https://github.com/ClickHouse/ClickHouse/issues/67476). [#69583](https://github.com/ClickHouse/ClickHouse/pull/69583) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add more contexts to the debug action and use it broadly. [#69599](https://github.com/ClickHouse/ClickHouse/pull/69599) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Try to fix data race in `WriteBufferFromHTTPServerResponse`. [#69601](https://github.com/ClickHouse/ClickHouse/pull/69601) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Remove explicit announce from local replica in ReadFromMergeTree. [#69602](https://github.com/ClickHouse/ClickHouse/pull/69602) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Unification of FunctionSecretArgumentsFinder. [#69615](https://github.com/ClickHouse/ClickHouse/pull/69615) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Reorder some tests (follow-up to [#69514](https://github.com/ClickHouse/ClickHouse/issues/69514)). [#69626](https://github.com/ClickHouse/ClickHouse/pull/69626) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fix invalid `clickhouse-format` invocation in tests. [#69642](https://github.com/ClickHouse/ClickHouse/pull/69642) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Try fix `02447_drop_database_replica`. [#69655](https://github.com/ClickHouse/ClickHouse/pull/69655) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* S3Queue small refactoring. [#69672](https://github.com/ClickHouse/ClickHouse/pull/69672) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Update assert. [#69673](https://github.com/ClickHouse/ClickHouse/pull/69673) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Bump libpq from v14.3 to v15.8. [#69674](https://github.com/ClickHouse/ClickHouse/pull/69674) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Try fix asserts failure in `HashJoin`. [#69682](https://github.com/ClickHouse/ClickHouse/pull/69682) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Prohibit `ALTER TABLE ... ADD INDEX ... TYPE` inverted if setting = 0. [#69684](https://github.com/ClickHouse/ClickHouse/pull/69684) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fixed 2 bugs in Remove Recursive implementation:. [#69690](https://github.com/ClickHouse/ClickHouse/pull/69690) ([Mikhail Artemenko](https://github.com/Michicosun)).
|
||||
* New profile events for parallel replicas. [#69706](https://github.com/ClickHouse/ClickHouse/pull/69706) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Update README.md - Meetups. [#69714](https://github.com/ClickHouse/ClickHouse/pull/69714) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* added some edge cases for printf tests. [#69737](https://github.com/ClickHouse/ClickHouse/pull/69737) ([max-vostrikov](https://github.com/max-vostrikov)).
|
||||
* Bump libpq to v16.4. [#69741](https://github.com/ClickHouse/ClickHouse/pull/69741) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix parallel replicas protocol after [#68424](https://github.com/ClickHouse/ClickHouse/issues/68424). [#69744](https://github.com/ClickHouse/ClickHouse/pull/69744) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* CI: Include PostgreSQL in sparse checkout script. [#69746](https://github.com/ClickHouse/ClickHouse/pull/69746) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Get rid of the import cycle when env_helper can't be imported from `ci_*.py` files. Now, `env_helper` won't import download helper or basically anything from the `ci` module. The report is the best place for the GH job info since we use it mostly there. [#69750](https://github.com/ClickHouse/ClickHouse/pull/69750) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Save CREATE QUERY with the KeeperMap engine with evaluated parameters. It is important because, for example, we can use the `currentDatabase()` function in the KeeperMap `zk_root_path` argument, and we want it to point to the same path even if we move this table to another database. [#69751](https://github.com/ClickHouse/ClickHouse/pull/69751) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fix native macOS build. [#69767](https://github.com/ClickHouse/ClickHouse/pull/69767) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backported in [#69974](https://github.com/ClickHouse/ClickHouse/issues/69974): S3Queue: support having deprecated settings to not fail server startup. [#69769](https://github.com/ClickHouse/ClickHouse/pull/69769) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Limit fast tests to 30 seconds. [#69781](https://github.com/ClickHouse/ClickHouse/pull/69781) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69824](https://github.com/ClickHouse/ClickHouse/issues/69824): Allow cyrillic characters in generated contributor names. [#69820](https://github.com/ClickHouse/ClickHouse/pull/69820) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69912](https://github.com/ClickHouse/ClickHouse/issues/69912): Add Proj Obsolete Setting. [#69883](https://github.com/ClickHouse/ClickHouse/pull/69883) ([Shichao](https://github.com/jsc0218)).
|
||||
* Backported in [#69899](https://github.com/ClickHouse/ClickHouse/issues/69899): Revert "Merge pull request [#69032](https://github.com/ClickHouse/ClickHouse/issues/69032) from alexon1234/include_real_time_execution_in_http_header". [#69885](https://github.com/ClickHouse/ClickHouse/pull/69885) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#69931](https://github.com/ClickHouse/ClickHouse/issues/69931): RIPE is an acronym and thus should be capital. RIPE stands for **R**ACE **I**ntegrity **P**rimitives **E**valuation and RACE stands for **R**esearch and Development in **A**dvanced **C**ommunications **T**echnologies in **E**urope. [#69901](https://github.com/ClickHouse/ClickHouse/pull/69901) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Backported in [#70034](https://github.com/ClickHouse/ClickHouse/issues/70034): Revert "Add RIPEMD160 function". [#70005](https://github.com/ClickHouse/ClickHouse/pull/70005) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
|
||||
@ -1629,6 +1629,17 @@ The second overload emulates TimescaleDB's `time_bucket()` function, respectivel
|
||||
``` SQL
|
||||
SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));
|
||||
```
|
||||
|
||||
result:
|
||||
|
||||
``` reference
|
||||
┌───toStartOfInterval(...)─┐
|
||||
│ 2023-01-01 14:44:30 │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
Aliases: `time_bucket`, `date_bin`.
|
||||
|
||||
**See Also**
|
||||
- [date_trunc](#date_trunc)
|
||||
|
||||
|
||||
@ -688,40 +688,6 @@ SELECT kostikConsistentHash(16045690984833335023, 2);
|
||||
└───────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## RIPEMD160
|
||||
|
||||
Produces [RIPEMD-160](https://en.wikipedia.org/wiki/RIPEMD) hash value.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
RIPEMD160(input)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `input`: Input string. [String](../data-types/string.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A [UInt256](../data-types/int-uint.md) hash value where the 160-bit RIPEMD-160 hash is stored in the first 20 bytes. The remaining 12 bytes are zero-padded.
|
||||
|
||||
**Example**
|
||||
|
||||
Use the [hex](../functions/encoding-functions.md/#hex) function to represent the result as a hex-encoded string.
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT hex(RIPEMD160('The quick brown fox jumps over the lazy dog'));
|
||||
```
|
||||
|
||||
```response
|
||||
┌─hex(RIPEMD160('The quick brown fox jumps over the lazy dog'))─┐
|
||||
│ 37F332F68DB77BD9D7EDD4969571AD671CF9DD3B │
|
||||
└───────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## murmurHash2_32, murmurHash2_64
|
||||
|
||||
Produces a [MurmurHash2](https://github.com/aappleby/smhasher) hash value.
|
||||
|
||||
@ -288,7 +288,7 @@ roundToExp2(num)
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `num`: A number representing an age in years. [UInt](../data-types/int-uint.md)/[Float](../data-types/float.md).
|
||||
- `num`: A number to round. [UInt](../data-types/int-uint.md)/[Float](../data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
|
||||
@ -4137,6 +4137,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an exception.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal32(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal32('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(9, S)`. [Decimal32(S)](../data-types/int-uint.md).
|
||||
@ -4328,6 +4334,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an error.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal32OrDefault(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal32OrDefault('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(9, S)` if successful, otherwise returns the default value if passed or `0` if not. [Decimal32(S)](../data-types/decimal.md).
|
||||
@ -4391,6 +4403,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an exception.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal64(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal64('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(18, S)`. [Decimal64(S)](../data-types/int-uint.md).
|
||||
@ -4582,6 +4600,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an error.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal64OrDefault(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal64OrDefault('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(18, S)` if successful, otherwise returns the default value if passed or `0` if not. [Decimal64(S)](../data-types/decimal.md).
|
||||
@ -4645,6 +4669,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an exception.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal128(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal128('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(38, S)`. [Decimal128(S)](../data-types/int-uint.md).
|
||||
@ -4836,6 +4866,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an error.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal128OrDefault(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal128OrDefault('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(38, S)` if successful, otherwise returns the default value if passed or `0` if not. [Decimal128(S)](../data-types/decimal.md).
|
||||
@ -4899,6 +4935,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an exception.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal256(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal256('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(76, S)`. [Decimal256(S)](../data-types/int-uint.md).
|
||||
@ -5090,6 +5132,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an error.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal256OrDefault(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal256OrDefault('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(76, S)` if successful, otherwise returns the default value if passed or `0` if not. [Decimal256(S)](../data-types/decimal.md).
|
||||
|
||||
@ -14,7 +14,7 @@ To revoke privileges, use the [REVOKE](../../sql-reference/statements/revoke.md)
|
||||
## Granting Privilege Syntax
|
||||
|
||||
``` sql
|
||||
GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} TO {user | role | CURRENT_USER} [,...] [WITH GRANT OPTION] [WITH REPLACE OPTION]
|
||||
GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table[*]|db[*].*|*.*|table[*]|*} TO {user | role | CURRENT_USER} [,...] [WITH GRANT OPTION] [WITH REPLACE OPTION]
|
||||
```
|
||||
|
||||
- `privilege` — Type of privilege.
|
||||
@ -68,12 +68,53 @@ It means that `john` has the permission to execute:
|
||||
|
||||
Also `john` has the `GRANT OPTION` privilege, so it can grant other users with privileges of the same or smaller scope.
|
||||
|
||||
Specifying privileges you can use asterisk (`*`) instead of a table or a database name. For example, the `GRANT SELECT ON db.* TO john` query allows `john` to execute the `SELECT` query over all the tables in `db` database. Also, you can omit database name. In this case privileges are granted for current database. For example, `GRANT SELECT ON * TO john` grants the privilege on all the tables in the current database, `GRANT SELECT ON mytable TO john` grants the privilege on the `mytable` table in the current database.
|
||||
|
||||
Access to the `system` database is always allowed (since this database is used for processing queries).
|
||||
|
||||
You can grant multiple privileges to multiple accounts in one query. The query `GRANT SELECT, INSERT ON *.* TO john, robin` allows accounts `john` and `robin` to execute the `INSERT` and `SELECT` queries over all the tables in all the databases on the server.
|
||||
|
||||
## Wildcard grants
|
||||
|
||||
Specifying privileges you can use asterisk (`*`) instead of a table or a database name. For example, the `GRANT SELECT ON db.* TO john` query allows `john` to execute the `SELECT` query over all the tables in `db` database.
|
||||
Also, you can omit database name. In this case privileges are granted for current database.
|
||||
For example, `GRANT SELECT ON * TO john` grants the privilege on all the tables in the current database, `GRANT SELECT ON mytable TO john` grants the privilege on the `mytable` table in the current database.
|
||||
|
||||
You can also put asterisks at the end of a table or a database name. This feature allows you to grant privileges on an abstract prefix of the table's path.
|
||||
Example: `GRANT SELECT ON db.my_tables* TO john`. This query allows `john` to execute the `SELECT` query over all the `db` database tables with the prefix `my_tables*`.
|
||||
|
||||
More examples:
|
||||
|
||||
`GRANT SELECT ON db.my_tables* TO john`
|
||||
```sql
|
||||
SELECT * FROM db.my_tables -- granted
|
||||
SELECT * FROM db.my_tables_0 -- granted
|
||||
SELECT * FROM db.my_tables_1 -- granted
|
||||
|
||||
SELECT * FROM db.other_table -- not_granted
|
||||
SELECT * FROM db2.my_tables -- not_granted
|
||||
```
|
||||
|
||||
`GRANT SELECT ON db*.* TO john`
|
||||
```sql
|
||||
SELECT * FROM db.my_tables -- granted
|
||||
SELECT * FROM db.my_tables_0 -- granted
|
||||
SELECT * FROM db.my_tables_1 -- granted
|
||||
SELECT * FROM db.other_table -- granted
|
||||
SELECT * FROM db2.my_tables -- granted
|
||||
```
|
||||
|
||||
All newly created tables within granted paths will automatically inherit all grants from their parents.
|
||||
For example, if you run the `GRANT SELECT ON db.* TO john` query and then create a new table `db.new_table`, the user `john` will be able to run the `SELECT * FROM db.new_table` query.
|
||||
|
||||
You can specify asterisk **only** for the prefixes:
|
||||
```sql
|
||||
GRANT SELECT ON db.* TO john -- correct
|
||||
GRANT SELECT ON db*.* TO john -- correct
|
||||
|
||||
GRANT SELECT ON *.my_table TO john -- wrong
|
||||
GRANT SELECT ON foo*bar TO john -- wrong
|
||||
GRANT SELECT ON *suffix TO john -- wrong
|
||||
```
|
||||
|
||||
## Privileges
|
||||
|
||||
Privilege is a permission to execute specific kind of queries.
|
||||
|
||||
@ -58,7 +58,9 @@ KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'
|
||||
KILL QUERY WHERE user='username' SYNC
|
||||
```
|
||||
|
||||
:::tip If you are killing a query in ClickHouse Cloud or in a self-managed cluster, then be sure to use the ```ON CLUSTER [cluster-name]``` option, in order to ensure the query is killed on all replicas:::
|
||||
:::tip
|
||||
If you are killing a query in ClickHouse Cloud or in a self-managed cluster, then be sure to use the ```ON CLUSTER [cluster-name]``` option, in order to ensure the query is killed on all replicas
|
||||
:::
|
||||
|
||||
Read-only users can only stop their own queries.
|
||||
|
||||
|
||||
@ -468,6 +468,53 @@ WITH toDateTime64('2020-01-01 10:20:30.999', 3) AS dt64 SELECT toStartOfSecond(d
|
||||
`toStartOfInterval(t, INTERVAL 1 day)` возвращает то же самое, что и `toStartOfDay(t)`,
|
||||
`toStartOfInterval(t, INTERVAL 15 minute)` возвращает то же самое, что и `toStartOfFifteenMinutes(t)`, и т.п.
|
||||
|
||||
Вычисление происхожит относительно конкретного времени:
|
||||
|
||||
| Interval | Start |
|
||||
|-------------|------------------------|
|
||||
| YEAR | year 0 |
|
||||
| QUARTER | 1900 Q1 |
|
||||
| MONTH | 1900 January |
|
||||
| WEEK | 1970, 1st week (01-05) |
|
||||
| DAY | 1970-01-01 |
|
||||
| HOUR | (*) |
|
||||
| MINUTE | 1970-01-01 00:00:00 |
|
||||
| SECOND | 1970-01-01 00:00:00 |
|
||||
| MILLISECOND | 1970-01-01 00:00:00 |
|
||||
| MICROSECOND | 1970-01-01 00:00:00 |
|
||||
| NANOSECOND | 1970-01-01 00:00:00 |
|
||||
|
||||
(*) часовые интервалы особенные: Вычисления всегда происходят относительно 00:00:00 (полночь) текущего дня. В результате, только
|
||||
часовые значения полезны только между 1 и 23.
|
||||
|
||||
Если была указана единица измерения `WEEK`, то `toStartOfInterval` предполагает, что неделя начинается в понедельник. Важно, что это поведение отдичается от поведения функции `toStartOfWeek` в которой неделя по умолчанию начинается у воскресенье.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
```sql
|
||||
toStartOfInterval(value, INTERVAL x unit[, time_zone])
|
||||
toStartOfInterval(value, INTERVAL x unit[, origin[, time_zone]])
|
||||
```
|
||||
|
||||
Вторая перегрузка эмулирует функцию `time_bucket()` из TimescaleDB, и функцию `date_bin()` из PostgreSQL, например:
|
||||
|
||||
``` SQL
|
||||
SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` reference
|
||||
┌───toStartOfInterval(...)─┐
|
||||
│ 2023-01-01 14:44:30 │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
Синонимы: `time_bucket`, `date_bin`.
|
||||
|
||||
**См. также**
|
||||
- [date_trunc](#date_trunc)
|
||||
|
||||
## toTime {#totime}
|
||||
|
||||
Переводит дату-с-временем на некоторую фиксированную дату, сохраняя при этом время.
|
||||
|
||||
@ -124,40 +124,6 @@ SELECT hex(sipHash128('foo', '\x01', 3));
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
## RIPEMD160
|
||||
|
||||
Генерирует [RIPEMD-160](https://en.wikipedia.org/wiki/RIPEMD) хеш строки.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
```sql
|
||||
RIPEMD160(input)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `input`: Строка [String](../data-types/string.md)
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- [UInt256](../data-types/int-uint.md), где 160-битный хеш RIPEMD-160 хранится в первых 20 байтах. Оставшиеся 12 байт заполняются нулями.
|
||||
|
||||
**Пример**
|
||||
|
||||
Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой
|
||||
|
||||
Запрос:
|
||||
|
||||
```sql
|
||||
SELECT hex(RIPEMD160('The quick brown fox jumps over the lazy dog'));
|
||||
```
|
||||
Результат:
|
||||
```response
|
||||
┌─hex(RIPEMD160('The quick brown fox jumps over the lazy dog'))─┐
|
||||
│ 37F332F68DB77BD9D7EDD4969571AD671CF9DD3B │
|
||||
└───────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## cityHash64 {#cityhash64}
|
||||
|
||||
Генерирует 64-х битное значение [CityHash](https://github.com/google/cityhash).
|
||||
|
||||
@ -26,7 +26,7 @@ public:
|
||||
String path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
|
||||
std::optional<String> path_to = getValueFromCommandLineOptionsWithOptional<String>(options, "path-to");
|
||||
|
||||
auto in = disk.getDisk()->readFile(path_from);
|
||||
auto in = disk.getDisk()->readFile(path_from, getReadSettings());
|
||||
std::unique_ptr<WriteBufferFromFileBase> out = {};
|
||||
if (path_to.has_value())
|
||||
{
|
||||
|
||||
@ -39,7 +39,7 @@ public:
|
||||
else
|
||||
{
|
||||
String relative_path_from = disk.getRelativeFromRoot(path_from.value());
|
||||
return disk.getDisk()->readFile(relative_path_from);
|
||||
return disk.getDisk()->readFile(relative_path_from, getReadSettings());
|
||||
}
|
||||
}();
|
||||
|
||||
|
||||
@ -7,42 +7,34 @@ max-branches=50
|
||||
max-nested-blocks=10
|
||||
max-statements=200
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
src_paths = ["src", "tests/ci", "tests/sqllogic"]
|
||||
|
||||
[tool.pylint.FORMAT]
|
||||
#ignore-long-lines = (# )?<?https?://\S+>?$
|
||||
|
||||
[tool.pylint.'MESSAGES CONTROL']
|
||||
# pytest.mark.parametrize is not callable (not-callable)
|
||||
disable = '''
|
||||
pointless-string-statement,
|
||||
line-too-long,
|
||||
missing-docstring,
|
||||
too-few-public-methods,
|
||||
invalid-name,
|
||||
too-many-arguments,
|
||||
keyword-arg-before-vararg,
|
||||
too-many-locals,
|
||||
too-many-instance-attributes,
|
||||
cell-var-from-loop,
|
||||
fixme,
|
||||
too-many-public-methods,
|
||||
wildcard-import,
|
||||
unused-wildcard-import,
|
||||
singleton-comparison,
|
||||
not-callable,
|
||||
redefined-outer-name,
|
||||
broad-except,
|
||||
bare-except,
|
||||
no-else-return,
|
||||
global-statement,
|
||||
f-string-without-interpolation,
|
||||
consider-using-with,
|
||||
use-maxsplit-arg,
|
||||
'''
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
src_paths = ["src", "tests/ci", "tests/sqllogic"]
|
||||
|
||||
[tool.black]
|
||||
required-version = 24
|
||||
|
||||
[tool.pylint.SIMILARITIES]
|
||||
# due to SQL
|
||||
min-similarity-lines=1000
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -2,6 +2,8 @@
|
||||
|
||||
#include <base/types.h>
|
||||
#include <Access/Common/AccessRightsElement.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
@ -47,6 +49,13 @@ public:
|
||||
void grant(const AccessRightsElement & element);
|
||||
void grant(const AccessRightsElements & elements);
|
||||
|
||||
void grantWildcard(const AccessFlags & flags);
|
||||
void grantWildcard(const AccessFlags & flags, std::string_view database);
|
||||
void grantWildcard(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void grantWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void grantWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void grantWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
|
||||
void grantWithGrantOption(const AccessFlags & flags);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
@ -56,6 +65,13 @@ public:
|
||||
void grantWithGrantOption(const AccessRightsElement & element);
|
||||
void grantWithGrantOption(const AccessRightsElements & elements);
|
||||
|
||||
void grantWildcardWithGrantOption(const AccessFlags & flags);
|
||||
void grantWildcardWithGrantOption(const AccessFlags & flags, std::string_view database);
|
||||
void grantWildcardWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void grantWildcardWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void grantWildcardWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void grantWildcardWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
|
||||
/// Revokes a specified access granted earlier on a specified database/table/column.
|
||||
/// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear();
|
||||
void revoke(const AccessFlags & flags);
|
||||
@ -67,6 +83,13 @@ public:
|
||||
void revoke(const AccessRightsElement & element);
|
||||
void revoke(const AccessRightsElements & elements);
|
||||
|
||||
void revokeWildcard(const AccessFlags & flags);
|
||||
void revokeWildcard(const AccessFlags & flags, std::string_view database);
|
||||
void revokeWildcard(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void revokeWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void revokeWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void revokeWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
|
||||
void revokeGrantOption(const AccessFlags & flags);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
@ -76,6 +99,13 @@ public:
|
||||
void revokeGrantOption(const AccessRightsElement & element);
|
||||
void revokeGrantOption(const AccessRightsElements & elements);
|
||||
|
||||
void revokeWildcardGrantOption(const AccessFlags & flags);
|
||||
void revokeWildcardGrantOption(const AccessFlags & flags, std::string_view database);
|
||||
void revokeWildcardGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void revokeWildcardGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void revokeWildcardGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void revokeWildcardGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
|
||||
/// Whether a specified access granted.
|
||||
bool isGranted(const AccessFlags & flags) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database) const;
|
||||
@ -86,6 +116,13 @@ public:
|
||||
bool isGranted(const AccessRightsElement & element) const;
|
||||
bool isGranted(const AccessRightsElements & elements) const;
|
||||
|
||||
bool isGrantedWildcard(const AccessFlags & flags) const;
|
||||
bool isGrantedWildcard(const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGrantedWildcard(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGrantedWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGrantedWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGrantedWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
|
||||
bool hasGrantOption(const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
@ -95,6 +132,13 @@ public:
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
|
||||
bool hasGrantOptionWildcard(const AccessFlags & flags) const;
|
||||
bool hasGrantOptionWildcard(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOptionWildcard(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOptionWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOptionWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOptionWildcard(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
|
||||
/// Checks if a given `access_rights` is a subset for the current access rights.
|
||||
bool contains(const AccessRights & access_rights) const;
|
||||
bool containsWithGrantOption(const AccessRights & access_rights) const;
|
||||
@ -111,9 +155,7 @@ public:
|
||||
const AccessFlags & flags,
|
||||
const AccessFlags & min_flags_with_children,
|
||||
const AccessFlags & max_flags_with_children,
|
||||
std::string_view database,
|
||||
std::string_view table,
|
||||
std::string_view column,
|
||||
const size_t level,
|
||||
bool grant_option)>;
|
||||
void modifyFlags(const ModifyFlagsFunction & function);
|
||||
|
||||
@ -123,48 +165,50 @@ public:
|
||||
/// Makes full access rights (GRANT ALL ON *.* WITH GRANT OPTION).
|
||||
static AccessRights getFullAccess();
|
||||
|
||||
/// Methods for tests
|
||||
void dumpTree(WriteBuffer & buffer) const;
|
||||
std::vector<String> dumpNodes() const;
|
||||
|
||||
private:
|
||||
template <bool with_grant_option, typename... Args>
|
||||
template <bool with_grant_option, bool wildcard, typename... Args>
|
||||
void grantImpl(const AccessFlags & flags, const Args &... args);
|
||||
|
||||
template <bool with_grant_option>
|
||||
template <bool with_grant_option, bool wildcard>
|
||||
void grantImpl(const AccessRightsElement & element);
|
||||
|
||||
template <bool with_grant_option>
|
||||
template <bool with_grant_option, bool wildcard>
|
||||
void grantImpl(const AccessRightsElements & elements);
|
||||
|
||||
template <bool with_grant_option>
|
||||
template <bool with_grant_option, bool wildcard>
|
||||
void grantImplHelper(const AccessRightsElement & element);
|
||||
|
||||
template <bool grant_option, typename... Args>
|
||||
template <bool grant_option, bool wildcard, typename... Args>
|
||||
void revokeImpl(const AccessFlags & flags, const Args &... args);
|
||||
|
||||
template <bool grant_option>
|
||||
template <bool grant_option, bool wildcard>
|
||||
void revokeImpl(const AccessRightsElement & element);
|
||||
|
||||
template <bool grant_option>
|
||||
template <bool grant_option, bool wildcard>
|
||||
void revokeImpl(const AccessRightsElements & elements);
|
||||
|
||||
template <bool grant_option>
|
||||
template <bool grant_option, bool wildcard>
|
||||
void revokeImplHelper(const AccessRightsElement & element);
|
||||
|
||||
template <bool grant_option, typename... Args>
|
||||
template <bool grant_option, bool wildcard, typename... Args>
|
||||
bool isGrantedImpl(const AccessFlags & flags, const Args &... args) const;
|
||||
|
||||
template <bool grant_option>
|
||||
template <bool grant_option, bool wildcard>
|
||||
bool isGrantedImpl(const AccessRightsElement & element) const;
|
||||
|
||||
template <bool grant_option>
|
||||
template <bool grant_option, bool wildcard>
|
||||
bool isGrantedImpl(const AccessRightsElements & elements) const;
|
||||
|
||||
template <bool grant_option>
|
||||
bool containsImpl(const AccessRights & other) const;
|
||||
|
||||
template <bool grant_option>
|
||||
template <bool grant_option, bool wildcard>
|
||||
bool isGrantedImplHelper(const AccessRightsElement & element) const;
|
||||
|
||||
void logTree() const;
|
||||
|
||||
struct Node;
|
||||
std::unique_ptr<Node> root;
|
||||
std::unique_ptr<Node> root_with_grant_option;
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
#include <Access/Common/AccessRightsElement.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <Parsers/IAST.h>
|
||||
|
||||
#include <boost/range/algorithm_ext/erase.hpp>
|
||||
|
||||
|
||||
@ -7,48 +11,6 @@ namespace DB
|
||||
{
|
||||
namespace
|
||||
{
|
||||
void formatColumnNames(const Strings & columns, String & result)
|
||||
{
|
||||
result += "(";
|
||||
bool need_comma = false;
|
||||
for (const auto & column : columns)
|
||||
{
|
||||
if (need_comma)
|
||||
result += ", ";
|
||||
need_comma = true;
|
||||
result += backQuoteIfNeed(column);
|
||||
}
|
||||
result += ")";
|
||||
}
|
||||
|
||||
void formatONClause(const AccessRightsElement & element, String & result)
|
||||
{
|
||||
result += "ON ";
|
||||
if (element.isGlobalWithParameter())
|
||||
{
|
||||
if (element.any_parameter)
|
||||
result += "*";
|
||||
else
|
||||
result += backQuoteIfNeed(element.parameter);
|
||||
}
|
||||
else if (element.any_database)
|
||||
{
|
||||
result += "*.*";
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!element.database.empty())
|
||||
{
|
||||
result += backQuoteIfNeed(element.database);
|
||||
result += ".";
|
||||
}
|
||||
if (element.any_table)
|
||||
result += "*";
|
||||
else
|
||||
result += backQuoteIfNeed(element.table);
|
||||
}
|
||||
}
|
||||
|
||||
void formatOptions(bool grant_option, bool is_partial_revoke, String & result)
|
||||
{
|
||||
if (is_partial_revoke)
|
||||
@ -67,20 +29,16 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
void formatAccessFlagsWithColumns(const AccessFlags & access_flags, const Strings & columns, bool any_column, String & result)
|
||||
void formatAccessFlagsWithColumns(const AccessRightsElement & element, String & result)
|
||||
{
|
||||
String columns_as_str;
|
||||
if (!any_column)
|
||||
if (!element.anyColumn())
|
||||
{
|
||||
if (columns.empty())
|
||||
{
|
||||
result += "USAGE";
|
||||
return;
|
||||
}
|
||||
formatColumnNames(columns, columns_as_str);
|
||||
WriteBufferFromString buffer(columns_as_str);
|
||||
element.formatColumnNames(buffer);
|
||||
}
|
||||
|
||||
auto keywords = access_flags.toKeywords();
|
||||
auto keywords = element.access_flags.toKeywords();
|
||||
if (keywords.empty())
|
||||
{
|
||||
result += "USAGE";
|
||||
@ -101,9 +59,13 @@ namespace
|
||||
String toStringImpl(const AccessRightsElement & element, bool with_options)
|
||||
{
|
||||
String result;
|
||||
formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column, result);
|
||||
formatAccessFlagsWithColumns(element, result);
|
||||
result += " ";
|
||||
formatONClause(element, result);
|
||||
|
||||
WriteBufferFromOwnString buffer;
|
||||
element.formatONClause(buffer);
|
||||
result += buffer.str();
|
||||
|
||||
if (with_options)
|
||||
formatOptions(element.grant_option, element.is_partial_revoke, result);
|
||||
return result;
|
||||
@ -123,7 +85,7 @@ namespace
|
||||
|
||||
if (!part.empty())
|
||||
part += ", ";
|
||||
formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column, part);
|
||||
formatAccessFlagsWithColumns(element, part);
|
||||
|
||||
bool next_element_uses_same_table_and_options = false;
|
||||
if (i != elements.size() - 1)
|
||||
@ -138,7 +100,10 @@ namespace
|
||||
if (!next_element_uses_same_table_and_options)
|
||||
{
|
||||
part += " ";
|
||||
formatONClause(element, part);
|
||||
WriteBufferFromOwnString buffer;
|
||||
element.formatONClause(buffer);
|
||||
part += buffer.str();
|
||||
|
||||
if (with_options)
|
||||
formatOptions(element.grant_option, element.is_partial_revoke, part);
|
||||
if (result.empty())
|
||||
@ -153,14 +118,66 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
void AccessRightsElement::formatColumnNames(WriteBuffer & buffer) const
|
||||
{
|
||||
buffer << "(";
|
||||
bool need_comma = false;
|
||||
for (const auto & column : columns)
|
||||
{
|
||||
if (std::exchange(need_comma, true))
|
||||
buffer << ", ";
|
||||
buffer << backQuoteIfNeed(column);
|
||||
if (wildcard)
|
||||
buffer << "*";
|
||||
}
|
||||
buffer << ")";
|
||||
}
|
||||
|
||||
void AccessRightsElement::formatONClause(WriteBuffer & buffer, bool hilite) const
|
||||
{
|
||||
buffer << (hilite ? IAST::hilite_keyword : "") << "ON " << (hilite ? IAST::hilite_none : "");
|
||||
if (isGlobalWithParameter())
|
||||
{
|
||||
if (anyParameter())
|
||||
buffer << "*";
|
||||
else
|
||||
{
|
||||
buffer << backQuoteIfNeed(parameter);
|
||||
if (wildcard)
|
||||
buffer << "*";
|
||||
}
|
||||
}
|
||||
else if (anyDatabase())
|
||||
buffer << "*.*";
|
||||
else if (!table.empty())
|
||||
{
|
||||
if (!database.empty())
|
||||
buffer << backQuoteIfNeed(database) << ".";
|
||||
|
||||
buffer << backQuoteIfNeed(table);
|
||||
|
||||
if (columns.empty() && wildcard)
|
||||
buffer << "*";
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer << backQuoteIfNeed(database);
|
||||
|
||||
if (wildcard)
|
||||
buffer << "*";
|
||||
|
||||
buffer << ".*";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
|
||||
: access_flags(access_flags_), database(database_), parameter(database_), any_database(false), any_parameter(false)
|
||||
: access_flags(access_flags_), database(database_), parameter(database_)
|
||||
{
|
||||
}
|
||||
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_)
|
||||
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
|
||||
: access_flags(access_flags_), database(database_), table(table_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -170,10 +187,6 @@ AccessRightsElement::AccessRightsElement(
|
||||
, database(database_)
|
||||
, table(table_)
|
||||
, columns({String{column_}})
|
||||
, any_database(false)
|
||||
, any_table(false)
|
||||
, any_column(false)
|
||||
, any_parameter(false)
|
||||
{
|
||||
}
|
||||
|
||||
@ -182,7 +195,7 @@ AccessRightsElement::AccessRightsElement(
|
||||
std::string_view database_,
|
||||
std::string_view table_,
|
||||
const std::vector<std::string_view> & columns_)
|
||||
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
|
||||
: access_flags(access_flags_), database(database_), table(table_)
|
||||
{
|
||||
columns.resize(columns_.size());
|
||||
for (size_t i = 0; i != columns_.size(); ++i)
|
||||
@ -195,22 +208,18 @@ AccessRightsElement::AccessRightsElement(
|
||||
, database(database_)
|
||||
, table(table_)
|
||||
, columns(columns_)
|
||||
, any_database(false)
|
||||
, any_table(false)
|
||||
, any_column(false)
|
||||
, any_parameter(false)
|
||||
{
|
||||
}
|
||||
|
||||
void AccessRightsElement::eraseNonGrantable()
|
||||
{
|
||||
if (isGlobalWithParameter() && !any_parameter)
|
||||
if (isGlobalWithParameter() && !anyParameter())
|
||||
access_flags &= AccessFlags::allFlagsGrantableOnGlobalWithParameterLevel();
|
||||
else if (!any_column)
|
||||
else if (!anyColumn())
|
||||
access_flags &= AccessFlags::allFlagsGrantableOnColumnLevel();
|
||||
else if (!any_table)
|
||||
else if (!anyTable())
|
||||
access_flags &= AccessFlags::allFlagsGrantableOnTableLevel();
|
||||
else if (!any_database)
|
||||
else if (!anyDatabase())
|
||||
access_flags &= AccessFlags::allFlagsGrantableOnDatabaseLevel();
|
||||
else
|
||||
access_flags &= AccessFlags::allFlagsGrantableOnGlobalLevel();
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/Common/AccessFlags.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <tuple>
|
||||
|
||||
|
||||
@ -17,10 +18,8 @@ struct AccessRightsElement
|
||||
Strings columns;
|
||||
String parameter;
|
||||
|
||||
bool any_database = true;
|
||||
bool any_table = true;
|
||||
bool any_column = true;
|
||||
bool any_parameter = false;
|
||||
bool wildcard = false;
|
||||
bool default_database = false;
|
||||
|
||||
bool grant_option = false;
|
||||
bool is_partial_revoke = false;
|
||||
@ -47,28 +46,32 @@ struct AccessRightsElement
|
||||
AccessRightsElement(
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_);
|
||||
|
||||
bool empty() const { return !access_flags || (!any_column && columns.empty()); }
|
||||
bool empty() const { return !access_flags || (!anyColumn() && columns.empty()); }
|
||||
|
||||
auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns, any_parameter, parameter, grant_option, is_partial_revoke); }
|
||||
bool anyDatabase() const { return database.empty() && table.empty() && !default_database; }
|
||||
bool anyTable() const { return table.empty(); }
|
||||
bool anyColumn() const { return columns.empty(); }
|
||||
bool anyParameter() const { return parameter.empty(); }
|
||||
|
||||
auto toTuple() const { return std::tie(access_flags, default_database, database, table, columns, parameter, wildcard, grant_option, is_partial_revoke); }
|
||||
friend bool operator==(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() == right.toTuple(); }
|
||||
friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return !(left == right); }
|
||||
|
||||
bool sameDatabaseAndTableAndParameter(const AccessRightsElement & other) const
|
||||
{
|
||||
return sameDatabaseAndTable(other) && sameParameter(other);
|
||||
return sameDatabaseAndTable(other) && sameParameter(other) && (wildcard == other.wildcard);
|
||||
}
|
||||
|
||||
bool sameParameter(const AccessRightsElement & other) const
|
||||
{
|
||||
return (parameter == other.parameter) && (any_parameter == other.any_parameter)
|
||||
return (parameter == other.parameter) && (anyParameter() == other.anyParameter())
|
||||
&& (access_flags.getParameterType() == other.access_flags.getParameterType())
|
||||
&& (isGlobalWithParameter() == other.isGlobalWithParameter());
|
||||
}
|
||||
|
||||
bool sameDatabaseAndTable(const AccessRightsElement & other) const
|
||||
{
|
||||
return (database == other.database) && (any_database == other.any_database)
|
||||
&& (table == other.table) && (any_table == other.any_table);
|
||||
return (database == other.database) && (table == other.table) && (anyDatabase() == other.anyDatabase()) && (anyTable() == other.anyTable());
|
||||
}
|
||||
|
||||
bool sameOptions(const AccessRightsElement & other) const
|
||||
@ -79,7 +82,7 @@ struct AccessRightsElement
|
||||
/// Resets flags which cannot be granted.
|
||||
void eraseNonGrantable();
|
||||
|
||||
bool isEmptyDatabase() const { return !any_database && database.empty(); }
|
||||
bool isEmptyDatabase() const { return database.empty() and !anyDatabase(); }
|
||||
|
||||
/// If the database is empty, replaces it with `current_database`. Otherwise does nothing.
|
||||
void replaceEmptyDatabase(const String & current_database);
|
||||
@ -90,6 +93,9 @@ struct AccessRightsElement
|
||||
String toString() const;
|
||||
String toStringWithoutOptions() const;
|
||||
String toStringForAccessTypeSource() const;
|
||||
|
||||
void formatColumnNames(WriteBuffer & buffer) const;
|
||||
void formatONClause(WriteBuffer & buffer, bool hilite = false) const;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -71,12 +71,9 @@ namespace
|
||||
auto modifier = [&](const AccessFlags & flags,
|
||||
const AccessFlags & min_flags_with_children,
|
||||
const AccessFlags & max_flags_with_children,
|
||||
std::string_view database,
|
||||
std::string_view table,
|
||||
std::string_view column,
|
||||
const size_t level,
|
||||
bool /* grant_option */) -> AccessFlags
|
||||
{
|
||||
size_t level = !database.empty() + !table.empty() + !column.empty();
|
||||
AccessFlags res = flags;
|
||||
|
||||
/// CREATE_TABLE => CREATE_VIEW, DROP_TABLE => DROP_VIEW, ALTER_TABLE => ALTER_VIEW
|
||||
@ -567,7 +564,7 @@ std::shared_ptr<const AccessRights> ContextAccess::getAccessRightsWithImplicit()
|
||||
}
|
||||
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard, typename... Args>
|
||||
bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const
|
||||
{
|
||||
if (user_was_dropped)
|
||||
@ -620,10 +617,20 @@ bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlag
|
||||
|
||||
auto acs = getAccessRightsWithImplicit();
|
||||
bool granted;
|
||||
if constexpr (grant_option)
|
||||
granted = acs->hasGrantOption(flags, args...);
|
||||
if constexpr (wildcard)
|
||||
{
|
||||
if constexpr (grant_option)
|
||||
granted = acs->hasGrantOptionWildcard(flags, args...);
|
||||
else
|
||||
granted = acs->isGrantedWildcard(flags, args...);
|
||||
}
|
||||
else
|
||||
granted = acs->isGranted(flags, args...);
|
||||
{
|
||||
if constexpr (grant_option)
|
||||
granted = acs->hasGrantOption(flags, args...);
|
||||
else
|
||||
granted = acs->isGranted(flags, args...);
|
||||
}
|
||||
|
||||
if (!granted)
|
||||
{
|
||||
@ -738,99 +745,104 @@ bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlag
|
||||
return access_granted();
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard>
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(context, flags);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option, wildcard>(context, flags);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard, typename... Args>
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(context, flags, database.empty() ? params.current_database : database, args...);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option, wildcard>(context, flags, database.empty() ? params.current_database : database, args...);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard>
|
||||
bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const
|
||||
{
|
||||
assert(!element.grant_option || grant_option);
|
||||
if (element.isGlobalWithParameter())
|
||||
{
|
||||
if (element.any_parameter)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags);
|
||||
if (element.anyParameter())
|
||||
return checkAccessImpl<throw_if_denied, grant_option, wildcard>(context, element.access_flags);
|
||||
else
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.parameter);
|
||||
return checkAccessImpl<throw_if_denied, grant_option, wildcard>(context, element.access_flags, element.parameter);
|
||||
}
|
||||
else if (element.any_database)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags);
|
||||
else if (element.any_table)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database);
|
||||
else if (element.any_column)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database, element.table);
|
||||
else if (element.anyDatabase())
|
||||
return checkAccessImpl<throw_if_denied, grant_option, wildcard>(context, element.access_flags);
|
||||
else if (element.anyTable())
|
||||
return checkAccessImpl<throw_if_denied, grant_option, wildcard>(context, element.access_flags, element.database);
|
||||
else if (element.anyColumn())
|
||||
return checkAccessImpl<throw_if_denied, grant_option, wildcard>(context, element.access_flags, element.database, element.table);
|
||||
else
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database, element.table, element.columns);
|
||||
return checkAccessImpl<throw_if_denied, grant_option, wildcard>(context, element.access_flags, element.database, element.table, element.columns);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard>
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const
|
||||
{
|
||||
if constexpr (grant_option)
|
||||
if (element.wildcard)
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, true>(context, element);
|
||||
if (element.grant_option)
|
||||
return checkAccessImplHelper<throw_if_denied, true, true>(context, element);
|
||||
else
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option, true>(context, element);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (element.grant_option)
|
||||
return checkAccessImplHelper<throw_if_denied, true>(context, element);
|
||||
return checkAccessImplHelper<throw_if_denied, true, wildcard>(context, element);
|
||||
else
|
||||
return checkAccessImplHelper<throw_if_denied, false>(context, element);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option, wildcard>(context, element);
|
||||
}
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard>
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const
|
||||
{
|
||||
for (const auto & element : elements)
|
||||
if (!checkAccessImpl<throw_if_denied, grant_option>(context, element))
|
||||
{
|
||||
if (!checkAccessImpl<throw_if_denied, grant_option, wildcard>(context, element))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, false>(context, flags); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(context, flags, database); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(context, flags, database, table); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(context, flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, false>(context, element); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(context, elements); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, false, false>(context, flags); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false, false>(context, flags, database); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false, false>(context, flags, database, table); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false, false>(context, flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, false, false>(context, element); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, false, false>(context, elements); }
|
||||
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, true>(context, flags); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(context, flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(context, flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(context, flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, true>(context, element); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(context, elements); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, true, false>(context, flags); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true, false>(context, flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true, false>(context, flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true, false>(context, flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, true, false>(context, element); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, true, false>(context, elements); }
|
||||
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, false>(context, flags); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(context, flags, database); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(context, flags, database, table); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, false>(context, element); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, false>(context, elements); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, false, false>(context, flags); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false, false>(context, flags, database); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false, false>(context, flags, database, table); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false, false>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, false, false>(context, element); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, false, false>(context, elements); }
|
||||
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, true>(context, flags); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(context, flags, database); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(context, flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, true>(context, element); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, true>(context, elements); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, true, false>(context, flags); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true, false>(context, flags, database); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true, false>(context, flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true, false>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, true, false>(context, element); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, true, false>(context, elements); }
|
||||
|
||||
|
||||
template <bool throw_if_denied, typename Container, typename GetNameFunction>
|
||||
|
||||
@ -144,22 +144,22 @@ private:
|
||||
void setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> & roles_info_) const TSA_REQUIRES(mutex);
|
||||
void calculateAccessRights() const TSA_REQUIRES(mutex);
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard>
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard, typename... Args>
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard>
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard>
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard, typename... Args>
|
||||
bool checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
template <bool throw_if_denied, bool grant_option, bool wildcard>
|
||||
bool checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
|
||||
@ -1,8 +1,241 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <Access/AccessRights.h>
|
||||
#include <Access/AccessRights.cpp> // NOLINT(bugprone-suspicious-include)
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
|
||||
#include <list>
|
||||
|
||||
using namespace DB;
|
||||
|
||||
TEST(AccessRights, Radix)
|
||||
{
|
||||
AccessRights root;
|
||||
root.grant(AccessType::SELECT, "team");
|
||||
root.grant(AccessType::SELECT, "toast");
|
||||
root.grant(AccessType::SELECT, "toaster");
|
||||
root.grant(AccessType::INSERT, "toaster", "bread");
|
||||
root.grant(AccessType::ALTER_ADD_COLUMN, "toaster", "bread", "jam");
|
||||
|
||||
root.grantWildcard(AccessType::CREATE_TABLE, "t");
|
||||
|
||||
WriteBufferFromOwnString out;
|
||||
root.dumpTree(out);
|
||||
|
||||
ASSERT_EQ(out.str(),
|
||||
"Tree(): level=GLOBAL_LEVEL, name=NULL, flags=USAGE, min_flags=USAGE, max_flags=SELECT, INSERT, ALTER ADD COLUMN, CREATE TABLE, wildcard_grant=false, num_children=1\n"
|
||||
"Tree(): - level=DATABASE_LEVEL, name=t, flags=CREATE TABLE, min_flags=CREATE TABLE, max_flags=SELECT, INSERT, ALTER ADD COLUMN, CREATE TABLE, wildcard_grant=true, num_children=2\n"
|
||||
"Tree(): -- level=DATABASE_LEVEL, name=eam, flags=CREATE TABLE, min_flags=CREATE TABLE, max_flags=SELECT, CREATE TABLE, wildcard_grant=false, num_children=1\n"
|
||||
"Tree(): --- level=DATABASE_LEVEL, name=NULL, flags=SELECT, CREATE TABLE, min_flags=SELECT, CREATE TABLE, max_flags=SELECT, CREATE TABLE, wildcard_grant=false, num_children=0\n"
|
||||
"Tree(): -- level=DATABASE_LEVEL, name=oast, flags=CREATE TABLE, min_flags=CREATE TABLE, max_flags=SELECT, INSERT, ALTER ADD COLUMN, CREATE TABLE, wildcard_grant=false, num_children=2\n"
|
||||
"Tree(): --- level=DATABASE_LEVEL, name=NULL, flags=SELECT, CREATE TABLE, min_flags=SELECT, CREATE TABLE, max_flags=SELECT, CREATE TABLE, wildcard_grant=false, num_children=0\n"
|
||||
"Tree(): --- level=DATABASE_LEVEL, name=er, flags=CREATE TABLE, min_flags=CREATE TABLE, max_flags=SELECT, INSERT, ALTER ADD COLUMN, CREATE TABLE, wildcard_grant=false, num_children=1\n"
|
||||
"Tree(): ---- level=DATABASE_LEVEL, name=NULL, flags=SELECT, CREATE TABLE, min_flags=SELECT, CREATE TABLE, max_flags=SELECT, INSERT, ALTER ADD COLUMN, CREATE TABLE, wildcard_grant=false, num_children=1\n"
|
||||
"Tree(): ----- level=TABLE_LEVEL, name=bread, flags=SELECT, CREATE TABLE, min_flags=SELECT, CREATE TABLE, max_flags=SELECT, INSERT, ALTER ADD COLUMN, CREATE TABLE, wildcard_grant=false, num_children=1\n"
|
||||
"Tree(): ------ level=TABLE_LEVEL, name=NULL, flags=SELECT, INSERT, CREATE TABLE, min_flags=SELECT, INSERT, CREATE TABLE, max_flags=SELECT, INSERT, ALTER ADD COLUMN, CREATE TABLE, wildcard_grant=false, num_children=1\n"
|
||||
"Tree(): ------- level=COLUMN_LEVEL, name=jam, flags=SELECT, INSERT, min_flags=SELECT, INSERT, max_flags=SELECT, INSERT, ALTER ADD COLUMN, wildcard_grant=false, num_children=1\n"
|
||||
"Tree(): -------- level=COLUMN_LEVEL, name=NULL, flags=SELECT, INSERT, ALTER ADD COLUMN, min_flags=SELECT, INSERT, ALTER ADD COLUMN, max_flags=SELECT, INSERT, ALTER ADD COLUMN, wildcard_grant=false, num_children=0\n");
|
||||
}
|
||||
|
||||
TEST(AccessRights, GrantWildcard)
|
||||
{
|
||||
AccessRights root;
|
||||
root.grant(AccessType::SELECT, "team");
|
||||
root.grant(AccessType::SELECT, "toast");
|
||||
root.grant(AccessType::SELECT, "toaster");
|
||||
root.grant(AccessType::INSERT, "toaster", "bread");
|
||||
root.grant(AccessType::ALTER_ADD_COLUMN, "toaster", "bread", "jam");
|
||||
|
||||
root.grantWildcard(AccessType::CREATE_TABLE, "t");
|
||||
|
||||
ASSERT_EQ(root.isGranted(AccessType::CREATE_TABLE, "tick"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::CREATE_TABLE, "team"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::CREATE_TABLE, "to"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "t"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "to"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "team"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "toaster"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "toaster", "bread"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_ADD_COLUMN, "toaster", "bread", "jam"), true);
|
||||
|
||||
ASSERT_EQ(root.toString(), "GRANT CREATE TABLE ON t*.*, GRANT SELECT ON team.*, GRANT SELECT ON toast.*, GRANT SELECT ON toaster.*, GRANT INSERT, ALTER ADD COLUMN(jam) ON toaster.bread");
|
||||
|
||||
root.revokeWildcard(AccessType::CREATE_TABLE, "t");
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON team.*, GRANT SELECT ON toast.*, GRANT SELECT ON toaster.*, GRANT INSERT, ALTER ADD COLUMN(jam) ON toaster.bread");
|
||||
|
||||
root.revokeWildcard(AccessType::SELECT, "t");
|
||||
ASSERT_EQ(root.toString(), "GRANT INSERT, ALTER ADD COLUMN(jam) ON toaster.bread");
|
||||
|
||||
root.revokeWildcard(AccessType::ALL, "t");
|
||||
|
||||
ASSERT_EQ(root.toString(), "GRANT USAGE ON *.*");
|
||||
|
||||
root.grant(AccessType::SELECT);
|
||||
root.revokeWildcard(AccessType::SELECT, "test");
|
||||
root.grant(AccessType::SELECT, "tester", "foo");
|
||||
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON *.*, REVOKE SELECT ON test*.*, GRANT SELECT ON tester.foo");
|
||||
|
||||
root.grant(AccessType::SELECT);
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON *.*");
|
||||
|
||||
root = {};
|
||||
root.grant(AccessType::SELECT, "test");
|
||||
root.grantWildcard(AccessType::CREATE_TABLE, "test");
|
||||
ASSERT_EQ(root.toString(), "GRANT CREATE TABLE ON test*.*, GRANT SELECT ON test.*");
|
||||
|
||||
root = {};
|
||||
root.grant(AccessType::SELECT, "test");
|
||||
root.grantWildcard(AccessType::SELECT, "test");
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON test*.*");
|
||||
|
||||
root = {};
|
||||
root.grantWildcard(AccessType::SELECT, "default", "test");
|
||||
root.grantWildcard(AccessType::SELECT, "default", "t");
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON default.t*");
|
||||
|
||||
root = {};
|
||||
root.grant(AccessType::SELECT, "default", "t");
|
||||
root.grantWildcard(AccessType::INSERT, "default", "t");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "default", "t"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "default", "t"), true);
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON default.t, GRANT INSERT ON default.t*");
|
||||
|
||||
root.revoke(AccessType::INSERT, "default", "t");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "default", "t"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "default", "t"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "default", "test"), true);
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON default.t, GRANT INSERT ON default.t*, REVOKE INSERT ON default.t");
|
||||
|
||||
root = {};
|
||||
root.grant(AccessType::SELECT, "default", "t");
|
||||
root.revokeWildcard(AccessType::SELECT, "default", "t", "col");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "default", "t"), false);
|
||||
ASSERT_EQ(root.isGrantedWildcard(AccessType::SELECT, "default", "t"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "default", "t", "col"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "default", "t", "col1"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "default", "t", "co"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "default", "t", "test"), true);
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON default.t, REVOKE SELECT(col*) ON default.t");
|
||||
|
||||
root = {};
|
||||
root.grantWildcard(AccessType::ALTER_UPDATE, "prod");
|
||||
root.grant(AccessType::ALTER_UPDATE, "prod", "users");
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "prod"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "prod", "users"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "prod", "orders"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "prod"), false);
|
||||
ASSERT_EQ(root.toString(), "GRANT ALTER UPDATE ON prod*.*");
|
||||
|
||||
root.revoke(AccessType::ALTER_UPDATE, "prod", "users");
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "prod", "users"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "prod", "orders"), true);
|
||||
ASSERT_EQ(root.toString(), "GRANT ALTER UPDATE ON prod*.*, REVOKE ALTER UPDATE ON prod.users");
|
||||
|
||||
root.grantWildcard(AccessType::ALTER_DELETE, "prod");
|
||||
root.grant(AccessType::ALTER_DELETE, "prod", "archive");
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_DELETE, "prod", "archive"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_DELETE, "prod", "current"), true);
|
||||
|
||||
root.revokeWildcard(AccessType::ALTER_DELETE, "prod");
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_DELETE, "prod", "archive"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_DELETE, "prod", "current"), false);
|
||||
|
||||
ASSERT_EQ(root.toString(), "GRANT ALTER UPDATE ON prod*.*, REVOKE ALTER UPDATE ON prod.users");
|
||||
|
||||
root = {};
|
||||
root.grantWildcard(AccessType::SELECT, "test");
|
||||
root.grantWildcard(AccessType::INSERT, "test");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "test"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "test"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "testdata"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "testdata"), true);
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT, INSERT ON test*.*");
|
||||
|
||||
root.revokeWildcard(AccessType::SELECT, "test");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "test"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "test"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "testdata"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "testdata"), true);
|
||||
ASSERT_EQ(root.isGrantedWildcard(AccessType::INSERT, "testdata"), true);
|
||||
ASSERT_EQ(root.isGrantedWildcard(AccessType::INSERT, "test"), true);
|
||||
ASSERT_EQ(root.toString(), "GRANT INSERT ON test*.*");
|
||||
|
||||
root = {};
|
||||
root.grant(AccessType::SELECT, "foo");
|
||||
root.grantWildcard(AccessType::SELECT, "foo", "bar");
|
||||
ASSERT_EQ(root.isGrantedWildcard(AccessType::SELECT, "foo"), false);
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON foo.*");
|
||||
|
||||
root = {};
|
||||
root.grantWildcard(AccessType::ALTER_UPDATE, "");
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, ""), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "anything"), true);
|
||||
|
||||
root.revokeWildcard(AccessType::ALTER_UPDATE, "");
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, ""), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "anything"), false);
|
||||
|
||||
root.grantWildcard(AccessType::CREATE_VIEW, "proj");
|
||||
ASSERT_EQ(root.isGranted(AccessType::CREATE_VIEW, "project"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::CREATE_VIEW, "pro"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::CREATE_VIEW, "projX"), true);
|
||||
|
||||
root.revokeWildcard(AccessType::CREATE_VIEW, "proj");
|
||||
ASSERT_EQ(root.isGranted(AccessType::CREATE_VIEW, "project"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::CREATE_VIEW, "projX"), false);
|
||||
|
||||
root = {};
|
||||
root.grantWildcard(AccessType::SELECT, "db");
|
||||
root.grantWildcard(AccessType::INSERT, "db");
|
||||
root.grantWildcard(AccessType::ALTER_UPDATE, "db");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "db"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "db"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "db"), true);
|
||||
|
||||
root.revokeWildcard(AccessType::SELECT, "db");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "db"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "db"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "db"), true);
|
||||
|
||||
root.revokeWildcard(AccessType::INSERT, "db");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "db"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "db"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "db"), true);
|
||||
|
||||
root.revokeWildcard(AccessType::ALTER_UPDATE, "db");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "db"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "db"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::ALTER_UPDATE, "db"), false);
|
||||
|
||||
root = {};
|
||||
root.grant(AccessType::SELECT, "db", "table");
|
||||
root.revoke(AccessType::SELECT, "db", "table", "a");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "db", "table", "a"), false);
|
||||
|
||||
root = {};
|
||||
root.grant(AccessType::SHOW_NAMED_COLLECTIONS);
|
||||
root.revoke(AccessType::SHOW_NAMED_COLLECTIONS, "collection1");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SHOW_NAMED_COLLECTIONS, "collection1"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SHOW_NAMED_COLLECTIONS, "collection2"), true);
|
||||
|
||||
root = {};
|
||||
root.grant(AccessType::SHOW_NAMED_COLLECTIONS);
|
||||
root.revokeWildcard(AccessType::SHOW_NAMED_COLLECTIONS, "collection");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SHOW_NAMED_COLLECTIONS, "collection1"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SHOW_NAMED_COLLECTIONS, "collection2"), false);
|
||||
ASSERT_EQ(root.isGranted(AccessType::SHOW_NAMED_COLLECTIONS, "foo"), true);
|
||||
|
||||
root = {};
|
||||
root.grantWildcardWithGrantOption(AccessType::SELECT, "db");
|
||||
root.grant(AccessType::INSERT, "db", "table");
|
||||
ASSERT_EQ(root.isGranted(AccessType::SELECT, "db", "table"), true);
|
||||
ASSERT_EQ(root.hasGrantOption(AccessType::SELECT, "db_1", "table"), true);
|
||||
ASSERT_EQ(root.hasGrantOptionWildcard(AccessType::SELECT, "db", "table"), true);
|
||||
ASSERT_EQ(root.hasGrantOptionWildcard(AccessType::SELECT, "db"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "db", "table"), true);
|
||||
ASSERT_EQ(root.isGranted(AccessType::INSERT, "db", "other_table"), false);
|
||||
ASSERT_EQ(root.toString(), "GRANT SELECT ON db*.* WITH GRANT OPTION, GRANT INSERT ON db.`table`");
|
||||
}
|
||||
|
||||
TEST(AccessRights, Union)
|
||||
{
|
||||
@ -56,8 +289,37 @@ TEST(AccessRights, Union)
|
||||
"SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, "
|
||||
"SYSTEM UNLOAD PRIMARY KEY, dictGet ON db1.*, GRANT TABLE ENGINE ON db1, "
|
||||
"GRANT SET DEFINER ON db1, GRANT NAMED COLLECTION ADMIN ON db1");
|
||||
}
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1", "tb1", Strings{"col1", "col2", "test"});
|
||||
rhs.grant(AccessType::SELECT, "db1", "tb1");
|
||||
lhs.makeUnion(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT ON db1.tb1");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1", "tb1", Strings{"col1", "col2", "test"});
|
||||
rhs.grantWildcardWithGrantOption(AccessType::SELECT, "db1", "tb1");
|
||||
rhs.revokeWildcardGrantOption(AccessType::SELECT, "db1", "tb1", "col");
|
||||
lhs.makeUnion(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT ON db1.tb1* WITH GRANT OPTION, REVOKE GRANT OPTION SELECT(col*) ON db1.tb1");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1", "tb1", Strings{"col1", "col2", "test"});
|
||||
rhs.grantWildcardWithGrantOption(AccessType::SELECT, "db1", "tb1", "col");
|
||||
lhs.makeUnion(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT(col*) ON db1.tb1 WITH GRANT OPTION, GRANT SELECT(test) ON db1.tb1");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT);
|
||||
lhs.revoke(AccessType::SELECT, "test");
|
||||
rhs.grant(AccessType::SELECT, "test", "table");
|
||||
lhs.makeUnion(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT ON *.*, REVOKE SELECT ON test.*, GRANT SELECT ON test.`table`");
|
||||
}
|
||||
|
||||
TEST(AccessRights, Intersection)
|
||||
{
|
||||
@ -101,4 +363,138 @@ TEST(AccessRights, Intersection)
|
||||
rhs.grant(AccessType::ALL, "db1");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT INSERT ON db1.*");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1", "tb1", Strings{"col1", "col2", "test"});
|
||||
rhs.grant(AccessType::SELECT, "db1", "tb1");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT(col1, col2, test) ON db1.tb1");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1", "tb1", Strings{"col1", "col2", "test"});
|
||||
rhs.grantWildcardWithGrantOption(AccessType::SELECT, "db1", "tb1");
|
||||
rhs.revokeWildcardGrantOption(AccessType::SELECT, "db1", "tb1", "col");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT(col1, col2, test) ON db1.tb1");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1", "tb1", Strings{"col1", "col2", "test"});
|
||||
rhs.grantWildcard(AccessType::SELECT, "db1", "tb1", "col");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT(col1, col2) ON db1.tb1");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1", "tb1", Strings{"col1", "col2", "test"});
|
||||
lhs.grantWithGrantOption(AccessType::SELECT, "db1", "tb1", "col1");
|
||||
rhs.grantWildcard(AccessType::SELECT, "db1", "tb1", "col");
|
||||
rhs.grantWithGrantOption(AccessType::SELECT, "db1", "tb1", "col1");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT(col1) ON db1.tb1 WITH GRANT OPTION, GRANT SELECT(col2) ON db1.tb1");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "team");
|
||||
lhs.grant(AccessType::SELECT, "test");
|
||||
lhs.grant(AccessType::SELECT, "toast");
|
||||
lhs.grant(AccessType::SELECT, "toaster");
|
||||
rhs.grant(AccessType::SELECT, "test");
|
||||
rhs.grant(AccessType::SELECT, "tear");
|
||||
rhs.grant(AccessType::SELECT, "team");
|
||||
rhs.grant(AccessType::SELECT, "tea");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT ON team.*, GRANT SELECT ON test.*");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "team");
|
||||
lhs.grant(AccessType::SELECT, "test");
|
||||
lhs.grantWildcard(AccessType::SELECT, "toast");
|
||||
rhs.grant(AccessType::SELECT, "test");
|
||||
rhs.grant(AccessType::SELECT, "tear");
|
||||
rhs.grant(AccessType::SELECT, "team");
|
||||
rhs.grant(AccessType::SELECT, "tea");
|
||||
rhs.grant(AccessType::SELECT, "toaster");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT ON team.*, GRANT SELECT ON test.*, GRANT SELECT ON toaster.*");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "team");
|
||||
lhs.grantWildcard(AccessType::SELECT, "toast");
|
||||
rhs.grantWildcard(AccessType::SELECT, "tea");
|
||||
rhs.grant(AccessType::SELECT, "toaster", "foo");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT ON team.*, GRANT SELECT ON toaster.foo");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
rhs.grantWildcard(AccessType::SELECT, "toaster");
|
||||
lhs.grantWildcard(AccessType::SELECT, "toast");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT ON toaster*.*");
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grantWildcard(AccessType::SELECT, "toast");
|
||||
rhs.grant(AccessType::SELECT, "toaster");
|
||||
lhs.makeIntersection(rhs);
|
||||
ASSERT_EQ(lhs.toString(), "GRANT SELECT ON toaster.*");
|
||||
}
|
||||
|
||||
TEST(AccessRights, Contains)
|
||||
{
|
||||
AccessRights lhs, rhs;
|
||||
lhs.grant(AccessType::SELECT, "db1");
|
||||
rhs.grant(AccessType::SELECT, "db1", "tb1");
|
||||
ASSERT_EQ(lhs.contains(rhs), true);
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1");
|
||||
lhs.grant(AccessType::SELECT, "db2");
|
||||
rhs.grant(AccessType::SELECT, "db23", "tb1");
|
||||
rhs.grant(AccessType::SELECT, "db24", "tb1");
|
||||
ASSERT_EQ(lhs.contains(rhs), false);
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::SELECT, "db1");
|
||||
lhs.grant(AccessType::SELECT, "db2");
|
||||
rhs.grant(AccessType::SELECT, "db2", "tb1");
|
||||
ASSERT_EQ(lhs.contains(rhs), true);
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grant(AccessType::ALL, "db1");
|
||||
rhs.grant(AccessType::SELECT, "db1", "tb1");
|
||||
rhs.grant(AccessType::SELECT, "db1", "tb2", "col1");
|
||||
ASSERT_EQ(lhs.contains(rhs), true);
|
||||
|
||||
lhs = {};
|
||||
rhs = {};
|
||||
lhs.grantWildcard(AccessType::SELECT, "db");
|
||||
rhs.grant(AccessType::SELECT, "db1");
|
||||
ASSERT_EQ(lhs.contains(rhs), true);
|
||||
}
|
||||
|
||||
TEST(AccessRights, Iterator)
|
||||
{
|
||||
AccessRights root;
|
||||
root.grant(AccessType::SELECT, "team");
|
||||
root.grant(AccessType::SELECT, "toast");
|
||||
root.grant(AccessType::SELECT, "toaster");
|
||||
root.grant(AccessType::INSERT, "toaster", "bread");
|
||||
root.grant(AccessType::ALTER_ADD_COLUMN, "toaster", "bread", "jam");
|
||||
root.grantWildcard(AccessType::CREATE_TABLE, "t");
|
||||
|
||||
auto res = root.dumpNodes();
|
||||
ASSERT_EQ(res.size(), 4);
|
||||
ASSERT_EQ(res[0], "t");
|
||||
ASSERT_EQ(res[1], "team");
|
||||
ASSERT_EQ(res[2], "toast");
|
||||
ASSERT_EQ(res[3], "toaster");
|
||||
}
|
||||
|
||||
@ -40,6 +40,10 @@ Field zeroField(const Field & value)
|
||||
case Field::Types::Int128: return static_cast<Int128>(0);
|
||||
case Field::Types::UInt256: return static_cast<UInt256>(0);
|
||||
case Field::Types::Int256: return static_cast<Int256>(0);
|
||||
case Field::Types::Decimal32: return static_cast<Decimal32>(0);
|
||||
case Field::Types::Decimal64: return static_cast<Decimal64>(0);
|
||||
case Field::Types::Decimal128: return static_cast<Decimal128>(0);
|
||||
case Field::Types::Decimal256: return static_cast<Decimal256>(0);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
@ -423,6 +423,14 @@ QueryTreeNodePtr IdentifierResolver::tryResolveTableIdentifierFromDatabaseCatalo
|
||||
else
|
||||
storage = DatabaseCatalog::instance().tryGetTable(storage_id, context);
|
||||
|
||||
if (!storage && storage_id.hasUUID())
|
||||
{
|
||||
// If `storage_id` has UUID, it is possible that the UUID is removed from `DatabaseCatalog` after `context->resolveStorageID(storage_id)`
|
||||
// We try to get the table with the database name and the table name.
|
||||
auto database = DatabaseCatalog::instance().tryGetDatabase(storage_id.getDatabaseName());
|
||||
if (database)
|
||||
storage = database->tryGetTable(table_name, context);
|
||||
}
|
||||
if (!storage)
|
||||
return {};
|
||||
|
||||
|
||||
@ -218,6 +218,28 @@ void QueryAnalyzer::resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & ta
|
||||
}
|
||||
}
|
||||
|
||||
void QueryAnalyzer::resolveConstantExpression(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context)
|
||||
{
|
||||
IdentifierResolveScope scope(node, nullptr /*parent_scope*/);
|
||||
|
||||
if (!scope.context)
|
||||
scope.context = context;
|
||||
|
||||
auto node_type = node->getNodeType();
|
||||
|
||||
if (table_expression && node_type != QueryTreeNodeType::QUERY && node_type != QueryTreeNodeType::UNION)
|
||||
{
|
||||
scope.expression_join_tree_node = table_expression;
|
||||
validateTableExpressionModifiers(scope.expression_join_tree_node, scope);
|
||||
initializeTableExpressionData(scope.expression_join_tree_node, scope);
|
||||
}
|
||||
|
||||
if (node_type == QueryTreeNodeType::LIST)
|
||||
resolveExpressionNodeList(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
|
||||
else
|
||||
resolveExpressionNode(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
|
||||
}
|
||||
|
||||
std::optional<JoinTableSide> QueryAnalyzer::getColumnSideFromJoinTree(const QueryTreeNodePtr & resolved_identifier, const JoinNode & join_node)
|
||||
{
|
||||
if (resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
|
||||
|
||||
@ -119,6 +119,8 @@ public:
|
||||
|
||||
void resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context);
|
||||
|
||||
void resolveConstantExpression(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context);
|
||||
|
||||
private:
|
||||
/// Utility functions
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <Common/tests/gtest_global_context.h>
|
||||
#include <Backups/BackupEntryFromAppendOnlyFile.h>
|
||||
#include <Backups/BackupEntryFromImmutableFile.h>
|
||||
#include <Backups/BackupEntryFromSmallFile.h>
|
||||
@ -217,8 +218,9 @@ TEST_F(BackupEntriesTest, PartialChecksumBeforeFullChecksum)
|
||||
|
||||
TEST_F(BackupEntriesTest, BackupEntryFromSmallFile)
|
||||
{
|
||||
auto read_settings = getContext().context->getReadSettings();
|
||||
writeFile(local_disk, "a.txt");
|
||||
auto entry = std::make_shared<BackupEntryFromSmallFile>(local_disk, "a.txt", ReadSettings{});
|
||||
auto entry = std::make_shared<BackupEntryFromSmallFile>(local_disk, "a.txt", read_settings);
|
||||
|
||||
local_disk->removeFile("a.txt");
|
||||
|
||||
@ -234,12 +236,13 @@ TEST_F(BackupEntriesTest, BackupEntryFromSmallFile)
|
||||
|
||||
TEST_F(BackupEntriesTest, DecryptedEntriesFromEncryptedDisk)
|
||||
{
|
||||
auto read_settings = getContext().context->getReadSettings();
|
||||
{
|
||||
writeFile(encrypted_disk, "a.txt");
|
||||
std::pair<BackupEntryPtr, bool /* partial_checksum_allowed */> test_cases[]
|
||||
= {{std::make_shared<BackupEntryFromImmutableFile>(encrypted_disk, "a.txt"), false},
|
||||
{std::make_shared<BackupEntryFromAppendOnlyFile>(encrypted_disk, "a.txt"), true},
|
||||
{std::make_shared<BackupEntryFromSmallFile>(encrypted_disk, "a.txt", ReadSettings{}), true}};
|
||||
{std::make_shared<BackupEntryFromSmallFile>(encrypted_disk, "a.txt", read_settings), true}};
|
||||
for (const auto & [entry, partial_checksum_allowed] : test_cases)
|
||||
{
|
||||
EXPECT_EQ(entry->getSize(), 9);
|
||||
@ -258,7 +261,7 @@ TEST_F(BackupEntriesTest, DecryptedEntriesFromEncryptedDisk)
|
||||
BackupEntryPtr entries[]
|
||||
= {std::make_shared<BackupEntryFromImmutableFile>(encrypted_disk, "empty.txt"),
|
||||
std::make_shared<BackupEntryFromAppendOnlyFile>(encrypted_disk, "empty.txt"),
|
||||
std::make_shared<BackupEntryFromSmallFile>(encrypted_disk, "empty.txt", ReadSettings{})};
|
||||
std::make_shared<BackupEntryFromSmallFile>(encrypted_disk, "empty.txt", read_settings)};
|
||||
for (const auto & entry : entries)
|
||||
{
|
||||
EXPECT_EQ(entry->getSize(), 0);
|
||||
@ -283,12 +286,13 @@ TEST_F(BackupEntriesTest, DecryptedEntriesFromEncryptedDisk)
|
||||
|
||||
TEST_F(BackupEntriesTest, EncryptedEntriesFromEncryptedDisk)
|
||||
{
|
||||
auto read_settings = getContext().context->getReadSettings();
|
||||
{
|
||||
writeFile(encrypted_disk, "a.txt");
|
||||
BackupEntryPtr entries[]
|
||||
= {std::make_shared<BackupEntryFromImmutableFile>(encrypted_disk, "a.txt", /* copy_encrypted= */ true),
|
||||
std::make_shared<BackupEntryFromAppendOnlyFile>(encrypted_disk, "a.txt", /* copy_encrypted= */ true),
|
||||
std::make_shared<BackupEntryFromSmallFile>(encrypted_disk, "a.txt", ReadSettings{}, /* copy_encrypted= */ true)};
|
||||
std::make_shared<BackupEntryFromSmallFile>(encrypted_disk, "a.txt", read_settings, /* copy_encrypted= */ true)};
|
||||
|
||||
auto encrypted_checksum = getChecksum(entries[0]);
|
||||
EXPECT_NE(encrypted_checksum, NO_CHECKSUM);
|
||||
@ -322,7 +326,7 @@ TEST_F(BackupEntriesTest, EncryptedEntriesFromEncryptedDisk)
|
||||
BackupEntryPtr entries[]
|
||||
= {std::make_shared<BackupEntryFromImmutableFile>(encrypted_disk, "empty.txt", /* copy_encrypted= */ true),
|
||||
std::make_shared<BackupEntryFromAppendOnlyFile>(encrypted_disk, "empty.txt", /* copy_encrypted= */ true),
|
||||
std::make_shared<BackupEntryFromSmallFile>(encrypted_disk, "empty.txt", ReadSettings{}, /* copy_encrypted= */ true)};
|
||||
std::make_shared<BackupEntryFromSmallFile>(encrypted_disk, "empty.txt", read_settings, /* copy_encrypted= */ true)};
|
||||
for (const auto & entry : entries)
|
||||
{
|
||||
EXPECT_EQ(entry->getSize(), 0);
|
||||
|
||||
@ -452,7 +452,7 @@
|
||||
M(553, LZMA_STREAM_ENCODER_FAILED) \
|
||||
M(554, LZMA_STREAM_DECODER_FAILED) \
|
||||
M(555, ROCKSDB_ERROR) \
|
||||
M(556, SYNC_MYSQL_USER_ACCESS_ERROR)\
|
||||
M(556, SYNC_MYSQL_USER_ACCESS_ERROR) \
|
||||
M(557, UNKNOWN_UNION) \
|
||||
M(558, EXPECTED_ALL_OR_DISTINCT) \
|
||||
M(559, INVALID_GRPC_QUERY_INFO) \
|
||||
@ -578,7 +578,7 @@
|
||||
M(697, CANNOT_RESTORE_TO_NONENCRYPTED_DISK) \
|
||||
M(698, INVALID_REDIS_STORAGE_TYPE) \
|
||||
M(699, INVALID_REDIS_TABLE_STRUCTURE) \
|
||||
M(700, USER_SESSION_LIMIT_EXCEEDED) \
|
||||
M(700, USER_SESSION_LIMIT_EXCEEDED) \
|
||||
M(701, CLUSTER_DOESNT_EXIST) \
|
||||
M(702, CLIENT_INFO_DOES_NOT_MATCH) \
|
||||
M(703, INVALID_IDENTIFIER) \
|
||||
@ -610,15 +610,16 @@
|
||||
M(729, ILLEGAL_TIME_SERIES_TAGS) \
|
||||
M(730, REFRESH_FAILED) \
|
||||
M(731, QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE) \
|
||||
\
|
||||
M(733, TABLE_IS_BEING_RESTARTED) \
|
||||
\
|
||||
M(900, DISTRIBUTED_CACHE_ERROR) \
|
||||
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
|
||||
\
|
||||
\
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
M(1000, POCO_EXCEPTION) \
|
||||
M(1001, STD_EXCEPTION) \
|
||||
M(1002, UNKNOWN_EXCEPTION) \
|
||||
/* See END */
|
||||
/* See END */
|
||||
|
||||
#ifdef APPLY_FOR_EXTERNAL_ERROR_CODES
|
||||
#define APPLY_FOR_ERROR_CODES(M) APPLY_FOR_BUILTIN_ERROR_CODES(M) APPLY_FOR_EXTERNAL_ERROR_CODES(M)
|
||||
|
||||
@ -176,7 +176,7 @@ void FileChecker::load()
|
||||
if (!fileReallyExists(files_info_path))
|
||||
return;
|
||||
|
||||
std::unique_ptr<ReadBuffer> in = disk ? disk->readFile(files_info_path) : std::make_unique<ReadBufferFromFile>(files_info_path);
|
||||
std::unique_ptr<ReadBuffer> in = disk ? disk->readFile(files_info_path, getReadSettings()) : std::make_unique<ReadBufferFromFile>(files_info_path);
|
||||
WriteBufferFromOwnString out;
|
||||
|
||||
/// The JSON library does not support whitespace. We delete them. Inefficient.
|
||||
|
||||
@ -1,6 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Core/Settings.h>
|
||||
|
||||
namespace DB::Setting
|
||||
{
|
||||
extern const SettingsString local_filesystem_read_method;
|
||||
}
|
||||
|
||||
struct ContextHolder
|
||||
{
|
||||
@ -13,6 +19,7 @@ struct ContextHolder
|
||||
{
|
||||
context->makeGlobalContext();
|
||||
context->setPath("./");
|
||||
const_cast<DB::Settings &>(context->getSettingsRef())[DB::Setting::local_filesystem_read_method] = "pread";
|
||||
}
|
||||
|
||||
ContextHolder(ContextHolder &&) = default;
|
||||
|
||||
@ -217,7 +217,7 @@ public:
|
||||
std::move(file_buf),
|
||||
/* compressi)on level = */ 3,
|
||||
/* append_to_existing_file_ = */ mode == WriteMode::Append,
|
||||
[latest_log_disk, path = current_file_description->path] { return latest_log_disk->readFile(path); });
|
||||
[latest_log_disk, path = current_file_description->path, read_settings = getReadSettings()] { return latest_log_disk->readFile(path, read_settings); });
|
||||
|
||||
prealloc_done = false;
|
||||
}
|
||||
@ -601,7 +601,7 @@ public:
|
||||
explicit ChangelogReader(ChangelogFileDescriptionPtr changelog_description_) : changelog_description(changelog_description_)
|
||||
{
|
||||
compression_method = chooseCompressionMethod(changelog_description->path, "");
|
||||
auto read_buffer_from_file = changelog_description->disk->readFile(changelog_description->path);
|
||||
auto read_buffer_from_file = changelog_description->disk->readFile(changelog_description->path, getReadSettings());
|
||||
read_buf = wrapReadBufferWithCompressionMethod(std::move(read_buffer_from_file), compression_method);
|
||||
}
|
||||
|
||||
@ -728,7 +728,7 @@ void LogEntryStorage::prefetchCommitLogs()
|
||||
[&]
|
||||
{
|
||||
const auto & [changelog_description, position, count] = prefetch_file_info;
|
||||
auto file = changelog_description->disk->readFile(changelog_description->path, ReadSettings());
|
||||
auto file = changelog_description->disk->readFile(changelog_description->path, getReadSettings());
|
||||
file->seek(position, SEEK_SET);
|
||||
LOG_TRACE(
|
||||
log, "Prefetching {} log entries from path {}, from position {}", count, changelog_description->path, position);
|
||||
@ -1266,7 +1266,7 @@ LogEntryPtr LogEntryStorage::getEntry(uint64_t index) const
|
||||
[&]
|
||||
{
|
||||
const auto & [changelog_description, position, size] = it->second;
|
||||
auto file = changelog_description->disk->readFile(changelog_description->path, ReadSettings());
|
||||
auto file = changelog_description->disk->readFile(changelog_description->path, getReadSettings());
|
||||
file->seek(position, SEEK_SET);
|
||||
LOG_TRACE(
|
||||
log,
|
||||
@ -1392,7 +1392,7 @@ LogEntriesPtr LogEntryStorage::getLogEntriesBetween(uint64_t start, uint64_t end
|
||||
[&]
|
||||
{
|
||||
const auto & [file_description, start_position, count] = *read_info;
|
||||
auto file = file_description->disk->readFile(file_description->path);
|
||||
auto file = file_description->disk->readFile(file_description->path, getReadSettings());
|
||||
file->seek(start_position, SEEK_SET);
|
||||
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
|
||||
@ -729,7 +729,7 @@ nuraft::ptr<nuraft::buffer> KeeperSnapshotManager<Storage>::deserializeSnapshotB
|
||||
{
|
||||
const auto & [snapshot_path, snapshot_disk, size] = *existing_snapshots.at(up_to_log_idx);
|
||||
WriteBufferFromNuraftBuffer writer;
|
||||
auto reader = snapshot_disk->readFile(snapshot_path);
|
||||
auto reader = snapshot_disk->readFile(snapshot_path, getReadSettings());
|
||||
copyData(*reader, writer);
|
||||
return writer.getBuffer();
|
||||
}
|
||||
|
||||
@ -173,7 +173,7 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh
|
||||
|
||||
LOG_INFO(log, "Will try to upload snapshot on {} to S3", snapshot_path);
|
||||
|
||||
auto snapshot_file = snapshot_disk->readFile(snapshot_path);
|
||||
auto snapshot_file = snapshot_disk->readFile(snapshot_path, getReadSettings());
|
||||
|
||||
auto snapshot_name = fs::path(snapshot_path).filename().string();
|
||||
auto lock_file = fmt::format(".{}_LOCK", snapshot_name);
|
||||
|
||||
@ -339,7 +339,7 @@ void KeeperStateManager::save_state(const nuraft::srv_state & state)
|
||||
{
|
||||
auto buf = disk->writeFile(copy_lock_file);
|
||||
buf->finalize();
|
||||
disk->copyFile(server_state_file_name, *disk, old_path);
|
||||
disk->copyFile(server_state_file_name, *disk, old_path, ReadSettings{});
|
||||
disk->removeFile(copy_lock_file);
|
||||
disk->removeFile(old_path);
|
||||
}
|
||||
@ -374,7 +374,7 @@ nuraft::ptr<nuraft::srv_state> KeeperStateManager::read_state()
|
||||
{
|
||||
try
|
||||
{
|
||||
auto read_buf = disk->readFile(path);
|
||||
auto read_buf = disk->readFile(path, getReadSettings());
|
||||
auto content_size = read_buf->getFileSize();
|
||||
|
||||
if (content_size == 0)
|
||||
|
||||
@ -40,6 +40,7 @@ namespace ErrorCodes
|
||||
extern const int FILE_ALREADY_EXISTS;
|
||||
extern const int INCORRECT_QUERY;
|
||||
extern const int ABORTED;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
class AtomicDatabaseTablesSnapshotIterator final : public DatabaseTablesSnapshotIterator
|
||||
@ -237,7 +238,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
|
||||
return;
|
||||
db.table_name_to_path.emplace(table_name_, table_data_path_);
|
||||
if (table_->storesDataOnDisk())
|
||||
db.tryCreateSymlink(table_name_, table_data_path_);
|
||||
db.tryCreateSymlink(table_);
|
||||
};
|
||||
|
||||
auto assert_can_move_mat_view = [inside_database](const StoragePtr & table_)
|
||||
@ -363,7 +364,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora
|
||||
throw;
|
||||
}
|
||||
if (table->storesDataOnDisk())
|
||||
tryCreateSymlink(query.getTable(), table_data_path);
|
||||
tryCreateSymlink(table);
|
||||
}
|
||||
|
||||
void DatabaseAtomic::commitAlterTable(const StorageID & table_id, const String & table_metadata_tmp_path, const String & table_metadata_path,
|
||||
@ -500,7 +501,17 @@ LoadTaskPtr DatabaseAtomic::startupDatabaseAsync(AsyncLoader & async_loader, Loa
|
||||
|
||||
fs::create_directories(path_to_table_symlinks);
|
||||
for (const auto & table : table_names)
|
||||
tryCreateSymlink(table.first, table.second, true);
|
||||
{
|
||||
/// All tables in database should be loaded at this point
|
||||
StoragePtr table_ptr = tryGetTable(table.first, getContext());
|
||||
if (table_ptr)
|
||||
{
|
||||
if (table_ptr->storesDataOnDisk())
|
||||
tryCreateSymlink(table_ptr, true);
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Table {} is not loaded before database startup", table.first);
|
||||
}
|
||||
});
|
||||
std::scoped_lock lock(mutex);
|
||||
return startup_atomic_database_task = makeLoadTask(async_loader, {job});
|
||||
@ -528,12 +539,17 @@ void DatabaseAtomic::stopLoading()
|
||||
DatabaseOrdinary::stopLoading();
|
||||
}
|
||||
|
||||
void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String & actual_data_path, bool if_data_path_exist)
|
||||
void DatabaseAtomic::tryCreateSymlink(const StoragePtr & table, bool if_data_path_exist)
|
||||
{
|
||||
try
|
||||
{
|
||||
String table_name = table->getStorageID().getTableName();
|
||||
|
||||
if (!table->storesDataOnDisk())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Table {} doesn't have data path to create symlink", table_name);
|
||||
|
||||
String link = path_to_table_symlinks + escapeForFileName(table_name);
|
||||
fs::path data = fs::canonical(getContext()->getPath()) / actual_data_path;
|
||||
fs::path data = fs::weakly_canonical(table->getDataPaths()[0]);
|
||||
|
||||
/// If it already points where needed.
|
||||
std::error_code ec;
|
||||
|
||||
@ -60,7 +60,7 @@ public:
|
||||
|
||||
UUID tryGetTableUUID(const String & table_name) const override;
|
||||
|
||||
void tryCreateSymlink(const String & table_name, const String & actual_data_path, bool if_data_path_exist = false);
|
||||
void tryCreateSymlink(const StoragePtr & table, bool if_data_path_exist = false);
|
||||
void tryRemoveSymlink(const String & table_name);
|
||||
|
||||
void waitDetachedTableNotInUse(const UUID & uuid) override;
|
||||
|
||||
@ -392,7 +392,7 @@ size_t DiskEncrypted::getFileSize(const String & path) const
|
||||
UInt128 DiskEncrypted::getEncryptedFileIV(const String & path) const
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
auto read_buffer = delegate->readFile(wrapped_path, ReadSettings().adjustBufferSize(FileEncryption::Header::kSize));
|
||||
auto read_buffer = delegate->readFile(wrapped_path, getReadSettings().adjustBufferSize(FileEncryption::Header::kSize));
|
||||
if (read_buffer->eof())
|
||||
return 0;
|
||||
auto header = readHeader(*read_buffer);
|
||||
|
||||
@ -78,7 +78,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskEncryptedTransaction::writeFile( //
|
||||
if (old_file_size)
|
||||
{
|
||||
/// Append mode: we continue to use the same header.
|
||||
auto read_buffer = delegate_disk->readFile(wrapped_path, ReadSettings().adjustBufferSize(FileEncryption::Header::kSize));
|
||||
auto read_buffer = delegate_disk->readFile(wrapped_path, getReadSettings().adjustBufferSize(FileEncryption::Header::kSize));
|
||||
header = readHeader(*read_buffer);
|
||||
key = current_settings.findKeyByFingerprint(header.key_fingerprint, path);
|
||||
}
|
||||
|
||||
@ -196,6 +196,7 @@ void IDisk::checkAccessImpl(const String & path)
|
||||
try
|
||||
{
|
||||
const std::string_view payload("test", 4);
|
||||
const auto read_settings = getReadSettings();
|
||||
|
||||
/// write
|
||||
{
|
||||
@ -215,7 +216,7 @@ try
|
||||
|
||||
/// read
|
||||
{
|
||||
auto file = readFile(path);
|
||||
auto file = readFile(path, read_settings);
|
||||
String buf(payload.size(), '0');
|
||||
file->readStrict(buf.data(), buf.size());
|
||||
if (buf != payload)
|
||||
@ -227,7 +228,7 @@ try
|
||||
|
||||
/// read with offset
|
||||
{
|
||||
auto file = readFile(path);
|
||||
auto file = readFile(path, read_settings);
|
||||
auto offset = 2;
|
||||
String buf(payload.size() - offset, '0');
|
||||
file->seek(offset, 0);
|
||||
|
||||
@ -209,7 +209,7 @@ public:
|
||||
const String & from_file_path,
|
||||
IDisk & to_disk,
|
||||
const String & to_file_path,
|
||||
const ReadSettings & read_settings = {},
|
||||
const ReadSettings & read_settings,
|
||||
const WriteSettings & write_settings = {},
|
||||
const std::function<void()> & cancellation_hook = {});
|
||||
|
||||
@ -219,7 +219,7 @@ public:
|
||||
/// Open the file for read and return ReadBufferFromFileBase object.
|
||||
virtual std::unique_ptr<ReadBufferFromFileBase> readFile( /// NOLINT
|
||||
const String & path,
|
||||
const ReadSettings & settings = ReadSettings{},
|
||||
const ReadSettings & settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const = 0;
|
||||
|
||||
|
||||
@ -47,13 +47,13 @@ public:
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
|
||||
const StoredObject & object,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
|
||||
const StoredObjects & objects,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
|
||||
@ -33,13 +33,13 @@ public:
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
|
||||
const StoredObject & object,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
|
||||
const StoredObjects & objects,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
|
||||
@ -155,7 +155,7 @@ public:
|
||||
const String & from_file_path,
|
||||
IDisk & to_disk,
|
||||
const String & to_file_path,
|
||||
const ReadSettings & read_settings = {},
|
||||
const ReadSettings & read_settings,
|
||||
const WriteSettings & write_settings = {},
|
||||
const std::function<void()> & cancellation_hook = {}
|
||||
) override;
|
||||
|
||||
@ -73,7 +73,7 @@ int DiskObjectStorageRemoteMetadataRestoreHelper::readSchemaVersion(IObjectStora
|
||||
if (!object_storage->exists(object))
|
||||
return version;
|
||||
|
||||
auto buf = object_storage->readObject(object);
|
||||
auto buf = object_storage->readObject(object, ReadSettings{});
|
||||
readIntText(version, *buf);
|
||||
|
||||
return version;
|
||||
|
||||
@ -65,13 +65,13 @@ public:
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
|
||||
const StoredObject & object,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
|
||||
const StoredObjects & objects,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
|
||||
@ -146,14 +146,14 @@ public:
|
||||
/// Read single object
|
||||
virtual std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
|
||||
const StoredObject & object,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const = 0;
|
||||
|
||||
/// Read multiple objects with common prefix
|
||||
virtual std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
|
||||
const StoredObjects & objects,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const = 0;
|
||||
|
||||
|
||||
@ -30,13 +30,13 @@ public:
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
|
||||
const StoredObject & object,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
|
||||
const StoredObjects & objects,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
|
||||
@ -66,7 +66,7 @@ DirectoryIteratorPtr MetadataStorageFromDisk::iterateDirectory(const std::string
|
||||
|
||||
std::string MetadataStorageFromDisk::readFileToString(const std::string & path) const
|
||||
{
|
||||
auto buf = disk->readFile(path);
|
||||
auto buf = disk->readFile(path, ReadSettings{});
|
||||
std::string result;
|
||||
readStringUntilEOF(result, *buf);
|
||||
return result;
|
||||
|
||||
@ -275,7 +275,7 @@ void WriteFileOperation::execute(std::unique_lock<SharedMutex> &)
|
||||
if (disk.exists(path))
|
||||
{
|
||||
existed = true;
|
||||
auto buf = disk.readFile(path);
|
||||
auto buf = disk.readFile(path, ReadSettings{});
|
||||
readStringUntilEOF(prev_data, *buf);
|
||||
}
|
||||
auto buf = disk.writeFile(path);
|
||||
|
||||
@ -150,7 +150,7 @@ std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMo
|
||||
if (validate_content)
|
||||
{
|
||||
std::string data;
|
||||
auto read_buf = object_storage->readObject(metadata_object);
|
||||
auto read_buf = object_storage->readObject(metadata_object, ReadSettings{});
|
||||
readStringUntilEOF(data, *read_buf);
|
||||
if (data != path_from)
|
||||
throw Exception(
|
||||
|
||||
@ -51,7 +51,7 @@ std::shared_ptr<InMemoryPathMap> loadPathPrefixMap(const std::string & metadata_
|
||||
|
||||
LoggerPtr log = getLogger("MetadataStorageFromPlainObjectStorage");
|
||||
|
||||
ReadSettings settings;
|
||||
auto settings = getReadSettings();
|
||||
settings.enable_filesystem_cache = false;
|
||||
settings.remote_fs_method = RemoteFSReadMethod::read;
|
||||
settings.remote_fs_buffer_size = 1024; /// These files are small.
|
||||
|
||||
@ -85,13 +85,13 @@ public:
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
|
||||
const StoredObject & object,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
|
||||
const StoredObjects & objects,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
|
||||
@ -35,13 +35,13 @@ public:
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
|
||||
const StoredObject & object,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
|
||||
const StoredObjects & objects,
|
||||
const ReadSettings & read_settings = ReadSettings{},
|
||||
const ReadSettings & read_settings,
|
||||
std::optional<size_t> read_hint = {},
|
||||
std::optional<size_t> file_size = {}) const override;
|
||||
|
||||
|
||||
@ -49,7 +49,7 @@ TEST_F(DiskTest, writeFile)
|
||||
|
||||
String data;
|
||||
{
|
||||
std::unique_ptr<DB::ReadBuffer> in = disk->readFile("test_file");
|
||||
std::unique_ptr<DB::ReadBuffer> in = disk->readFile("test_file", DB::getReadSettings());
|
||||
readString(data, *in);
|
||||
}
|
||||
|
||||
@ -65,10 +65,12 @@ TEST_F(DiskTest, readFile)
|
||||
writeString("test data", *out);
|
||||
}
|
||||
|
||||
auto read_settings = DB::getReadSettings();
|
||||
|
||||
// Test SEEK_SET
|
||||
{
|
||||
String buf(4, '0');
|
||||
std::unique_ptr<DB::SeekableReadBuffer> in = disk->readFile("test_file");
|
||||
std::unique_ptr<DB::SeekableReadBuffer> in = disk->readFile("test_file", read_settings);
|
||||
|
||||
in->seek(5, SEEK_SET);
|
||||
|
||||
@ -78,7 +80,7 @@ TEST_F(DiskTest, readFile)
|
||||
|
||||
// Test SEEK_CUR
|
||||
{
|
||||
std::unique_ptr<DB::SeekableReadBuffer> in = disk->readFile("test_file");
|
||||
std::unique_ptr<DB::SeekableReadBuffer> in = disk->readFile("test_file", read_settings);
|
||||
String buf(4, '0');
|
||||
|
||||
in->readStrict(buf.data(), 4);
|
||||
|
||||
@ -3322,124 +3322,138 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
/// Complex case: one or more columns are serialized as a nested message.
|
||||
for (const auto & [field_descriptor, suffix] : field_descriptors_with_suffixes)
|
||||
{
|
||||
if (!suffix.empty())
|
||||
if (suffix.empty())
|
||||
continue;
|
||||
|
||||
std::vector<size_t> nested_column_indices;
|
||||
std::vector<std::string_view> nested_column_names;
|
||||
nested_column_indices.reserve(num_columns - used_column_indices.size());
|
||||
nested_column_names.reserve(num_columns - used_column_indices.size());
|
||||
nested_column_indices.push_back(column_idx);
|
||||
nested_column_names.push_back(suffix);
|
||||
|
||||
for (size_t j : collections::range(column_idx + 1, num_columns))
|
||||
{
|
||||
/// Complex case: one or more columns are serialized as a nested message.
|
||||
std::vector<size_t> nested_column_indices;
|
||||
std::vector<std::string_view> nested_column_names;
|
||||
nested_column_indices.reserve(num_columns - used_column_indices.size());
|
||||
nested_column_names.reserve(num_columns - used_column_indices.size());
|
||||
nested_column_indices.push_back(column_idx);
|
||||
nested_column_names.push_back(suffix);
|
||||
if (used_column_indices_sorted.count(j))
|
||||
continue;
|
||||
std::string_view other_suffix;
|
||||
if (!columnNameStartsWithFieldName(column_names[j], *field_descriptor, other_suffix))
|
||||
continue;
|
||||
nested_column_indices.push_back(j);
|
||||
nested_column_names.push_back(other_suffix);
|
||||
}
|
||||
|
||||
for (size_t j : collections::range(column_idx + 1, num_columns))
|
||||
DataTypes nested_data_types;
|
||||
nested_data_types.reserve(nested_column_indices.size());
|
||||
for (size_t j : nested_column_indices)
|
||||
nested_data_types.push_back(data_types[j]);
|
||||
|
||||
/// Now we have up to `nested_message_column_names.size()` columns
|
||||
/// which can be serialized as one or many nested message(s)
|
||||
|
||||
/// If the field is repeated, and ALL matching columns are array, we serialize as an array of nested messages.
|
||||
/// Otherwise, we first try to serialize those columns as one nested message,
|
||||
/// then, if failed, as an array of nested messages (on condition if those columns are array).
|
||||
|
||||
bool repeated_field_matching_nested_columns_are_all_arrays = false;
|
||||
bool repeated_field_matching_nested_columns_have_some_arrays = false;
|
||||
if (field_descriptor->is_repeated())
|
||||
{
|
||||
repeated_field_matching_nested_columns_are_all_arrays = true;
|
||||
for (const auto & nested_data_type : nested_data_types)
|
||||
{
|
||||
if (used_column_indices_sorted.count(j))
|
||||
continue;
|
||||
std::string_view other_suffix;
|
||||
if (!columnNameStartsWithFieldName(column_names[j], *field_descriptor, other_suffix))
|
||||
continue;
|
||||
nested_column_indices.push_back(j);
|
||||
nested_column_names.push_back(other_suffix);
|
||||
if (nested_data_type->getTypeId() == TypeIndex::Array)
|
||||
repeated_field_matching_nested_columns_have_some_arrays = true;
|
||||
else
|
||||
repeated_field_matching_nested_columns_are_all_arrays = false;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<size_t> used_column_indices_in_nested;
|
||||
auto attempt_build_serializer = [&](const DataTypes & passed_nested_data_types)
|
||||
{
|
||||
return buildMessageSerializerImpl(
|
||||
nested_column_names.size(),
|
||||
nested_column_names.data(),
|
||||
passed_nested_data_types.data(),
|
||||
*field_descriptor->message_type(),
|
||||
/* with_length_delimiter = */ false,
|
||||
google_wrappers_special_treatment,
|
||||
field_descriptor,
|
||||
used_column_indices_in_nested,
|
||||
/* columns_are_reordered_outside = */ true,
|
||||
/* check_nested_while_filling_missing_columns = */ false);
|
||||
|
||||
/// `columns_are_reordered_outside` is true because column indices are
|
||||
/// going to be transformed and then written to the outer message,
|
||||
/// see next calls to add_field_serializer() further below.
|
||||
};
|
||||
|
||||
auto attempt_unwrap_and_build_array_serializer = [&]()
|
||||
{
|
||||
DataTypes unwrapped_nested_data_types;
|
||||
unwrapped_nested_data_types.reserve(nested_data_types.size());
|
||||
|
||||
for (DataTypePtr & dt : nested_data_types)
|
||||
unwrapped_nested_data_types.push_back(assert_cast<const DataTypeArray &>(*dt).getNestedType());
|
||||
|
||||
if (auto serializer = attempt_build_serializer(unwrapped_nested_data_types))
|
||||
{
|
||||
std::vector<std::string_view> column_names_used;
|
||||
column_names_used.reserve(used_column_indices_in_nested.size());
|
||||
for (const size_t i : used_column_indices_in_nested)
|
||||
column_names_used.emplace_back(nested_column_names[i]);
|
||||
|
||||
auto array_serializer = std::make_unique<ProtobufSerializerFlattenedNestedAsArrayOfNestedMessages>(
|
||||
std::move(column_names_used), field_descriptor, std::move(serializer), get_root_desc_function);
|
||||
|
||||
transformColumnIndices(used_column_indices_in_nested, nested_column_indices);
|
||||
add_field_serializer(column_name,std::move(used_column_indices_in_nested), *field_descriptor, std::move(array_serializer));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
DataTypes nested_data_types;
|
||||
nested_data_types.reserve(nested_column_indices.size());
|
||||
for (size_t j : nested_column_indices)
|
||||
nested_data_types.push_back(data_types[j]);
|
||||
return false;
|
||||
};
|
||||
|
||||
/// Now we have up to `nested_message_column_names.size()` columns
|
||||
/// which can be serialized as a nested message.
|
||||
/// if the protobuf field has the repeated label,
|
||||
/// for ALL matching nested cols, since they are all of type array
|
||||
/// try as ProtobufSerializerFlattenedNestedAsArrayOfNestedMessages
|
||||
if (repeated_field_matching_nested_columns_are_all_arrays)
|
||||
{
|
||||
if (attempt_unwrap_and_build_array_serializer())
|
||||
break;
|
||||
}
|
||||
|
||||
/// We will try to serialize those columns as one nested message,
|
||||
/// then, if failed, as an array of nested messages (on condition if those columns are array).
|
||||
bool has_fallback_to_array_of_nested_messages = false;
|
||||
if (field_descriptor->is_repeated())
|
||||
/// for ALL matching nested cols
|
||||
/// try as ProtobufSerializerMessage
|
||||
try
|
||||
{
|
||||
if (auto serializer = attempt_build_serializer(nested_data_types))
|
||||
{
|
||||
bool has_arrays
|
||||
= boost::range::find_if(
|
||||
nested_data_types, [](const DataTypePtr & dt) { return (dt->getTypeId() == TypeIndex::Array); })
|
||||
!= nested_data_types.end();
|
||||
if (has_arrays)
|
||||
has_fallback_to_array_of_nested_messages = true;
|
||||
transformColumnIndices(used_column_indices_in_nested, nested_column_indices);
|
||||
add_field_serializer(column_name,std::move(used_column_indices_in_nested), *field_descriptor, std::move(serializer));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to serialize those columns as one nested message.
|
||||
try
|
||||
{
|
||||
std::vector<size_t> used_column_indices_in_nested;
|
||||
auto nested_message_serializer = buildMessageSerializerImpl(
|
||||
nested_column_names.size(),
|
||||
nested_column_names.data(),
|
||||
nested_data_types.data(),
|
||||
*field_descriptor->message_type(),
|
||||
/* with_length_delimiter = */ false,
|
||||
google_wrappers_special_treatment,
|
||||
field_descriptor,
|
||||
used_column_indices_in_nested,
|
||||
/* columns_are_reordered_outside = */ true,
|
||||
/* check_nested_while_filling_missing_columns = */ false);
|
||||
catch (Exception & e)
|
||||
{
|
||||
if ((e.code() != ErrorCodes::PROTOBUF_FIELD_NOT_REPEATED) || !repeated_field_matching_nested_columns_have_some_arrays)
|
||||
throw;
|
||||
}
|
||||
|
||||
/// `columns_are_reordered_outside` is true because column indices are
|
||||
/// going to be transformed and then written to the outer message,
|
||||
/// see add_field_serializer() below.
|
||||
|
||||
if (nested_message_serializer)
|
||||
{
|
||||
transformColumnIndices(used_column_indices_in_nested, nested_column_indices);
|
||||
add_field_serializer(
|
||||
column_name,
|
||||
std::move(used_column_indices_in_nested),
|
||||
*field_descriptor,
|
||||
std::move(nested_message_serializer));
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
if ((e.code() != ErrorCodes::PROTOBUF_FIELD_NOT_REPEATED) || !has_fallback_to_array_of_nested_messages)
|
||||
throw;
|
||||
}
|
||||
|
||||
if (has_fallback_to_array_of_nested_messages)
|
||||
{
|
||||
/// Try to serialize those columns as an array of nested messages.
|
||||
removeNonArrayElements(nested_data_types, nested_column_names, nested_column_indices);
|
||||
for (DataTypePtr & dt : nested_data_types)
|
||||
dt = assert_cast<const DataTypeArray &>(*dt).getNestedType();
|
||||
|
||||
std::vector<size_t> used_column_indices_in_nested;
|
||||
auto nested_message_serializer = buildMessageSerializerImpl(
|
||||
nested_column_names.size(),
|
||||
nested_column_names.data(),
|
||||
nested_data_types.data(),
|
||||
*field_descriptor->message_type(),
|
||||
/* with_length_delimiter = */ false,
|
||||
google_wrappers_special_treatment,
|
||||
field_descriptor,
|
||||
used_column_indices_in_nested,
|
||||
/* columns_are_reordered_outside = */ true,
|
||||
/* check_nested_while_filling_missing_columns = */ false);
|
||||
|
||||
/// `columns_are_reordered_outside` is true because column indices are
|
||||
/// going to be transformed and then written to the outer message,
|
||||
/// see add_field_serializer() below.
|
||||
|
||||
if (nested_message_serializer)
|
||||
{
|
||||
std::vector<std::string_view> column_names_used;
|
||||
column_names_used.reserve(used_column_indices_in_nested.size());
|
||||
for (size_t i : used_column_indices_in_nested)
|
||||
column_names_used.emplace_back(nested_column_names[i]);
|
||||
auto field_serializer = std::make_unique<ProtobufSerializerFlattenedNestedAsArrayOfNestedMessages>(
|
||||
std::move(column_names_used), field_descriptor, std::move(nested_message_serializer), get_root_desc_function);
|
||||
transformColumnIndices(used_column_indices_in_nested, nested_column_indices);
|
||||
add_field_serializer(column_name, std::move(used_column_indices_in_nested), *field_descriptor, std::move(field_serializer));
|
||||
break;
|
||||
}
|
||||
}
|
||||
/// if the protobuf field has the repeated label,
|
||||
/// only for the SUBSET of matching nested cols that are of type Array,
|
||||
/// try as ProtobufSerializerFlattenedNestedAsArrayOfNestedMessages
|
||||
if (repeated_field_matching_nested_columns_have_some_arrays)
|
||||
{
|
||||
removeNonArrayElements(nested_data_types, nested_column_names, nested_column_indices);
|
||||
if (attempt_unwrap_and_build_array_serializer())
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,9 +19,7 @@
|
||||
#include <Common/HashTable/Hash.h>
|
||||
|
||||
#if USE_SSL
|
||||
# include <openssl/evp.h>
|
||||
# include <openssl/md5.h>
|
||||
# include <openssl/ripemd.h>
|
||||
#endif
|
||||
|
||||
#include <bit>
|
||||
@ -198,34 +196,6 @@ T combineHashesFunc(T t1, T t2)
|
||||
return HashFunction::apply(reinterpret_cast<const char *>(hashes), sizeof(hashes));
|
||||
}
|
||||
|
||||
#if USE_SSL
|
||||
struct RipeMD160Impl
|
||||
{
|
||||
static constexpr auto name = "RIPEMD160";
|
||||
using ReturnType = UInt256;
|
||||
|
||||
static UInt256 apply(const char * begin, size_t size)
|
||||
{
|
||||
UInt8 digest[RIPEMD160_DIGEST_LENGTH];
|
||||
|
||||
RIPEMD160(reinterpret_cast<const unsigned char *>(begin), size, reinterpret_cast<unsigned char *>(digest));
|
||||
|
||||
std::reverse(digest, digest + RIPEMD160_DIGEST_LENGTH);
|
||||
|
||||
UInt256 res = 0;
|
||||
std::memcpy(&res, digest, RIPEMD160_DIGEST_LENGTH);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
static UInt256 combineHashes(UInt256 h1, UInt256 h2)
|
||||
{
|
||||
return combineHashesFunc<UInt256, RipeMD160Impl>(h1, h2);
|
||||
}
|
||||
|
||||
static constexpr bool use_int_hash_for_pods = false;
|
||||
};
|
||||
#endif
|
||||
|
||||
struct SipHash64Impl
|
||||
{
|
||||
@ -1654,7 +1624,6 @@ using FunctionIntHash32 = FunctionIntHash<IntHash32Impl, NameIntHash32>;
|
||||
using FunctionIntHash64 = FunctionIntHash<IntHash64Impl, NameIntHash64>;
|
||||
#if USE_SSL
|
||||
using FunctionHalfMD5 = FunctionAnyHash<HalfMD5Impl>;
|
||||
using FunctionRipeMD160Hash = FunctionAnyHash<RipeMD160Impl>;
|
||||
#endif
|
||||
using FunctionSipHash128 = FunctionAnyHash<SipHash128Impl>;
|
||||
using FunctionSipHash128Keyed = FunctionAnyHash<SipHash128KeyedImpl, true, SipHash128KeyedImpl::Key, SipHash128KeyedImpl::KeyColumns>;
|
||||
@ -1683,7 +1652,6 @@ using FunctionXxHash64 = FunctionAnyHash<ImplXxHash64>;
|
||||
using FunctionXXH3 = FunctionAnyHash<ImplXXH3>;
|
||||
|
||||
using FunctionWyHash64 = FunctionAnyHash<ImplWyHash64>;
|
||||
|
||||
}
|
||||
|
||||
#pragma clang diagnostic pop
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
#include "FunctionsHashing.h"
|
||||
|
||||
#include <Functions/FunctionFactory.h>
|
||||
|
||||
/// FunctionsHashing instantiations are separated into files FunctionsHashing*.cpp
|
||||
/// to better parallelize the build procedure and avoid MSan build failure
|
||||
/// due to excessive resource consumption.
|
||||
namespace DB
|
||||
{
|
||||
#if USE_SSL
|
||||
REGISTER_FUNCTION(HashingRipe)
|
||||
{
|
||||
factory.registerFunction<FunctionRipeMD160Hash>(FunctionDocumentation{
|
||||
.description = "RIPEMD-160 hash function, primarily used in Bitcoin address generation.",
|
||||
.examples{{"", "SELECT hex(RIPEMD160('The quick brown fox jumps over the lazy dog'));", R"(
|
||||
┌─hex(RIPEMD160('The quick brown fox jumps over the lazy dog'))─┐
|
||||
│ 37F332F68DB77BD9D7EDD4969571AD671CF9DD3B │
|
||||
└───────────────────────────────────────────────────────────────┘
|
||||
)"}},
|
||||
.categories{"Hash"}});
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@ -8,11 +8,6 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
}
|
||||
|
||||
struct ReplaceStringTraits
|
||||
{
|
||||
enum class Replace : uint8_t
|
||||
@ -39,7 +34,11 @@ struct ReplaceStringImpl
|
||||
size_t input_rows_count)
|
||||
{
|
||||
if (needle.empty())
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
|
||||
{
|
||||
res_data.assign(haystack_data.begin(), haystack_data.end());
|
||||
res_offsets.assign(haystack_offsets.begin(), haystack_offsets.end());
|
||||
return;
|
||||
}
|
||||
|
||||
const UInt8 * const begin = haystack_data.data();
|
||||
const UInt8 * const end = haystack_data.data() + haystack_data.size();
|
||||
@ -145,34 +144,34 @@ struct ReplaceStringImpl
|
||||
const auto * const cur_needle_data = &needle_data[prev_needle_offset];
|
||||
const size_t cur_needle_length = needle_offsets[i] - prev_needle_offset - 1;
|
||||
|
||||
if (cur_needle_length == 0)
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
|
||||
|
||||
/// Using "slow" "stdlib searcher instead of Volnitsky because there is a different pattern in each row
|
||||
StdLibASCIIStringSearcher</*CaseInsensitive*/ false> searcher(cur_needle_data, cur_needle_length);
|
||||
|
||||
const auto * last_match = static_cast<UInt8 *>(nullptr);
|
||||
const auto * start_pos = cur_haystack_data;
|
||||
const auto * const cur_haystack_end = cur_haystack_data + cur_haystack_length;
|
||||
|
||||
while (start_pos < cur_haystack_end)
|
||||
if (cur_needle_length)
|
||||
{
|
||||
if (const auto * const match = searcher.search(start_pos, cur_haystack_end); match != cur_haystack_end)
|
||||
/// Using "slow" "stdlib searcher instead of Volnitsky because there is a different pattern in each row
|
||||
StdLibASCIIStringSearcher</*CaseInsensitive*/ false> searcher(cur_needle_data, cur_needle_length);
|
||||
|
||||
while (start_pos < cur_haystack_end)
|
||||
{
|
||||
/// Copy prefix before match
|
||||
copyToOutput(start_pos, match - start_pos, res_data, res_offset);
|
||||
if (const auto * const match = searcher.search(start_pos, cur_haystack_end); match != cur_haystack_end)
|
||||
{
|
||||
/// Copy prefix before match
|
||||
copyToOutput(start_pos, match - start_pos, res_data, res_offset);
|
||||
|
||||
/// Insert replacement for match
|
||||
copyToOutput(replacement.data(), replacement.size(), res_data, res_offset);
|
||||
/// Insert replacement for match
|
||||
copyToOutput(replacement.data(), replacement.size(), res_data, res_offset);
|
||||
|
||||
last_match = match;
|
||||
start_pos = match + cur_needle_length;
|
||||
last_match = match;
|
||||
start_pos = match + cur_needle_length;
|
||||
|
||||
if constexpr (replace == ReplaceStringTraits::Replace::First)
|
||||
if constexpr (replace == ReplaceStringTraits::Replace::First)
|
||||
break;
|
||||
}
|
||||
else
|
||||
break;
|
||||
}
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
/// Copy suffix after last match
|
||||
@ -200,7 +199,11 @@ struct ReplaceStringImpl
|
||||
chassert(haystack_offsets.size() == replacement_offsets.size());
|
||||
|
||||
if (needle.empty())
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
|
||||
{
|
||||
res_data.assign(haystack_data.begin(), haystack_data.end());
|
||||
res_offsets.assign(haystack_offsets.begin(), haystack_offsets.end());
|
||||
return;
|
||||
}
|
||||
|
||||
res_data.reserve(haystack_data.size());
|
||||
res_offsets.resize(input_rows_count);
|
||||
@ -291,36 +294,35 @@ struct ReplaceStringImpl
|
||||
const auto * const cur_replacement_data = &replacement_data[prev_replacement_offset];
|
||||
const size_t cur_replacement_length = replacement_offsets[i] - prev_replacement_offset - 1;
|
||||
|
||||
if (cur_needle_length == 0)
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
|
||||
|
||||
/// Using "slow" "stdlib searcher instead of Volnitsky because there is a different pattern in each row
|
||||
StdLibASCIIStringSearcher</*CaseInsensitive*/ false> searcher(cur_needle_data, cur_needle_length);
|
||||
|
||||
const auto * last_match = static_cast<UInt8 *>(nullptr);
|
||||
const auto * start_pos = cur_haystack_data;
|
||||
const auto * const cur_haystack_end = cur_haystack_data + cur_haystack_length;
|
||||
|
||||
while (start_pos < cur_haystack_end)
|
||||
if (cur_needle_length)
|
||||
{
|
||||
if (const auto * const match = searcher.search(start_pos, cur_haystack_end); match != cur_haystack_end)
|
||||
/// Using "slow" "stdlib searcher instead of Volnitsky because there is a different pattern in each row
|
||||
StdLibASCIIStringSearcher</*CaseInsensitive*/ false> searcher(cur_needle_data, cur_needle_length);
|
||||
|
||||
while (start_pos < cur_haystack_end)
|
||||
{
|
||||
/// Copy prefix before match
|
||||
copyToOutput(start_pos, match - start_pos, res_data, res_offset);
|
||||
if (const auto * const match = searcher.search(start_pos, cur_haystack_end); match != cur_haystack_end)
|
||||
{
|
||||
/// Copy prefix before match
|
||||
copyToOutput(start_pos, match - start_pos, res_data, res_offset);
|
||||
|
||||
/// Insert replacement for match
|
||||
copyToOutput(cur_replacement_data, cur_replacement_length, res_data, res_offset);
|
||||
/// Insert replacement for match
|
||||
copyToOutput(cur_replacement_data, cur_replacement_length, res_data, res_offset);
|
||||
|
||||
last_match = match;
|
||||
start_pos = match + cur_needle_length;
|
||||
last_match = match;
|
||||
start_pos = match + cur_needle_length;
|
||||
|
||||
if constexpr (replace == ReplaceStringTraits::Replace::First)
|
||||
if constexpr (replace == ReplaceStringTraits::Replace::First)
|
||||
break;
|
||||
}
|
||||
else
|
||||
break;
|
||||
}
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
/// Copy suffix after last match
|
||||
size_t bytes = (last_match == nullptr) ? (cur_haystack_end - cur_haystack_data + 1)
|
||||
: (cur_haystack_end - last_match - cur_needle_length + 1);
|
||||
@ -346,7 +348,21 @@ struct ReplaceStringImpl
|
||||
size_t input_rows_count)
|
||||
{
|
||||
if (needle.empty())
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
|
||||
{
|
||||
chassert(input_rows_count == haystack_data.size() / n);
|
||||
/// Since ColumnFixedString does not have a zero byte at the end, while ColumnString does,
|
||||
/// we need to split haystack_data into strings of length n, add 1 zero byte to the end of each string
|
||||
/// and then copy to res_data, ref: ColumnString.h and ColumnFixedString.h
|
||||
res_data.reserve(haystack_data.size() + input_rows_count);
|
||||
res_offsets.resize(input_rows_count);
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
res_data.insert(res_data.end(), haystack_data.begin() + i * n, haystack_data.begin() + (i + 1) * n);
|
||||
res_data.push_back(0);
|
||||
res_offsets[i] = (i + 1) * n + 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const UInt8 * const begin = haystack_data.data();
|
||||
const UInt8 * const end = haystack_data.data() + haystack_data.size();
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <Columns/ColumnsDateTime.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Common/DateLUTImpl.h>
|
||||
@ -420,6 +421,8 @@ private:
|
||||
REGISTER_FUNCTION(ToStartOfInterval)
|
||||
{
|
||||
factory.registerFunction<FunctionToStartOfInterval>();
|
||||
factory.registerAlias("time_bucket", "toStartOfInterval", FunctionFactory::Case::Insensitive);
|
||||
factory.registerAlias("date_bin", "toStartOfInterval", FunctionFactory::Case::Insensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
22
src/IO/ReadSettings.cpp
Normal file
22
src/IO/ReadSettings.cpp
Normal file
@ -0,0 +1,22 @@
|
||||
#include <IO/ReadSettings.h>
|
||||
#include <Common/CurrentThread.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Core/Settings.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
ReadSettings getReadSettings()
|
||||
{
|
||||
auto query_context = CurrentThread::getQueryContext();
|
||||
if (query_context)
|
||||
return query_context->getReadSettings();
|
||||
|
||||
auto global_context = Context::getGlobalContextInstance();
|
||||
if (global_context)
|
||||
return global_context->getReadSettings();
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
}
|
||||
@ -62,9 +62,13 @@ enum class RemoteFSReadMethod : uint8_t
|
||||
|
||||
class MMappedFileCache;
|
||||
class PageCache;
|
||||
class Context;
|
||||
|
||||
struct ReadSettings
|
||||
{
|
||||
ReadSettings() = default;
|
||||
explicit ReadSettings(const Context & context);
|
||||
|
||||
/// Method to use reading from local filesystem.
|
||||
LocalFSReadMethod local_fs_method = LocalFSReadMethod::pread;
|
||||
/// Method to use reading from remote filesystem.
|
||||
@ -136,4 +140,6 @@ struct ReadSettings
|
||||
}
|
||||
};
|
||||
|
||||
ReadSettings getReadSettings();
|
||||
|
||||
}
|
||||
|
||||
@ -3190,32 +3190,4 @@ const ActionsDAG::Node * FindOriginalNodeForOutputName::find(const String & outp
|
||||
return it->second;
|
||||
}
|
||||
|
||||
FindAliasForInputName::FindAliasForInputName(const ActionsDAG & actions_)
|
||||
{
|
||||
const auto & actions_outputs = actions_.getOutputs();
|
||||
for (const auto * output_node : actions_outputs)
|
||||
{
|
||||
/// find input node which corresponds to alias
|
||||
const auto * node = output_node;
|
||||
while (node && node->type == ActionsDAG::ActionType::ALIAS)
|
||||
{
|
||||
/// alias has only one child
|
||||
chassert(node->children.size() == 1);
|
||||
node = node->children.front();
|
||||
}
|
||||
if (node && node->type == ActionsDAG::ActionType::INPUT)
|
||||
/// node can have several aliases but we consider only the first one
|
||||
index.emplace(node->result_name, output_node);
|
||||
}
|
||||
}
|
||||
|
||||
const ActionsDAG::Node * FindAliasForInputName::find(const String & name)
|
||||
{
|
||||
const auto it = index.find(name);
|
||||
if (it == index.end())
|
||||
return nullptr;
|
||||
|
||||
return it->second;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -491,18 +491,6 @@ private:
|
||||
NameToNodeIndex index;
|
||||
};
|
||||
|
||||
class FindAliasForInputName
|
||||
{
|
||||
using NameToNodeIndex = std::unordered_map<std::string_view, const ActionsDAG::Node *>;
|
||||
|
||||
public:
|
||||
explicit FindAliasForInputName(const ActionsDAG & actions);
|
||||
const ActionsDAG::Node * find(const String & name);
|
||||
|
||||
private:
|
||||
NameToNodeIndex index;
|
||||
};
|
||||
|
||||
/// This is an ugly way to bypass impossibility to forward declare ActionDAG::Node.
|
||||
struct ActionDAGNodes
|
||||
{
|
||||
|
||||
@ -252,13 +252,13 @@ namespace ErrorCodes
|
||||
extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int INVALID_SETTING_VALUE;
|
||||
extern const int UNKNOWN_READ_METHOD;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int UNKNOWN_FUNCTION;
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH;
|
||||
extern const int CLUSTER_DOESNT_EXIST;
|
||||
extern const int SET_NON_GRANTED_ROLE;
|
||||
extern const int UNKNOWN_READ_METHOD;
|
||||
}
|
||||
|
||||
#define SHUTDOWN(log, desc, ptr, method) do \
|
||||
|
||||
@ -12,12 +12,14 @@
|
||||
#include <Interpreters/ExpressionAnalyzer.h>
|
||||
#include <Interpreters/TreeRewriter.h>
|
||||
#include <Processors/QueryPlan/FilterStep.h>
|
||||
#include <Common/Logger.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace Setting
|
||||
{
|
||||
extern const SettingsBool allow_experimental_analyzer;
|
||||
extern const SettingsString additional_result_filter;
|
||||
extern const SettingsUInt64 max_bytes_to_read;
|
||||
extern const SettingsUInt64 max_bytes_to_read_leaf;
|
||||
@ -47,7 +49,13 @@ IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery(
|
||||
/// it's possible that new analyzer will be enabled in ::getQueryProcessingStage method
|
||||
/// of the underlying storage when all other parts of infrastructure are not ready for it
|
||||
/// (built with old analyzer).
|
||||
context->setSetting("allow_experimental_analyzer", false);
|
||||
if (context->getSettingsRef()[Setting::allow_experimental_analyzer])
|
||||
{
|
||||
LOG_TRACE(getLogger("IInterpreterUnionOrSelectQuery"),
|
||||
"The new analyzer is enabled, but the old interpreter is used. It can be a bug, please report it. Will disable 'allow_experimental_analyzer' setting (for query: {})",
|
||||
query_ptr->formatForLogging());
|
||||
context->setSetting("allow_experimental_analyzer", false);
|
||||
}
|
||||
|
||||
if (options.shard_num)
|
||||
context->addSpecialScalar(
|
||||
|
||||
@ -1588,7 +1588,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
|
||||
if (need_add_to_database && !database)
|
||||
throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(database_name));
|
||||
|
||||
if (create.replace_table)
|
||||
if (create.replace_table
|
||||
|| (create.replace_view && (database->getEngineName() == "Atomic" || database->getEngineName() == "Replicated")))
|
||||
{
|
||||
chassert(!ddl_guard);
|
||||
return doCreateOrReplaceTable(create, properties, mode);
|
||||
@ -1944,16 +1945,16 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create,
|
||||
|
||||
auto ast_rename = std::make_shared<ASTRenameQuery>(ASTRenameQuery::Elements{std::move(elem)});
|
||||
ast_rename->dictionary = create.is_dictionary;
|
||||
if (create.create_or_replace)
|
||||
if (create.create_or_replace || create.replace_view)
|
||||
{
|
||||
/// CREATE OR REPLACE TABLE
|
||||
/// CREATE OR REPLACE TABLE/VIEW
|
||||
/// Will execute ordinary RENAME instead of EXCHANGE if the target table does not exist
|
||||
ast_rename->rename_if_cannot_exchange = true;
|
||||
ast_rename->exchange = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
/// REPLACE TABLE
|
||||
/// REPLACE TABLE/VIEW
|
||||
/// Will execute EXCHANGE query and fail if the target table does not exist
|
||||
ast_rename->exchange = true;
|
||||
}
|
||||
|
||||
@ -165,8 +165,7 @@ void InterpreterSelectIntersectExceptQuery::buildQueryPlan(QueryPlan & query_pla
|
||||
limits,
|
||||
0,
|
||||
result_header.getNames(),
|
||||
false,
|
||||
settings[Setting::optimize_distinct_in_order]);
|
||||
false);
|
||||
|
||||
query_plan.addStep(std::move(distinct_step));
|
||||
}
|
||||
|
||||
@ -1801,7 +1801,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
|
||||
if (!joined_plan)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no joined plan for query");
|
||||
|
||||
auto add_sorting = [&settings, this] (QueryPlan & plan, const Names & key_names, JoinTableSide join_pos)
|
||||
auto add_sorting = [this] (QueryPlan & plan, const Names & key_names, JoinTableSide join_pos)
|
||||
{
|
||||
SortDescription order_descr;
|
||||
order_descr.reserve(key_names.size());
|
||||
@ -1813,9 +1813,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
|
||||
auto sorting_step = std::make_unique<SortingStep>(
|
||||
plan.getCurrentDataStream(),
|
||||
std::move(order_descr),
|
||||
0 /* LIMIT */,
|
||||
sort_settings,
|
||||
settings[Setting::optimize_sorting_by_input_stream_properties]);
|
||||
0 /* LIMIT */, sort_settings);
|
||||
sorting_step->setStepDescription(fmt::format("Sort {} before JOIN", join_pos));
|
||||
plan.addStep(std::move(sorting_step));
|
||||
};
|
||||
@ -2128,8 +2126,7 @@ static void executeMergeAggregatedImpl(
|
||||
const NamesAndTypesList & aggregation_keys,
|
||||
const NamesAndTypesLists & aggregation_keys_list,
|
||||
const AggregateDescriptions & aggregates,
|
||||
bool should_produce_results_in_order_of_bucket_number,
|
||||
SortDescription group_by_sort_description)
|
||||
bool should_produce_results_in_order_of_bucket_number)
|
||||
{
|
||||
auto keys = aggregation_keys.getNames();
|
||||
|
||||
@ -2169,7 +2166,6 @@ static void executeMergeAggregatedImpl(
|
||||
should_produce_results_in_order_of_bucket_number,
|
||||
settings[Setting::max_block_size],
|
||||
settings[Setting::aggregation_in_order_max_block_bytes],
|
||||
std::move(group_by_sort_description),
|
||||
settings[Setting::enable_memory_bound_merging_of_aggregation_results]);
|
||||
|
||||
query_plan.addStep(std::move(merging_aggregated));
|
||||
@ -2857,10 +2853,6 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool
|
||||
{
|
||||
const Settings & settings = context->getSettingsRef();
|
||||
|
||||
/// Used to determine if we should use memory bound merging strategy.
|
||||
auto group_by_sort_description
|
||||
= !query_analyzer->useGroupingSetKey() ? getSortDescriptionFromGroupBy(getSelectQuery()) : SortDescription{};
|
||||
|
||||
const bool should_produce_results_in_order_of_bucket_number = options.to_stage == QueryProcessingStage::WithMergeableState
|
||||
&& (settings[Setting::distributed_aggregation_memory_efficient] || settings[Setting::enable_memory_bound_merging_of_aggregation_results]);
|
||||
const bool parallel_replicas_from_merge_tree = storage->isMergeTree() && context->canUseParallelReplicasOnInitiator();
|
||||
@ -2875,8 +2867,7 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool
|
||||
query_analyzer->aggregationKeys(),
|
||||
query_analyzer->aggregationKeysList(),
|
||||
query_analyzer->aggregates(),
|
||||
should_produce_results_in_order_of_bucket_number,
|
||||
std::move(group_by_sort_description));
|
||||
should_produce_results_in_order_of_bucket_number);
|
||||
}
|
||||
|
||||
|
||||
@ -3046,8 +3037,7 @@ void InterpreterSelectQuery::executeWindow(QueryPlan & query_plan)
|
||||
window.full_sort_description,
|
||||
window.partition_by,
|
||||
0 /* LIMIT */,
|
||||
sort_settings,
|
||||
settings[Setting::optimize_sorting_by_input_stream_properties]);
|
||||
sort_settings);
|
||||
sorting_step->setStepDescription("Sorting for window '" + window.window_name + "'");
|
||||
query_plan.addStep(std::move(sorting_step));
|
||||
}
|
||||
@ -3097,8 +3087,6 @@ void InterpreterSelectQuery::executeOrder(QueryPlan & query_plan, InputOrderInfo
|
||||
return;
|
||||
}
|
||||
|
||||
const Settings & settings = context->getSettingsRef();
|
||||
|
||||
SortingStep::Settings sort_settings(*context);
|
||||
|
||||
/// Merge the sorted blocks.
|
||||
@ -3106,8 +3094,7 @@ void InterpreterSelectQuery::executeOrder(QueryPlan & query_plan, InputOrderInfo
|
||||
query_plan.getCurrentDataStream(),
|
||||
output_order_descr,
|
||||
limit,
|
||||
sort_settings,
|
||||
settings[Setting::optimize_sorting_by_input_stream_properties]);
|
||||
sort_settings);
|
||||
|
||||
sorting_step->setStepDescription("Sorting for ORDER BY");
|
||||
query_plan.addStep(std::move(sorting_step));
|
||||
@ -3162,8 +3149,7 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before
|
||||
limits,
|
||||
limit_for_distinct,
|
||||
columns,
|
||||
pre_distinct,
|
||||
settings[Setting::optimize_distinct_in_order]);
|
||||
pre_distinct);
|
||||
|
||||
if (pre_distinct)
|
||||
distinct_step->setStepDescription("Preliminary DISTINCT");
|
||||
|
||||
@ -351,7 +351,11 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan)
|
||||
SizeLimits limits(settings[Setting::max_rows_in_distinct], settings[Setting::max_bytes_in_distinct], settings[Setting::distinct_overflow_mode]);
|
||||
|
||||
auto distinct_step = std::make_unique<DistinctStep>(
|
||||
query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false, settings[Setting::optimize_distinct_in_order]);
|
||||
query_plan.getCurrentDataStream(),
|
||||
limits,
|
||||
0,
|
||||
result_header.getNames(),
|
||||
false);
|
||||
|
||||
query_plan.addStep(std::move(distinct_step));
|
||||
}
|
||||
|
||||
@ -4,6 +4,9 @@
|
||||
#include <Columns/ColumnSet.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Analyzer/Resolve/QueryAnalyzer.h>
|
||||
#include <Analyzer/QueryTreeBuilder.h>
|
||||
#include <Analyzer/TableNode.h>
|
||||
#include <Core/Block.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
@ -15,18 +18,30 @@
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include <Interpreters/convertFieldToType.h>
|
||||
#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
|
||||
#include <Interpreters/ExpressionAnalyzer.h>
|
||||
#include <Interpreters/FunctionNameNormalizer.h>
|
||||
#include <Interpreters/ReplaceQueryParameterVisitor.h>
|
||||
#include <Interpreters/SelectQueryOptions.h>
|
||||
#include <Interpreters/TreeRewriter.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTSubquery.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Planner/CollectSets.h>
|
||||
#include <Planner/CollectTableExpressionData.h>
|
||||
#include <Planner/PlannerActionsVisitor.h>
|
||||
#include <Planner/PlannerContext.h>
|
||||
#include <Planner/Utils.h>
|
||||
#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
|
||||
#include <Processors/Executors/PullingPipelineExecutor.h>
|
||||
#include <Storages/MergeTree/KeyCondition.h>
|
||||
#include <Storages/ColumnsDescription.h>
|
||||
#include <Storages/StorageDummy.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
@ -35,6 +50,7 @@ namespace DB
|
||||
namespace Setting
|
||||
{
|
||||
extern const SettingsBool normalize_function_names;
|
||||
extern const SettingsBool allow_experimental_analyzer;
|
||||
}
|
||||
|
||||
namespace ErrorCodes
|
||||
@ -76,34 +92,74 @@ std::optional<EvaluateConstantExpressionResult> evaluateConstantExpressionImpl(c
|
||||
ReplaceQueryParameterVisitor param_visitor(context->getQueryParameters());
|
||||
param_visitor.visit(ast);
|
||||
|
||||
/// Notice: function name normalization is disabled when it's a secondary query, because queries are either
|
||||
/// already normalized on initiator node, or not normalized and should remain unnormalized for
|
||||
/// compatibility.
|
||||
if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY
|
||||
&& context->getSettingsRef()[Setting::normalize_function_names])
|
||||
FunctionNameNormalizer::visit(ast.get());
|
||||
|
||||
auto syntax_result = TreeRewriter(context, no_throw).analyze(ast, source_columns);
|
||||
if (!syntax_result)
|
||||
return {};
|
||||
|
||||
/// AST potentially could be transformed to literal during TreeRewriter analyze.
|
||||
/// For example if we have SQL user defined function that return literal AS subquery.
|
||||
if (ASTLiteral * literal = ast->as<ASTLiteral>())
|
||||
return getFieldAndDataTypeFromLiteral(literal);
|
||||
|
||||
auto actions = ExpressionAnalyzer(ast, syntax_result, context).getConstActionsDAG();
|
||||
String result_name;
|
||||
|
||||
ColumnPtr result_column;
|
||||
DataTypePtr result_type;
|
||||
String result_name = ast->getColumnName();
|
||||
for (const auto & action_node : actions.getOutputs())
|
||||
if (context->getSettingsRef()[Setting::allow_experimental_analyzer])
|
||||
{
|
||||
if ((action_node->result_name == result_name) && action_node->column)
|
||||
result_name = ast->getColumnName();
|
||||
|
||||
auto execution_context = Context::createCopy(context);
|
||||
auto expression = buildQueryTree(ast, execution_context);
|
||||
|
||||
ColumnsDescription fake_column_descriptions(source_columns);
|
||||
auto storage = std::make_shared<StorageDummy>(StorageID{"dummy", "dummy"}, fake_column_descriptions);
|
||||
QueryTreeNodePtr fake_table_expression = std::make_shared<TableNode>(storage, execution_context);
|
||||
|
||||
QueryAnalyzer analyzer(false);
|
||||
analyzer.resolveConstantExpression(expression, fake_table_expression, execution_context);
|
||||
|
||||
GlobalPlannerContextPtr global_planner_context = std::make_shared<GlobalPlannerContext>(nullptr, nullptr, FiltersForTableExpressionMap{});
|
||||
auto planner_context = std::make_shared<PlannerContext>(execution_context, global_planner_context, SelectQueryOptions{});
|
||||
|
||||
collectSourceColumns(expression, planner_context, false /*keep_alias_columns*/);
|
||||
collectSets(expression, *planner_context);
|
||||
|
||||
auto actions = buildActionsDAGFromExpressionNode(expression, {}, planner_context);
|
||||
|
||||
if (actions.getOutputs().size() != 1)
|
||||
{
|
||||
result_column = action_node->column;
|
||||
result_type = action_node->result_type;
|
||||
break;
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "ActionsDAG contains more than 1 output for expression: {}", ast->formatForLogging());
|
||||
}
|
||||
|
||||
const auto & output = actions.getOutputs()[0];
|
||||
if (output->column)
|
||||
{
|
||||
result_column = output->column;
|
||||
result_type = output->result_type;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Notice: function name normalization is disabled when it's a secondary query, because queries are either
|
||||
/// already normalized on initiator node, or not normalized and should remain unnormalized for
|
||||
/// compatibility.
|
||||
if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY
|
||||
&& context->getSettingsRef()[Setting::normalize_function_names])
|
||||
FunctionNameNormalizer::visit(ast.get());
|
||||
|
||||
result_name = ast->getColumnName();
|
||||
|
||||
auto syntax_result = TreeRewriter(context, no_throw).analyze(ast, source_columns);
|
||||
if (!syntax_result)
|
||||
return {};
|
||||
|
||||
/// AST potentially could be transformed to literal during TreeRewriter analyze.
|
||||
/// For example if we have SQL user defined function that return literal AS subquery.
|
||||
if (ASTLiteral * literal = ast->as<ASTLiteral>())
|
||||
return getFieldAndDataTypeFromLiteral(literal);
|
||||
|
||||
auto actions = ExpressionAnalyzer(ast, syntax_result, context).getConstActionsDAG();
|
||||
|
||||
for (const auto & action_node : actions.getOutputs())
|
||||
{
|
||||
if ((action_node->result_name == result_name) && action_node->column)
|
||||
{
|
||||
result_column = action_node->column;
|
||||
result_type = action_node->result_type;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -13,46 +13,6 @@ namespace ErrorCodes
|
||||
|
||||
namespace
|
||||
{
|
||||
void formatColumnNames(const Strings & columns, const IAST::FormatSettings & settings)
|
||||
{
|
||||
settings.ostr << "(";
|
||||
bool need_comma = false;
|
||||
for (const auto & column : columns)
|
||||
{
|
||||
if (std::exchange(need_comma, true))
|
||||
settings.ostr << ", ";
|
||||
settings.ostr << backQuoteIfNeed(column);
|
||||
}
|
||||
settings.ostr << ")";
|
||||
}
|
||||
|
||||
|
||||
void formatONClause(const AccessRightsElement & element, const IAST::FormatSettings & settings)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ON " << (settings.hilite ? IAST::hilite_none : "");
|
||||
if (element.isGlobalWithParameter())
|
||||
{
|
||||
if (element.any_parameter)
|
||||
settings.ostr << "*";
|
||||
else
|
||||
settings.ostr << backQuoteIfNeed(element.parameter);
|
||||
}
|
||||
else if (element.any_database)
|
||||
{
|
||||
settings.ostr << "*.*";
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!element.database.empty())
|
||||
settings.ostr << backQuoteIfNeed(element.database) << ".";
|
||||
if (element.any_table)
|
||||
settings.ostr << "*";
|
||||
else
|
||||
settings.ostr << backQuoteIfNeed(element.table);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void formatElementsWithoutOptions(const AccessRightsElements & elements, const IAST::FormatSettings & settings)
|
||||
{
|
||||
bool no_output = true;
|
||||
@ -60,7 +20,7 @@ namespace
|
||||
{
|
||||
const auto & element = elements[i];
|
||||
auto keywords = element.access_flags.toKeywords();
|
||||
if (keywords.empty() || (!element.any_column && element.columns.empty()))
|
||||
if (keywords.empty() || (!element.anyColumn() && element.columns.empty()))
|
||||
continue;
|
||||
|
||||
for (const auto & keyword : keywords)
|
||||
@ -69,8 +29,8 @@ namespace
|
||||
settings.ostr << ", ";
|
||||
|
||||
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << keyword << (settings.hilite ? IAST::hilite_none : "");
|
||||
if (!element.any_column)
|
||||
formatColumnNames(element.columns, settings);
|
||||
if (!element.anyColumn())
|
||||
element.formatColumnNames(settings.ostr);
|
||||
}
|
||||
|
||||
bool next_element_on_same_db_and_table = false;
|
||||
@ -86,7 +46,7 @@ namespace
|
||||
if (!next_element_on_same_db_and_table)
|
||||
{
|
||||
settings.ostr << " ";
|
||||
formatONClause(element, settings);
|
||||
element.formatONClause(settings.ostr, settings.hilite);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -123,7 +123,6 @@ namespace
|
||||
return false;
|
||||
|
||||
String database_name, table_name, parameter;
|
||||
bool any_database = false, any_table = false, any_parameter = false;
|
||||
|
||||
size_t is_global_with_parameter = 0;
|
||||
for (const auto & elem : access_and_columns)
|
||||
@ -135,40 +134,35 @@ namespace
|
||||
if (!ParserKeyword{Keyword::ON}.ignore(pos, expected))
|
||||
return false;
|
||||
|
||||
bool wildcard = false;
|
||||
bool default_database = false;
|
||||
if (is_global_with_parameter && is_global_with_parameter == access_and_columns.size())
|
||||
{
|
||||
ASTPtr parameter_ast;
|
||||
if (ParserToken{TokenType::Asterisk}.ignore(pos, expected))
|
||||
if (!ParserToken{TokenType::Asterisk}.ignore(pos, expected))
|
||||
{
|
||||
any_parameter = true;
|
||||
if (ParserIdentifier{}.parse(pos, parameter_ast, expected))
|
||||
parameter = getIdentifierName(parameter_ast);
|
||||
else
|
||||
return false;
|
||||
}
|
||||
else if (ParserIdentifier{}.parse(pos, parameter_ast, expected))
|
||||
{
|
||||
any_parameter = false;
|
||||
parameter = getIdentifierName(parameter_ast);
|
||||
}
|
||||
else
|
||||
return false;
|
||||
|
||||
any_database = any_table = true;
|
||||
if (ParserToken{TokenType::Asterisk}.ignore(pos, expected))
|
||||
wildcard = true;
|
||||
}
|
||||
else if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, database_name, any_database, table_name, any_table))
|
||||
{
|
||||
else if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, database_name, table_name, wildcard, default_database))
|
||||
return false;
|
||||
}
|
||||
|
||||
for (auto & [access_flags, columns] : access_and_columns)
|
||||
{
|
||||
AccessRightsElement element;
|
||||
element.access_flags = access_flags;
|
||||
element.any_column = columns.empty();
|
||||
element.columns = std::move(columns);
|
||||
element.any_database = any_database;
|
||||
element.database = database_name;
|
||||
element.any_table = any_table;
|
||||
element.any_parameter = any_parameter;
|
||||
element.table = table_name;
|
||||
element.parameter = parameter;
|
||||
element.wildcard = wildcard;
|
||||
element.default_database = default_database;
|
||||
res_elements.emplace_back(std::move(element));
|
||||
}
|
||||
|
||||
@ -202,15 +196,15 @@ namespace
|
||||
|
||||
String database_name;
|
||||
String table_name;
|
||||
bool any_database = false;
|
||||
bool any_table = false;
|
||||
if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, database_name, any_database, table_name, any_table))
|
||||
bool wildcard = false;
|
||||
bool default_database = false;
|
||||
if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, database_name, table_name, wildcard, default_database))
|
||||
return false;
|
||||
|
||||
default_element.any_database = any_database;
|
||||
default_element.database = database_name;
|
||||
default_element.any_table = any_table;
|
||||
default_element.table = table_name;
|
||||
default_element.wildcard = wildcard;
|
||||
default_element.default_database = default_database;
|
||||
elements.push_back(std::move(default_element));
|
||||
}
|
||||
|
||||
@ -228,13 +222,13 @@ namespace
|
||||
if (!element.empty())
|
||||
return false;
|
||||
|
||||
if (!element.any_column)
|
||||
if (!element.anyColumn())
|
||||
throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the column level", old_flags.toString());
|
||||
else if (!element.any_table)
|
||||
else if (!element.anyTable())
|
||||
throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the table level", old_flags.toString());
|
||||
else if (!element.any_database)
|
||||
else if (!element.anyDatabase())
|
||||
throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the database level", old_flags.toString());
|
||||
else if (!element.any_parameter)
|
||||
else if (!element.anyParameter())
|
||||
throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the global with parameter level", old_flags.toString());
|
||||
else
|
||||
throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted", old_flags.toString());
|
||||
|
||||
@ -26,15 +26,14 @@ namespace
|
||||
return IParserBase::wrapParseImpl(pos, [&]
|
||||
{
|
||||
String res_database, res_table_name;
|
||||
bool is_any_database = false;
|
||||
bool is_any_table = false;
|
||||
|
||||
if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, res_database, is_any_database, res_table_name, is_any_table)
|
||||
|| is_any_database)
|
||||
bool wildcard = false;
|
||||
bool default_database = false;
|
||||
if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, res_database, res_table_name, wildcard, default_database) || (res_database.empty() && res_table_name.empty() && !default_database))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
else if (is_any_table)
|
||||
else if (res_table_name.empty())
|
||||
{
|
||||
res_table_name = RowPolicyName::ANY_TABLE_MARK;
|
||||
}
|
||||
|
||||
@ -25,12 +25,12 @@ namespace
|
||||
return false;
|
||||
}
|
||||
|
||||
bool parseOnDBAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, bool & any_database, String & table, bool & any_table)
|
||||
bool parseOnDBAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, String & table, bool & wildcard, bool & default_database)
|
||||
{
|
||||
return IParserBase::wrapParseImpl(pos, [&]
|
||||
{
|
||||
return ParserKeyword{Keyword::ON}.ignore(pos, expected)
|
||||
&& parseDatabaseAndTableNameOrAsterisks(pos, expected, database, any_database, table, any_table);
|
||||
&& parseDatabaseAndTableNameOrAsterisks(pos, expected, database, table, wildcard, default_database);
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -74,10 +74,11 @@ bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected
|
||||
if (type == AccessEntityType::ROW_POLICY)
|
||||
{
|
||||
String database, table_name;
|
||||
bool any_database, any_table;
|
||||
if (parseOnDBAndTableName(pos, expected, database, any_database, table_name, any_table))
|
||||
bool wildcard = false;
|
||||
bool default_database = false;
|
||||
if (parseOnDBAndTableName(pos, expected, database, table_name, wildcard, default_database))
|
||||
{
|
||||
if (any_database)
|
||||
if (database.empty() && !default_database)
|
||||
all = true;
|
||||
else
|
||||
database_and_table_name.emplace(database, table_name);
|
||||
|
||||
@ -49,12 +49,12 @@ namespace
|
||||
return false;
|
||||
}
|
||||
|
||||
bool parseOnDBAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, bool & any_database, String & table, bool & any_table)
|
||||
bool parseOnDBAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, String & table, bool & wildcard, bool & default_database)
|
||||
{
|
||||
return IParserBase::wrapParseImpl(pos, [&]
|
||||
{
|
||||
return ParserKeyword{Keyword::ON}.ignore(pos, expected)
|
||||
&& parseDatabaseAndTableNameOrAsterisks(pos, expected, database, any_database, table, any_table);
|
||||
&& parseDatabaseAndTableNameOrAsterisks(pos, expected, database, table, wildcard, default_database);
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -108,12 +108,13 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe
|
||||
{
|
||||
ASTPtr ast;
|
||||
String database, table_name;
|
||||
bool any_database, any_table;
|
||||
bool wildcard = false;
|
||||
bool default_database = false;
|
||||
if (ParserRowPolicyNames{}.parse(pos, ast, expected))
|
||||
row_policy_names = typeid_cast<std::shared_ptr<ASTRowPolicyNames>>(ast);
|
||||
else if (parseOnDBAndTableName(pos, expected, database, any_database, table_name, any_table))
|
||||
else if (parseOnDBAndTableName(pos, expected, database, table_name, wildcard, default_database))
|
||||
{
|
||||
if (any_database)
|
||||
if (database.empty() && !default_database)
|
||||
all = true;
|
||||
else
|
||||
database_and_table_name.emplace(database, table_name);
|
||||
|
||||
@ -67,7 +67,7 @@ bool parseDatabaseAsAST(IParser::Pos & pos, Expected & expected, ASTPtr & databa
|
||||
}
|
||||
|
||||
|
||||
bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expected, String & database, bool & any_database, String & table, bool & any_table)
|
||||
bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expected, String & database, String & table, bool & wildcard, bool & default_database)
|
||||
{
|
||||
return IParserBase::wrapParseImpl(pos, [&]
|
||||
{
|
||||
@ -78,19 +78,16 @@ bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expecte
|
||||
&& ParserToken{TokenType::Asterisk}.ignore(pos, expected))
|
||||
{
|
||||
/// *.*
|
||||
any_database = true;
|
||||
database.clear();
|
||||
any_table = true;
|
||||
table.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
/// *
|
||||
pos = pos_before_dot;
|
||||
any_database = false;
|
||||
database.clear();
|
||||
any_table = true;
|
||||
table.clear();
|
||||
default_database = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -99,6 +96,9 @@ bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expecte
|
||||
if (identifier_parser.parse(pos, ast, expected))
|
||||
{
|
||||
String first_identifier = getIdentifierName(ast);
|
||||
if (ParserToken{TokenType::Asterisk}.ignore(pos, expected))
|
||||
wildcard = true;
|
||||
|
||||
auto pos_before_dot = pos;
|
||||
|
||||
if (ParserToken{TokenType::Dot}.ignore(pos, expected))
|
||||
@ -106,29 +106,31 @@ bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expecte
|
||||
if (ParserToken{TokenType::Asterisk}.ignore(pos, expected))
|
||||
{
|
||||
/// db.*
|
||||
any_database = false;
|
||||
database = std::move(first_identifier);
|
||||
any_table = true;
|
||||
table.clear();
|
||||
return true;
|
||||
}
|
||||
else if (identifier_parser.parse(pos, ast, expected))
|
||||
{
|
||||
/// db.table
|
||||
any_database = false;
|
||||
database = std::move(first_identifier);
|
||||
any_table = false;
|
||||
table = getIdentifierName(ast);
|
||||
if (ParserToken{TokenType::Asterisk}.ignore(pos, expected))
|
||||
wildcard = true;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// table
|
||||
pos = pos_before_dot;
|
||||
any_database = false;
|
||||
database.clear();
|
||||
any_table = false;
|
||||
table = std::move(first_identifier);
|
||||
default_database = true;
|
||||
|
||||
if (!wildcard && ParserToken{TokenType::Asterisk}.ignore(pos, expected))
|
||||
wildcard = true;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@ bool parseDatabaseAndTableName(IParser::Pos & pos, Expected & expected, String &
|
||||
bool parseDatabaseAndTableAsAST(IParser::Pos & pos, Expected & expected, ASTPtr & database, ASTPtr & table);
|
||||
|
||||
/// Parses [db.]name or [db.]* or [*.]*
|
||||
bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expected, String & database, bool & any_database, String & table, bool & any_table);
|
||||
bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expected, String & database, String & table, bool & wildcard, bool & default_database);
|
||||
|
||||
bool parseDatabase(IParser::Pos & pos, Expected & expected, String & database_str);
|
||||
|
||||
|
||||
@ -569,8 +569,6 @@ void addMergingAggregatedStep(QueryPlan & query_plan,
|
||||
parallel_replicas_from_merge_tree = it->second.isMergeTree() && query_context->canUseParallelReplicasOnInitiator();
|
||||
}
|
||||
|
||||
SortDescription group_by_sort_description;
|
||||
|
||||
auto merging_aggregated = std::make_unique<MergingAggregatedStep>(
|
||||
query_plan.getCurrentDataStream(),
|
||||
params,
|
||||
@ -584,7 +582,6 @@ void addMergingAggregatedStep(QueryPlan & query_plan,
|
||||
query_analysis_result.aggregation_should_produce_results_in_order_of_bucket_number,
|
||||
settings[Setting::max_block_size],
|
||||
settings[Setting::aggregation_in_order_max_block_bytes],
|
||||
std::move(group_by_sort_description),
|
||||
settings[Setting::enable_memory_bound_merging_of_aggregation_results]);
|
||||
query_plan.addStep(std::move(merging_aggregated));
|
||||
}
|
||||
@ -694,8 +691,7 @@ void addDistinctStep(QueryPlan & query_plan,
|
||||
limits,
|
||||
limit_hint_for_distinct,
|
||||
column_names,
|
||||
pre_distinct,
|
||||
settings[Setting::optimize_distinct_in_order]);
|
||||
pre_distinct);
|
||||
|
||||
distinct_step->setStepDescription(pre_distinct ? "Preliminary DISTINCT" : "DISTINCT");
|
||||
query_plan.addStep(std::move(distinct_step));
|
||||
@ -707,15 +703,13 @@ void addSortingStep(QueryPlan & query_plan,
|
||||
{
|
||||
const auto & sort_description = query_analysis_result.sort_description;
|
||||
const auto & query_context = planner_context->getQueryContext();
|
||||
const Settings & settings = query_context->getSettingsRef();
|
||||
SortingStep::Settings sort_settings(*query_context);
|
||||
|
||||
auto sorting_step = std::make_unique<SortingStep>(
|
||||
query_plan.getCurrentDataStream(),
|
||||
sort_description,
|
||||
query_analysis_result.partial_sorting_limit,
|
||||
sort_settings,
|
||||
settings[Setting::optimize_sorting_by_input_stream_properties]);
|
||||
sort_settings);
|
||||
sorting_step->setStepDescription("Sorting for ORDER BY");
|
||||
query_plan.addStep(std::move(sorting_step));
|
||||
}
|
||||
@ -1033,8 +1027,7 @@ void addWindowSteps(QueryPlan & query_plan,
|
||||
window_description.full_sort_description,
|
||||
window_description.partition_by,
|
||||
0 /*limit*/,
|
||||
sort_settings,
|
||||
settings[Setting::optimize_sorting_by_input_stream_properties]);
|
||||
sort_settings);
|
||||
sorting_step->setStepDescription("Sorting for window '" + window_description.window_name + "'");
|
||||
query_plan.addStep(std::move(sorting_step));
|
||||
}
|
||||
@ -1404,8 +1397,7 @@ void Planner::buildPlanForUnionNode()
|
||||
limits,
|
||||
0 /*limit hint*/,
|
||||
query_plan.getCurrentDataStream().header.getNames(),
|
||||
false /*pre distinct*/,
|
||||
settings[Setting::optimize_distinct_in_order]);
|
||||
false /*pre distinct*/);
|
||||
query_plan.addStep(std::move(distinct_step));
|
||||
}
|
||||
}
|
||||
|
||||
@ -1574,8 +1574,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
|
||||
plan.getCurrentDataStream(),
|
||||
std::move(sort_description),
|
||||
0 /*limit*/,
|
||||
sort_settings,
|
||||
settings[Setting::optimize_sorting_by_input_stream_properties]);
|
||||
sort_settings);
|
||||
sorting_step->setStepDescription(fmt::format("Sort {} before JOIN", join_table_side));
|
||||
plan.addStep(std::move(sorting_step));
|
||||
};
|
||||
|
||||
@ -130,27 +130,23 @@ AggregatingStep::AggregatingStep(
|
||||
, memory_bound_merging_of_aggregation_results_enabled(memory_bound_merging_of_aggregation_results_enabled_)
|
||||
, explicit_sorting_required_for_aggregation_in_order(explicit_sorting_required_for_aggregation_in_order_)
|
||||
{
|
||||
if (memoryBoundMergingWillBeUsed())
|
||||
{
|
||||
output_stream->sort_description = group_by_sort_description;
|
||||
output_stream->sort_scope = DataStream::SortScope::Global;
|
||||
}
|
||||
}
|
||||
|
||||
void AggregatingStep::applyOrder(SortDescription sort_description_for_merging_, SortDescription group_by_sort_description_)
|
||||
{
|
||||
sort_description_for_merging = std::move(sort_description_for_merging_);
|
||||
group_by_sort_description = std::move(group_by_sort_description_);
|
||||
|
||||
if (memoryBoundMergingWillBeUsed())
|
||||
{
|
||||
output_stream->sort_description = group_by_sort_description;
|
||||
output_stream->sort_scope = DataStream::SortScope::Global;
|
||||
}
|
||||
|
||||
explicit_sorting_required_for_aggregation_in_order = false;
|
||||
}
|
||||
|
||||
const SortDescription & AggregatingStep::getSortDescription() const
|
||||
{
|
||||
if (memoryBoundMergingWillBeUsed())
|
||||
return group_by_sort_description;
|
||||
|
||||
return IQueryPlanStep::getSortDescription();
|
||||
}
|
||||
|
||||
ActionsDAG AggregatingStep::makeCreatingMissingKeysForGroupingSetDAG(
|
||||
const Block & in_header,
|
||||
const Block & out_header,
|
||||
|
||||
@ -56,6 +56,8 @@ public:
|
||||
bool memoryBoundMergingWillBeUsed() const;
|
||||
void skipMerging() { skip_merging = true; }
|
||||
|
||||
const SortDescription & getSortDescription() const override;
|
||||
|
||||
bool canUseProjection() const;
|
||||
/// When we apply aggregate projection (which is full), this step will only merge data.
|
||||
/// Argument input_stream replaces current single input.
|
||||
@ -96,7 +98,7 @@ private:
|
||||
|
||||
/// These settings are used to determine if we should resize pipeline to 1 at the end.
|
||||
const bool should_produce_results_in_order_of_bucket_number;
|
||||
bool memory_bound_merging_of_aggregation_results_enabled;
|
||||
const bool memory_bound_merging_of_aggregation_results_enabled;
|
||||
bool explicit_sorting_required_for_aggregation_in_order;
|
||||
|
||||
Processors aggregating_in_order;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#include <Processors/QueryPlan/DistinctStep.h>
|
||||
#include <Processors/Transforms/DistinctSortedChunkTransform.h>
|
||||
#include <Processors/Transforms/DistinctSortedStreamTransform.h>
|
||||
#include <Processors/Transforms/DistinctSortedTransform.h>
|
||||
#include <Processors/Transforms/DistinctTransform.h>
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
@ -31,25 +31,12 @@ static ITransformingStep::Traits getTraits(bool pre_distinct)
|
||||
};
|
||||
}
|
||||
|
||||
static SortDescription getSortDescription(const SortDescription & input_sort_desc, const Names& columns)
|
||||
{
|
||||
SortDescription distinct_sort_desc;
|
||||
for (const auto & sort_column_desc : input_sort_desc)
|
||||
{
|
||||
if (std::find(begin(columns), end(columns), sort_column_desc.column_name) == columns.end())
|
||||
break;
|
||||
distinct_sort_desc.emplace_back(sort_column_desc);
|
||||
}
|
||||
return distinct_sort_desc;
|
||||
}
|
||||
|
||||
DistinctStep::DistinctStep(
|
||||
const DataStream & input_stream_,
|
||||
const SizeLimits & set_size_limits_,
|
||||
UInt64 limit_hint_,
|
||||
const Names & columns_,
|
||||
bool pre_distinct_,
|
||||
bool optimize_distinct_in_order_)
|
||||
bool pre_distinct_)
|
||||
: ITransformingStep(
|
||||
input_stream_,
|
||||
input_stream_.header,
|
||||
@ -58,7 +45,6 @@ DistinctStep::DistinctStep(
|
||||
, limit_hint(limit_hint_)
|
||||
, columns(columns_)
|
||||
, pre_distinct(pre_distinct_)
|
||||
, optimize_distinct_in_order(optimize_distinct_in_order_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -67,10 +53,7 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil
|
||||
if (!pre_distinct)
|
||||
pipeline.resize(1);
|
||||
|
||||
if (optimize_distinct_in_order)
|
||||
{
|
||||
const auto & input_stream = input_streams.back();
|
||||
const SortDescription distinct_sort_desc = getSortDescription(input_stream.sort_description, columns);
|
||||
if (!distinct_sort_desc.empty())
|
||||
{
|
||||
/// pre-distinct for sorted chunks
|
||||
@ -82,18 +65,17 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil
|
||||
if (stream_type != QueryPipelineBuilder::StreamType::Main)
|
||||
return nullptr;
|
||||
|
||||
return std::make_shared<DistinctSortedChunkTransform>(
|
||||
return std::make_shared<DistinctSortedStreamTransform>(
|
||||
header,
|
||||
set_size_limits,
|
||||
limit_hint,
|
||||
distinct_sort_desc,
|
||||
columns,
|
||||
input_stream.sort_scope == DataStream::SortScope::Stream);
|
||||
columns);
|
||||
});
|
||||
return;
|
||||
}
|
||||
/// final distinct for sorted stream (sorting inside and among chunks)
|
||||
if (input_stream.sort_scope == DataStream::SortScope::Global)
|
||||
else
|
||||
{
|
||||
if (pipeline.getNumStreams() != 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "DistinctStep with in-order expects single input");
|
||||
@ -122,8 +104,8 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil
|
||||
if (stream_type != QueryPipelineBuilder::StreamType::Main)
|
||||
return nullptr;
|
||||
|
||||
return std::make_shared<DistinctSortedChunkTransform>(
|
||||
header, set_size_limits, limit_hint, distinct_sort_desc, columns, true);
|
||||
return std::make_shared<DistinctSortedStreamTransform>(
|
||||
header, set_size_limits, limit_hint, distinct_sort_desc, columns);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
@ -14,8 +14,8 @@ public:
|
||||
const SizeLimits & set_size_limits_,
|
||||
UInt64 limit_hint_,
|
||||
const Names & columns_,
|
||||
bool pre_distinct_, /// If is enabled, execute distinct for separate streams. Otherwise, merge streams.
|
||||
bool optimize_distinct_in_order_);
|
||||
/// If is enabled, execute distinct for separate streams, otherwise for merged streams.
|
||||
bool pre_distinct_);
|
||||
|
||||
String getName() const override { return "Distinct"; }
|
||||
const Names & getColumnNames() const { return columns; }
|
||||
@ -29,6 +29,11 @@ public:
|
||||
|
||||
UInt64 getLimitHint() const { return limit_hint; }
|
||||
|
||||
const SizeLimits & getSetSizeLimits() const { return set_size_limits; }
|
||||
|
||||
void applyOrder(SortDescription sort_desc) { distinct_sort_desc = std::move(sort_desc); }
|
||||
const SortDescription & getSortDescription() const override { return distinct_sort_desc; }
|
||||
|
||||
private:
|
||||
void updateOutputStream() override;
|
||||
|
||||
@ -36,7 +41,7 @@ private:
|
||||
UInt64 limit_hint;
|
||||
const Names columns;
|
||||
bool pre_distinct;
|
||||
bool optimize_distinct_in_order;
|
||||
SortDescription distinct_sort_desc;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -10,14 +10,14 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
static ITransformingStep::Traits getTraits(const ActionsDAG & actions, const Block & header, const SortDescription & sort_description)
|
||||
static ITransformingStep::Traits getTraits(const ActionsDAG & actions)
|
||||
{
|
||||
return ITransformingStep::Traits
|
||||
{
|
||||
{
|
||||
.returns_single_stream = false,
|
||||
.preserves_number_of_streams = true,
|
||||
.preserves_sorting = actions.isSortingPreserved(header, sort_description),
|
||||
.preserves_sorting = false,
|
||||
},
|
||||
{
|
||||
.preserves_number_of_rows = !actions.hasArrayJoin(),
|
||||
@ -29,7 +29,7 @@ ExpressionStep::ExpressionStep(const DataStream & input_stream_, ActionsDAG acti
|
||||
: ITransformingStep(
|
||||
input_stream_,
|
||||
ExpressionTransform::transformHeader(input_stream_.header, actions_dag_),
|
||||
getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description))
|
||||
getTraits(actions_dag_))
|
||||
, actions_dag(std::move(actions_dag_))
|
||||
{
|
||||
}
|
||||
@ -78,16 +78,6 @@ void ExpressionStep::updateOutputStream()
|
||||
|
||||
if (!getDataStreamTraits().preserves_sorting)
|
||||
return;
|
||||
|
||||
FindAliasForInputName alias_finder(actions_dag);
|
||||
const auto & input_sort_description = getInputStreams().front().sort_description;
|
||||
for (size_t i = 0, s = input_sort_description.size(); i < s; ++i)
|
||||
{
|
||||
const auto & original_column = input_sort_description[i].column_name;
|
||||
const auto * alias_node = alias_finder.find(original_column);
|
||||
if (alias_node)
|
||||
output_stream->sort_description[i].column_name = alias_node->result_name;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -24,7 +24,7 @@ public:
|
||||
void describeActions(JSONBuilder::JSONMap & map) const override;
|
||||
void describeActions(FormatSettings & settings) const override;
|
||||
|
||||
const SortDescription & getSortDescription() const { return sort_description; }
|
||||
const SortDescription & getSortDescription() const override { return sort_description; }
|
||||
|
||||
private:
|
||||
void updateOutputStream() override;
|
||||
|
||||
@ -9,23 +9,14 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
static ITransformingStep::Traits getTraits(const ActionsDAG & expression, const Block & header, const SortDescription & sort_description, bool remove_filter_column, const String & filter_column_name)
|
||||
static ITransformingStep::Traits getTraits()
|
||||
{
|
||||
bool preserves_sorting = expression.isSortingPreserved(header, sort_description, remove_filter_column ? filter_column_name : "");
|
||||
if (remove_filter_column)
|
||||
{
|
||||
preserves_sorting &= std::find_if(
|
||||
begin(sort_description),
|
||||
end(sort_description),
|
||||
[&](const auto & column_desc) { return column_desc.column_name == filter_column_name; })
|
||||
== sort_description.end();
|
||||
}
|
||||
return ITransformingStep::Traits
|
||||
{
|
||||
{
|
||||
.returns_single_stream = false,
|
||||
.preserves_number_of_streams = true,
|
||||
.preserves_sorting = preserves_sorting,
|
||||
.preserves_sorting = false,
|
||||
},
|
||||
{
|
||||
.preserves_number_of_rows = false,
|
||||
@ -45,12 +36,16 @@ FilterStep::FilterStep(
|
||||
&actions_dag_,
|
||||
filter_column_name_,
|
||||
remove_filter_column_),
|
||||
getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description, remove_filter_column_, filter_column_name_))
|
||||
getTraits())
|
||||
, actions_dag(std::move(actions_dag_))
|
||||
, filter_column_name(std::move(filter_column_name_))
|
||||
, remove_filter_column(remove_filter_column_)
|
||||
{
|
||||
actions_dag.removeAliasesForFilter(filter_column_name);
|
||||
/// Removing aliases may result in unneeded ALIAS node in DAG.
|
||||
/// This should not be an issue by itself,
|
||||
/// but it might trigger an issue with duplicated names in Block after plan optimizations.
|
||||
actions_dag.removeUnusedActions(false, false);
|
||||
}
|
||||
|
||||
void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
|
||||
@ -109,16 +104,6 @@ void FilterStep::updateOutputStream()
|
||||
|
||||
if (!getDataStreamTraits().preserves_sorting)
|
||||
return;
|
||||
|
||||
FindAliasForInputName alias_finder(actions_dag);
|
||||
const auto & input_sort_description = getInputStreams().front().sort_description;
|
||||
for (size_t i = 0, s = input_sort_description.size(); i < s; ++i)
|
||||
{
|
||||
const auto & original_column = input_sort_description[i].column_name;
|
||||
const auto * alias_node = alias_finder.find(original_column);
|
||||
if (alias_node)
|
||||
output_stream->sort_description[i].column_name = alias_node->result_name;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -18,6 +18,12 @@ const DataStream & IQueryPlanStep::getOutputStream() const
|
||||
return *output_stream;
|
||||
}
|
||||
|
||||
const SortDescription & IQueryPlanStep::getSortDescription() const
|
||||
{
|
||||
static SortDescription empty;
|
||||
return empty;
|
||||
}
|
||||
|
||||
static void doDescribeHeader(const Block & header, size_t count, IQueryPlanStep::FormatSettings & settings)
|
||||
{
|
||||
String prefix(settings.offset, settings.indent_char);
|
||||
|
||||
@ -28,30 +28,6 @@ class DataStream
|
||||
public:
|
||||
Block header;
|
||||
|
||||
/// Sorting scope. Please keep the mutual order (more strong mode should have greater value).
|
||||
enum class SortScope : uint8_t
|
||||
{
|
||||
None = 0,
|
||||
Chunk = 1, /// Separate chunks are sorted
|
||||
Stream = 2, /// Each data steam is sorted
|
||||
Global = 3, /// Data is globally sorted
|
||||
};
|
||||
|
||||
/// It is not guaranteed that header has columns from sort_description.
|
||||
SortDescription sort_description = {};
|
||||
SortScope sort_scope = SortScope::None;
|
||||
|
||||
/// Things which may be added:
|
||||
/// * limit
|
||||
/// * estimated rows number
|
||||
/// * memory allocation context
|
||||
|
||||
bool hasEqualPropertiesWith(const DataStream & other) const
|
||||
{
|
||||
return sort_description == other.sort_description
|
||||
&& (sort_description.empty() || sort_scope == other.sort_scope);
|
||||
}
|
||||
|
||||
bool hasEqualHeaderWith(const DataStream & other) const
|
||||
{
|
||||
return blocksHaveEqualStructure(header, other.header);
|
||||
@ -88,6 +64,8 @@ public:
|
||||
const std::string & getStepDescription() const { return step_description; }
|
||||
void setStepDescription(std::string description) { step_description = std::move(description); }
|
||||
|
||||
virtual const SortDescription & getSortDescription() const;
|
||||
|
||||
struct FormatSettings
|
||||
{
|
||||
WriteBuffer & out;
|
||||
|
||||
@ -13,19 +13,14 @@ ITransformingStep::ITransformingStep(DataStream input_stream, Block output_heade
|
||||
output_stream = createOutputStream(input_streams.front(), std::move(output_header), data_stream_traits);
|
||||
}
|
||||
|
||||
/// TODO: cleanup in the following PR.
|
||||
DataStream ITransformingStep::createOutputStream(
|
||||
const DataStream & input_stream,
|
||||
[[maybe_unused]] const DataStream & input_stream,
|
||||
Block output_header,
|
||||
const DataStreamTraits & stream_traits)
|
||||
[[maybe_unused]] const DataStreamTraits & stream_traits)
|
||||
{
|
||||
DataStream output_stream{.header = std::move(output_header)};
|
||||
|
||||
if (stream_traits.preserves_sorting)
|
||||
{
|
||||
output_stream.sort_description = input_stream.sort_description;
|
||||
output_stream.sort_scope = input_stream.sort_scope;
|
||||
}
|
||||
|
||||
return output_stream;
|
||||
}
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h>
|
||||
#include <Processors/Transforms/MergingAggregatedTransform.h>
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
#include <Common/JSONBuilder.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -15,15 +16,6 @@ namespace ErrorCodes
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
static bool memoryBoundMergingWillBeUsed(
|
||||
const DataStream & input_stream,
|
||||
bool memory_bound_merging_of_aggregation_results_enabled,
|
||||
const SortDescription & group_by_sort_description)
|
||||
{
|
||||
return memory_bound_merging_of_aggregation_results_enabled && !group_by_sort_description.empty()
|
||||
&& input_stream.sort_scope >= DataStream::SortScope::Stream && input_stream.sort_description.hasPrefix(group_by_sort_description);
|
||||
}
|
||||
|
||||
static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number)
|
||||
{
|
||||
return ITransformingStep::Traits
|
||||
@ -50,7 +42,6 @@ MergingAggregatedStep::MergingAggregatedStep(
|
||||
bool should_produce_results_in_order_of_bucket_number_,
|
||||
size_t max_block_size_,
|
||||
size_t memory_bound_merging_max_block_bytes_,
|
||||
SortDescription group_by_sort_description_,
|
||||
bool memory_bound_merging_of_aggregation_results_enabled_)
|
||||
: ITransformingStep(
|
||||
input_stream_,
|
||||
@ -64,34 +55,15 @@ MergingAggregatedStep::MergingAggregatedStep(
|
||||
, memory_efficient_merge_threads(memory_efficient_merge_threads_)
|
||||
, max_block_size(max_block_size_)
|
||||
, memory_bound_merging_max_block_bytes(memory_bound_merging_max_block_bytes_)
|
||||
, group_by_sort_description(std::move(group_by_sort_description_))
|
||||
, should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_)
|
||||
, memory_bound_merging_of_aggregation_results_enabled(memory_bound_merging_of_aggregation_results_enabled_)
|
||||
{
|
||||
if (memoryBoundMergingWillBeUsed() && should_produce_results_in_order_of_bucket_number)
|
||||
{
|
||||
output_stream->sort_description = group_by_sort_description;
|
||||
output_stream->sort_scope = DataStream::SortScope::Global;
|
||||
}
|
||||
}
|
||||
|
||||
void MergingAggregatedStep::applyOrder(SortDescription sort_description, DataStream::SortScope sort_scope)
|
||||
void MergingAggregatedStep::applyOrder(SortDescription input_sort_description)
|
||||
{
|
||||
is_order_overwritten = true;
|
||||
overwritten_sort_scope = sort_scope;
|
||||
|
||||
auto & input_stream = input_streams.front();
|
||||
input_stream.sort_scope = sort_scope;
|
||||
input_stream.sort_description = sort_description;
|
||||
|
||||
/// Columns might be reordered during optimization, so we better to update sort description.
|
||||
group_by_sort_description = std::move(sort_description);
|
||||
|
||||
if (memoryBoundMergingWillBeUsed() && should_produce_results_in_order_of_bucket_number)
|
||||
{
|
||||
output_stream->sort_description = group_by_sort_description;
|
||||
output_stream->sort_scope = DataStream::SortScope::Global;
|
||||
}
|
||||
group_by_sort_description = std::move(input_sort_description);
|
||||
}
|
||||
|
||||
void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
|
||||
@ -159,11 +131,18 @@ void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, c
|
||||
void MergingAggregatedStep::describeActions(FormatSettings & settings) const
|
||||
{
|
||||
params.explain(settings.out, settings.offset);
|
||||
if (!group_by_sort_description.empty())
|
||||
{
|
||||
String prefix(settings.offset, settings.indent_char);
|
||||
settings.out << prefix << "Order: " << dumpSortDescription(group_by_sort_description) << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
void MergingAggregatedStep::describeActions(JSONBuilder::JSONMap & map) const
|
||||
{
|
||||
params.explain(map);
|
||||
if (!group_by_sort_description.empty())
|
||||
map.add("Order", dumpSortDescription(group_by_sort_description));
|
||||
}
|
||||
|
||||
void MergingAggregatedStep::updateOutputStream()
|
||||
@ -171,13 +150,19 @@ void MergingAggregatedStep::updateOutputStream()
|
||||
const auto & in_header = input_streams.front().header;
|
||||
output_stream = createOutputStream(input_streams.front(),
|
||||
MergingAggregatedTransform::appendGroupingIfNeeded(in_header, params.getHeader(in_header, final)), getDataStreamTraits());
|
||||
if (is_order_overwritten) /// overwrite order again
|
||||
applyOrder(group_by_sort_description, overwritten_sort_scope);
|
||||
}
|
||||
|
||||
bool MergingAggregatedStep::memoryBoundMergingWillBeUsed() const
|
||||
{
|
||||
return DB::memoryBoundMergingWillBeUsed(
|
||||
input_streams.front(), memory_bound_merging_of_aggregation_results_enabled, group_by_sort_description);
|
||||
return memory_bound_merging_of_aggregation_results_enabled && !group_by_sort_description.empty();
|
||||
}
|
||||
|
||||
const SortDescription & MergingAggregatedStep::getSortDescription() const
|
||||
{
|
||||
if (memoryBoundMergingWillBeUsed() && should_produce_results_in_order_of_bucket_number)
|
||||
return group_by_sort_description;
|
||||
|
||||
return IQueryPlanStep::getSortDescription();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -24,7 +24,6 @@ public:
|
||||
bool should_produce_results_in_order_of_bucket_number_,
|
||||
size_t max_block_size_,
|
||||
size_t memory_bound_merging_max_block_bytes_,
|
||||
SortDescription group_by_sort_description_,
|
||||
bool memory_bound_merging_of_aggregation_results_enabled_);
|
||||
|
||||
String getName() const override { return "MergingAggregated"; }
|
||||
@ -35,7 +34,9 @@ public:
|
||||
void describeActions(JSONBuilder::JSONMap & map) const override;
|
||||
void describeActions(FormatSettings & settings) const override;
|
||||
|
||||
void applyOrder(SortDescription input_sort_description, DataStream::SortScope sort_scope);
|
||||
void applyOrder(SortDescription input_sort_description);
|
||||
const SortDescription & getSortDescription() const override;
|
||||
const SortDescription & getGroupBySortDescription() const { return group_by_sort_description; }
|
||||
|
||||
bool memoryBoundMergingWillBeUsed() const;
|
||||
|
||||
@ -46,16 +47,13 @@ private:
|
||||
Aggregator::Params params;
|
||||
GroupingSetsParamsList grouping_sets_params;
|
||||
bool final;
|
||||
bool memory_efficient_aggregation;
|
||||
const bool memory_efficient_aggregation;
|
||||
size_t max_threads;
|
||||
size_t memory_efficient_merge_threads;
|
||||
const size_t max_block_size;
|
||||
const size_t memory_bound_merging_max_block_bytes;
|
||||
SortDescription group_by_sort_description;
|
||||
|
||||
bool is_order_overwritten = false;
|
||||
DataStream::SortScope overwritten_sort_scope = DataStream::SortScope::None;
|
||||
|
||||
/// These settings are used to determine if we should resize pipeline to 1 at the end.
|
||||
const bool should_produce_results_in_order_of_bucket_number;
|
||||
const bool memory_bound_merging_of_aggregation_results_enabled;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user