mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge remote-tracking branch 'rschu1ze/master' into more-mysql-compat
This commit is contained in:
Robert Schulze
commit
f4bc58ea92
@ -0,0 +1,16 @@
|
||||
version: '2.3'
|
||||
services:
|
||||
openldap:
|
||||
image: bitnami/openldap:2.6.6
|
||||
restart: always
|
||||
environment:
|
||||
LDAP_ROOT: dc=example,dc=org
|
||||
LDAP_ADMIN_DN: cn=admin,dc=example,dc=org
|
||||
LDAP_ADMIN_USERNAME: admin
|
||||
LDAP_ADMIN_PASSWORD: clickhouse
|
||||
LDAP_USER_DC: users
|
||||
LDAP_USERS: janedoe,johndoe
|
||||
LDAP_PASSWORDS: qwerty,qwertz
|
||||
LDAP_PORT_NUMBER: ${LDAP_INTERNAL_PORT:-1389}
|
||||
ports:
|
||||
- ${LDAP_EXTERNAL_PORT:-1389}:${LDAP_INTERNAL_PORT:-1389}
|
||||
@ -60,7 +60,7 @@ Before using cache, add it to `config.xml`
|
||||
- limit_size: Required. The maximum size(in bytes) of local cache files.
|
||||
- bytes_read_before_flush: Control bytes before flush to local filesystem when downloading file from remote filesystem. The default value is 1MB.
|
||||
|
||||
When ClickHouse is started up with local cache for remote filesystem enabled, users can still choose not to use cache with `settings use_local_cache_for_remote_fs = 0` in their query. `use_local_cache_for_remote_fs` is `false` in default.
|
||||
When ClickHouse is started up with local cache for remote filesystem enabled, users can still choose not to use cache with `settings use_local_cache_for_remote_storage = 0` in their query. `use_local_cache_for_remote_storage` is `1` by default.
|
||||
|
||||
### Query Hive Table with ORC Input Format
|
||||
|
||||
|
||||
@ -69,7 +69,7 @@ may return cached results then.
|
||||
The query cache can be cleared using statement `SYSTEM DROP QUERY CACHE`. The content of the query cache is displayed in system table
|
||||
`system.query_cache`. The number of query cache hits and misses since database start are shown as events "QueryCacheHits" and
|
||||
"QueryCacheMisses" in system table [system.events](system-tables/events.md). Both counters are only updated for `SELECT` queries which run
|
||||
with setting `use_query_cache = true`, other queries do not affect "QueryCacheMisses". Field `query_log_usage` in system table
|
||||
with setting `use_query_cache = true`, other queries do not affect "QueryCacheMisses". Field `query_cache_usage` in system table
|
||||
[system.query_log](system-tables/query_log.md) shows for each executed query whether the query result was written into or read from the
|
||||
query cache. Asynchronous metrics "QueryCacheEntries" and "QueryCacheBytes" in system table
|
||||
[system.asynchronous_metrics](system-tables/asynchronous_metrics.md) show how many entries / bytes the query cache currently contains.
|
||||
|
||||
@ -502,7 +502,7 @@ Possible values:
|
||||
Default value: 480.
|
||||
|
||||
After merging several parts into a new part, ClickHouse marks the original parts as inactive and deletes them only after `old_parts_lifetime` seconds.
|
||||
Inactive parts are removed if they are not used by current queries, i.e. if the `refcount` of the part is zero.
|
||||
Inactive parts are removed if they are not used by current queries, i.e. if the `refcount` of the part is 1.
|
||||
|
||||
`fsync` is not called for new parts, so for some time new parts exist only in the server's RAM (OS cache). If the server is rebooted spontaneously, new parts can be lost or damaged.
|
||||
To protect data inactive parts are not deleted immediately.
|
||||
|
||||
@ -3279,6 +3279,17 @@ Possible values:
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## use_mysql_types_in_show_columns {#use_mysql_types_in_show_columns}
|
||||
|
||||
Show the names of MySQL data types corresponding to ClickHouse data types in [SHOW COLUMNS](../../sql-reference/statements/show.md#show_columns) and SELECTs on [system.columns](../system-tables/columns.md).
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 - Show names of native ClickHouse data types.
|
||||
- 1 - Show names of MySQL data types corresponding to ClickHouse data types.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold}
|
||||
|
||||
Enables special logic to perform merges on replicas.
|
||||
|
||||
@ -14,7 +14,7 @@ The `system.columns` table contains the following columns (the column type is sh
|
||||
- `database` ([String](../../sql-reference/data-types/string.md)) — Database name.
|
||||
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
||||
- `name` ([String](../../sql-reference/data-types/string.md)) — Column name.
|
||||
- `type` ([String](../../sql-reference/data-types/string.md)) — Column type.
|
||||
- `type` ([String](../../sql-reference/data-types/string.md)) — Column type. If setting `[use_mysql_types_in_show_columns](../../operations/settings/settings.md#use_mysql_types_in_show_columns) = 1` (default: 0), then the equivalent type name in MySQL is shown.
|
||||
- `position` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Ordinal position of a column in a table starting with 1.
|
||||
- `default_kind` ([String](../../sql-reference/data-types/string.md)) — Expression type (`DEFAULT`, `MATERIALIZED`, `ALIAS`) for the default value, or an empty string if it is not defined.
|
||||
- `default_expression` ([String](../../sql-reference/data-types/string.md)) — Expression for the default value, or an empty string if it is not defined.
|
||||
|
||||
@ -34,10 +34,10 @@ Returns an array of selected substrings. Empty substrings may be selected when:
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
:::note
|
||||
The behavior of parameter `max_substrings` changed starting with ClickHouse v22.11. In versions older than that, `max_substrings` > 0 meant that `max_substring`-many splits were performed and that the remainder of the string was returned as the final element of the list.
|
||||
The behavior of parameter `max_substrings` changed starting with ClickHouse v22.11. In versions older than that, `max_substrings > 0` meant that `max_substring`-many splits were performed and that the remainder of the string was returned as the final element of the list.
|
||||
For example,
|
||||
- in v22.10: `SELECT splitByChar('=', 'a=b=c=d', 2); -- ['a','b','c=d']`
|
||||
- in v22.11: `SELECT splitByChar('=', 'a=b=c=d', 2); -- ['a','b']`
|
||||
- in v22.10: `SELECT splitByChar('=', 'a=b=c=d', 2);` returned `['a','b','c=d']`
|
||||
- in v22.11: `SELECT splitByChar('=', 'a=b=c=d', 2);` returned `['a','b']`
|
||||
|
||||
A behavior similar to ClickHouse pre-v22.11 can be achieved by setting
|
||||
[splitby_max_substrings_includes_remaining_string](../../operations/settings/settings.md#splitby_max_substrings_includes_remaining_string)
|
||||
|
||||
@ -189,7 +189,7 @@ Result:
|
||||
- [Create Tables](https://clickhouse.com/docs/en/getting-started/tutorial/#create-tables)
|
||||
- [SHOW CREATE TABLE](https://clickhouse.com/docs/en/sql-reference/statements/show/#show-create-table)
|
||||
|
||||
## SHOW COLUMNS
|
||||
## SHOW COLUMNS {#show_columns}
|
||||
|
||||
Displays a list of columns
|
||||
|
||||
@ -206,15 +206,15 @@ The optional keyword `EXTENDED` currently has no effect, it only exists for MySQ
|
||||
The optional keyword `FULL` causes the output to include the collation, comment and privilege columns.
|
||||
|
||||
The statement produces a result table with the following structure:
|
||||
- field - The name of the column (String)
|
||||
- type - The column data type (String)
|
||||
- null - `YES` if the column data type is Nullable, `NO` otherwise (String)
|
||||
- key - `PRI` if the column is part of the primary key, `SOR` if the column is part of the sorting key, empty otherwise (String)
|
||||
- default - Default expression of the column if it is of type `ALIAS`, `DEFAULT`, or `MATERIALIZED`, otherwise `NULL`. (Nullable(String))
|
||||
- extra - Additional information, currently unused (String)
|
||||
- collation - (only if `FULL` keyword was specified) Collation of the column, always `NULL` because ClickHouse has no per-column collations (Nullable(String))
|
||||
- comment - (only if `FULL` keyword was specified) Comment on the column (String)
|
||||
- privilege - (only if `FULL` keyword was specified) The privilege you have on this column, currently not available (String)
|
||||
- `field` - The name of the column (String)
|
||||
- `type` - The column data type. If setting `[use_mysql_types_in_show_columns](../../operations/settings/settings.md#use_mysql_types_in_show_columns) = 1` (default: 0), then the equivalent type name in MySQL is shown. (String)
|
||||
- `null` - `YES` if the column data type is Nullable, `NO` otherwise (String)
|
||||
- `key` - `PRI` if the column is part of the primary key, `SOR` if the column is part of the sorting key, empty otherwise (String)
|
||||
- `default` - Default expression of the column if it is of type `ALIAS`, `DEFAULT`, or `MATERIALIZED`, otherwise `NULL`. (Nullable(String))
|
||||
- `extra` - Additional information, currently unused (String)
|
||||
- `collation` - (only if `FULL` keyword was specified) Collation of the column, always `NULL` because ClickHouse has no per-column collations (Nullable(String))
|
||||
- `comment` - (only if `FULL` keyword was specified) Comment on the column (String)
|
||||
- `privilege` - (only if `FULL` keyword was specified) The privilege you have on this column, currently not available (String)
|
||||
|
||||
**Examples**
|
||||
|
||||
@ -286,21 +286,21 @@ equivalent. If no database is specified, the query assumes the current database
|
||||
The optional keyword `EXTENDED` currently has no effect, it only exists for MySQL compatibility.
|
||||
|
||||
The statement produces a result table with the following structure:
|
||||
- table - The name of the table. (String)
|
||||
- non_unique - Always `1` as ClickHouse does not support uniqueness constraints. (UInt8)
|
||||
- key_name - The name of the index, `PRIMARY` if the index is a primary key index. (String)
|
||||
- seq_in_index - For a primary key index, the position of the column starting from `1`. For a data skipping index: always `1`. (UInt8)
|
||||
- column_name - For a primary key index, the name of the column. For a data skipping index: `''` (empty string), see field "expression". (String)
|
||||
- collation - The sorting of the column in the index: `A` if ascending, `D` if descending, `NULL` if unsorted. (Nullable(String))
|
||||
- cardinality - An estimation of the index cardinality (number of unique values in the index). Currently always 0. (UInt64)
|
||||
- sub_part - Always `NULL` because ClickHouse does not support index prefixes like MySQL. (Nullable(String))
|
||||
- packed - Always `NULL` because ClickHouse does not support packed indexes (like MySQL). (Nullable(String))
|
||||
- null - Currently unused
|
||||
- index_type - The index type, e.g. `PRIMARY`, `MINMAX`, `BLOOM_FILTER` etc. (String)
|
||||
- comment - Additional information about the index, currently always `''` (empty string). (String)
|
||||
- index_comment - `''` (empty string) because indexes in ClickHouse cannot have a `COMMENT` field (like in MySQL). (String)
|
||||
- visible - If the index is visible to the optimizer, always `YES`. (String)
|
||||
- expression - For a data skipping index, the index expression. For a primary key index: `''` (empty string). (String)
|
||||
- `table` - The name of the table. (String)
|
||||
- `non_unique` - Always `1` as ClickHouse does not support uniqueness constraints. (UInt8)
|
||||
- `key_name` - The name of the index, `PRIMARY` if the index is a primary key index. (String)
|
||||
- `seq_in_index` - For a primary key index, the position of the column starting from `1`. For a data skipping index: always `1`. (UInt8)
|
||||
- `column_name` - For a primary key index, the name of the column. For a data skipping index: `''` (empty string), see field "expression". (String)
|
||||
- `collation` - The sorting of the column in the index: `A` if ascending, `D` if descending, `NULL` if unsorted. (Nullable(String))
|
||||
- `cardinality` - An estimation of the index cardinality (number of unique values in the index). Currently always 0. (UInt64)
|
||||
- `sub_part` - Always `NULL` because ClickHouse does not support index prefixes like MySQL. (Nullable(String))
|
||||
- `packed` - Always `NULL` because ClickHouse does not support packed indexes (like MySQL). (Nullable(String))
|
||||
- `null` - Currently unused
|
||||
- `index_type` - The index type, e.g. `PRIMARY`, `MINMAX`, `BLOOM_FILTER` etc. (String)
|
||||
- `comment` - Additional information about the index, currently always `''` (empty string). (String)
|
||||
- `index_comment` - `''` (empty string) because indexes in ClickHouse cannot have a `COMMENT` field (like in MySQL). (String)
|
||||
- `visible` - If the index is visible to the optimizer, always `YES`. (String)
|
||||
- `expression` - For a data skipping index, the index expression. For a primary key index: `''` (empty string). (String)
|
||||
|
||||
**Examples**
|
||||
|
||||
|
||||
@ -61,7 +61,7 @@ PARTITION BY expr
|
||||
- limit_size: 必需的。本地缓存文件的最大大小(单位为字节)。
|
||||
- bytes_read_before_flush: 从远程文件系统下载文件时，刷新到本地文件系统前的控制字节数。缺省值为1MB。
|
||||
|
||||
当ClickHouse为远程文件系统启用了本地缓存时，用户仍然可以选择不使用缓存，并在查询中设置`use_local_cache_for_remote_fs = 0 `, `use_local_cache_for_remote_fs` 默认为 `false`。
|
||||
当ClickHouse为远程文件系统启用了本地缓存时，用户仍然可以选择不使用缓存，并在查询中设置 `use_local_cache_for_remote_storage = 0`, `use_local_cache_for_remote_storage` 默认为 `1`。
|
||||
|
||||
### 查询 ORC 输入格式的Hive 表
|
||||
|
||||
|
||||
@ -84,8 +84,8 @@ function deb2tgz {
|
||||
FILE=$1
|
||||
PKG_NAME=${FILE##*/}; PKG_NAME=${PKG_NAME%%_*}
|
||||
PKG_DIR="$PKG_NAME-$CLICKHOUSE_VERSION_STRING"
|
||||
PKG_PATH="$OUTPUT_DIR/$PKG_NAME-$CLICKHOUSE_VERSION_STRING"
|
||||
TARBALL="$OUTPUT_DIR/$PKG_NAME-$CLICKHOUSE_VERSION_STRING-$DEB_ARCH.tgz"
|
||||
PKG_PATH="$OUTPUT_DIR/$PKG_DIR"
|
||||
TARBALL="$OUTPUT_DIR/$PKG_DIR-$DEB_ARCH.tgz"
|
||||
rm -rf "$PKG_PATH"
|
||||
dpkg-deb -R "$FILE" "$PKG_PATH"
|
||||
mkdir -p "$PKG_PATH/install"
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
# package sources should be placed in ${PWD}/root
|
||||
# nfpm should run from the same directory with a config
|
||||
name: "clickhouse-client"
|
||||
description: |
|
||||
Client binary for ClickHouse
|
||||
ClickHouse is a column-oriented database management system.
|
||||
that allows generating analytical data reports in real time.
|
||||
This package provides clickhouse-client, clickhouse-local and clickhouse-benchmark.
|
||||
|
||||
# Common packages config
|
||||
arch: "${DEB_ARCH}" # amd64, arm64
|
||||
platform: "linux"
|
||||
version: "${CLICKHOUSE_VERSION_STRING}"
|
||||
@ -9,19 +16,17 @@ homepage: "https://clickhouse.com"
|
||||
license: "Apache"
|
||||
section: "database"
|
||||
priority: "optional"
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
# Package specific content
|
||||
replaces:
|
||||
- clickhouse-compressor
|
||||
conflicts:
|
||||
- clickhouse-compressor
|
||||
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
description: |
|
||||
Client binary for ClickHouse
|
||||
ClickHouse is a column-oriented database management system.
|
||||
that allows generating analytical data reports in real time.
|
||||
This package provides clickhouse-client, clickhouse-local and clickhouse-benchmark.
|
||||
|
||||
overrides:
|
||||
deb:
|
||||
depends:
|
||||
@ -30,10 +35,6 @@ overrides:
|
||||
depends:
|
||||
- clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING}
|
||||
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
contents:
|
||||
- src: root/etc/clickhouse-client/config.xml
|
||||
dst: /etc/clickhouse-client/config.xml
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
# package sources should be placed in ${PWD}/root
|
||||
# nfpm should run from the same directory with a config
|
||||
name: "clickhouse-common-static-dbg"
|
||||
description: |
|
||||
debugging symbols for clickhouse-common-static
|
||||
This package contains the debugging symbols for clickhouse-common.
|
||||
|
||||
|
||||
#
|
||||
# Common packages config
|
||||
arch: "${DEB_ARCH}" # amd64, arm64
|
||||
platform: "linux"
|
||||
version: "${CLICKHOUSE_VERSION_STRING}"
|
||||
@ -9,21 +16,17 @@ homepage: "https://clickhouse.com"
|
||||
license: "Apache"
|
||||
section: "database"
|
||||
priority: "optional"
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
# Package specific content
|
||||
replaces:
|
||||
- clickhouse-common-dbg
|
||||
conflicts:
|
||||
- clickhouse-common-dbg
|
||||
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
description: |
|
||||
debugging symbols for clickhouse-common-static
|
||||
This package contains the debugging symbols for clickhouse-common.
|
||||
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
contents:
|
||||
- src: root/usr/lib/debug/usr/bin/clickhouse.debug
|
||||
dst: /usr/lib/debug/usr/bin/clickhouse.debug
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
# package sources should be placed in ${PWD}/root
|
||||
# nfpm should run from the same directory with a config
|
||||
name: "clickhouse-common-static"
|
||||
description: |
|
||||
Common files for ClickHouse
|
||||
ClickHouse is a column-oriented database management system
|
||||
that allows generating analytical data reports in real time.
|
||||
This package provides common files for both clickhouse server and client
|
||||
|
||||
# Common packages config
|
||||
arch: "${DEB_ARCH}" # amd64, arm64
|
||||
platform: "linux"
|
||||
version: "${CLICKHOUSE_VERSION_STRING}"
|
||||
@ -9,7 +16,12 @@ homepage: "https://clickhouse.com"
|
||||
license: "Apache"
|
||||
section: "database"
|
||||
priority: "optional"
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
# Package specific content
|
||||
replaces:
|
||||
- clickhouse-common
|
||||
- clickhouse-server-base
|
||||
@ -19,17 +31,6 @@ provides:
|
||||
suggests:
|
||||
- clickhouse-common-static-dbg
|
||||
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
description: |
|
||||
Common files for ClickHouse
|
||||
ClickHouse is a column-oriented database management system
|
||||
that allows generating analytical data reports in real time.
|
||||
This package provides common files for both clickhouse server and client
|
||||
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
contents:
|
||||
- src: root/usr/bin/clickhouse
|
||||
dst: /usr/bin/clickhouse
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
# package sources should be placed in ${PWD}/root
|
||||
# nfpm should run from the same directory with a config
|
||||
name: "clickhouse-keeper-dbg"
|
||||
description: |
|
||||
debugging symbols for clickhouse-keeper
|
||||
This package contains the debugging symbols for clickhouse-keeper.
|
||||
|
||||
|
||||
#
|
||||
# Common packages config
|
||||
arch: "${DEB_ARCH}" # amd64, arm64
|
||||
platform: "linux"
|
||||
version: "${CLICKHOUSE_VERSION_STRING}"
|
||||
@ -10,14 +17,11 @@ license: "Apache"
|
||||
section: "database"
|
||||
priority: "optional"
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
description: |
|
||||
debugging symbols for clickhouse-keeper
|
||||
This package contains the debugging symbols for clickhouse-keeper.
|
||||
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
# Package specific content
|
||||
contents:
|
||||
- src: root/usr/lib/debug/usr/bin/clickhouse-keeper.debug
|
||||
dst: /usr/lib/debug/usr/bin/clickhouse-keeper.debug
|
||||
|
||||
@ -3,12 +3,12 @@ set -e
|
||||
# set -x
|
||||
|
||||
PROGRAM=clickhouse-keeper
|
||||
KEEPER_USER=${KEEPER_USER:=clickhouse}
|
||||
KEEPER_GROUP=${KEEPER_GROUP:=clickhouse}
|
||||
KEEPER_USER=${KEEPER_USER:-clickhouse}
|
||||
KEEPER_GROUP=${KEEPER_GROUP:-clickhouse}
|
||||
# Please note that we don't support paths with whitespaces. This is rather ignorant.
|
||||
KEEPER_CONFDIR=${KEEPER_CONFDIR:=/etc/$PROGRAM}
|
||||
KEEPER_DATADIR=${KEEPER_DATADIR:=/var/lib/clickhouse}
|
||||
KEEPER_LOGDIR=${KEEPER_LOGDIR:=/var/log/$PROGRAM}
|
||||
KEEPER_CONFDIR=${KEEPER_CONFDIR:-/etc/$PROGRAM}
|
||||
KEEPER_DATADIR=${KEEPER_DATADIR:-/var/lib/clickhouse}
|
||||
KEEPER_LOGDIR=${KEEPER_LOGDIR:-/var/log/$PROGRAM}
|
||||
|
||||
[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule
|
||||
[ -f /etc/default/clickhouse-keeper ] && . /etc/default/clickhouse-keeper
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
# package sources should be placed in ${PWD}/root
|
||||
# nfpm should run from the same directory with a config
|
||||
name: "clickhouse-keeper"
|
||||
description: |
|
||||
Static clickhouse-keeper binary
|
||||
A stand-alone clickhouse-keeper package
|
||||
|
||||
|
||||
#
|
||||
# Common packages config
|
||||
arch: "${DEB_ARCH}" # amd64, arm64
|
||||
platform: "linux"
|
||||
version: "${CLICKHOUSE_VERSION_STRING}"
|
||||
@ -9,29 +16,25 @@ homepage: "https://clickhouse.com"
|
||||
license: "Apache"
|
||||
section: "database"
|
||||
priority: "optional"
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
# Package specific content
|
||||
conflicts:
|
||||
- clickhouse-server
|
||||
suggests:
|
||||
- clickhouse-keeper-dbg
|
||||
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
description: |
|
||||
Static clickhouse-keeper binary
|
||||
A stand-alone clickhouse-keeper package
|
||||
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
contents:
|
||||
- src: root/etc/clickhouse-keeper/keeper_config.xml
|
||||
dst: /etc/clickhouse-keeper/keeper_config.xml
|
||||
type: config|noreplace
|
||||
- src: root/usr/bin/clickhouse-keeper
|
||||
dst: /usr/bin/clickhouse-keeper
|
||||
- src: clickhouse-keeper.service
|
||||
dst: /lib/systemd/system/clickhouse-keeper.service
|
||||
- src: root/usr/bin/clickhouse-keeper
|
||||
dst: /usr/bin/clickhouse-keeper
|
||||
- src: clickhouse-keeper
|
||||
dst: /usr/bin/clickhouse-keeper-client
|
||||
type: symlink
|
||||
|
||||
@ -3,16 +3,21 @@ set -e
|
||||
# set -x
|
||||
|
||||
PROGRAM=clickhouse-server
|
||||
CLICKHOUSE_USER=${CLICKHOUSE_USER:=clickhouse}
|
||||
CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:=${CLICKHOUSE_USER}}
|
||||
CLICKHOUSE_USER=${CLICKHOUSE_USER:-clickhouse}
|
||||
CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:-${CLICKHOUSE_USER}}
|
||||
# Please note that we don't support paths with whitespaces. This is rather ignorant.
|
||||
CLICKHOUSE_CONFDIR=${CLICKHOUSE_CONFDIR:=/etc/clickhouse-server}
|
||||
CLICKHOUSE_DATADIR=${CLICKHOUSE_DATADIR:=/var/lib/clickhouse}
|
||||
CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR:=/var/log/clickhouse-server}
|
||||
CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin}
|
||||
CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse}
|
||||
CLICKHOUSE_CONFDIR=${CLICKHOUSE_CONFDIR:-/etc/clickhouse-server}
|
||||
CLICKHOUSE_DATADIR=${CLICKHOUSE_DATADIR:-/var/lib/clickhouse}
|
||||
CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR:-/var/log/clickhouse-server}
|
||||
CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:-/usr/bin}
|
||||
CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:-clickhouse}
|
||||
CLICKHOUSE_PIDDIR=/var/run/$PROGRAM
|
||||
|
||||
# Provide clickhouse-keeper
|
||||
KEEPER_CONFDIR=${KEEPER_CONFDIR:-/etc/clickhouse-keeper}
|
||||
KEEPER_DATADIR=${KEEPER_DATADIR:-/var/lib/clickhouse}
|
||||
KEEPER_LOGDIR=${KEEPER_LOGDIR:-/var/log/clickhouse-keeper}
|
||||
|
||||
[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule
|
||||
[ -f /etc/default/clickhouse ] && . /etc/default/clickhouse
|
||||
|
||||
@ -54,4 +59,20 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Setup clickhouse-keeper directories
|
||||
chown -R "${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}" "${KEEPER_CONFDIR}"
|
||||
chmod 0755 "${KEEPER_CONFDIR}"
|
||||
|
||||
if ! [ -d "${KEEPER_DATADIR}" ]; then
|
||||
mkdir -p "${KEEPER_DATADIR}"
|
||||
chown -R "${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}" "${KEEPER_DATADIR}"
|
||||
chmod 0700 "${KEEPER_DATADIR}"
|
||||
fi
|
||||
|
||||
if ! [ -d "${KEEPER_LOGDIR}" ]; then
|
||||
mkdir -p "${KEEPER_LOGDIR}"
|
||||
chown -R "${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}" "${KEEPER_LOGDIR}"
|
||||
chmod 0770 "${KEEPER_LOGDIR}"
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
# package sources should be placed in ${PWD}/root
|
||||
# nfpm should run from the same directory with a config
|
||||
name: "clickhouse-server"
|
||||
description: |
|
||||
Server binary for ClickHouse
|
||||
ClickHouse is a column-oriented database management system
|
||||
that allows generating analytical data reports in real time.
|
||||
This package provides clickhouse common configuration files
|
||||
|
||||
# Common packages config
|
||||
arch: "${DEB_ARCH}" # amd64, arm64
|
||||
platform: "linux"
|
||||
version: "${CLICKHOUSE_VERSION_STRING}"
|
||||
@ -9,24 +16,21 @@ homepage: "https://clickhouse.com"
|
||||
license: "Apache"
|
||||
section: "database"
|
||||
priority: "optional"
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
conflicts:
|
||||
- clickhouse-keeper
|
||||
# Package specific content
|
||||
replaces:
|
||||
- clickhouse-server-common
|
||||
- clickhouse-server-base
|
||||
provides:
|
||||
- clickhouse-keeper
|
||||
- clickhouse-server-common
|
||||
recommends:
|
||||
- libcap2-bin
|
||||
|
||||
maintainer: "ClickHouse Dev Team <packages+linux@clickhouse.com>"
|
||||
description: |
|
||||
Server binary for ClickHouse
|
||||
ClickHouse is a column-oriented database management system
|
||||
that allows generating analytical data reports in real time.
|
||||
This package provides clickhouse common configuration files
|
||||
|
||||
overrides:
|
||||
deb:
|
||||
depends:
|
||||
@ -35,10 +39,6 @@ overrides:
|
||||
depends:
|
||||
- clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING}
|
||||
|
||||
deb:
|
||||
fields:
|
||||
Source: clickhouse
|
||||
|
||||
contents:
|
||||
- src: root/etc/clickhouse-server/config.xml
|
||||
dst: /etc/clickhouse-server/config.xml
|
||||
@ -52,16 +52,25 @@ contents:
|
||||
dst: /lib/systemd/system/clickhouse-server.service
|
||||
- src: root/usr/bin/clickhouse-copier
|
||||
dst: /usr/bin/clickhouse-copier
|
||||
- src: root/usr/bin/clickhouse-report
|
||||
dst: /usr/bin/clickhouse-report
|
||||
- src: root/usr/bin/clickhouse-server
|
||||
dst: /usr/bin/clickhouse-server
|
||||
# clickhouse-keeper part
|
||||
- src: root/etc/clickhouse-keeper/keeper_config.xml
|
||||
dst: /etc/clickhouse-keeper/keeper_config.xml
|
||||
type: config|noreplace
|
||||
- src: clickhouse-keeper.service
|
||||
dst: /lib/systemd/system/clickhouse-keeper.service
|
||||
- src: clickhouse
|
||||
dst: /usr/bin/clickhouse-keeper
|
||||
type: symlink
|
||||
- src: clickhouse
|
||||
dst: /usr/bin/clickhouse-keeper-client
|
||||
type: symlink
|
||||
- src: root/usr/bin/clickhouse-report
|
||||
dst: /usr/bin/clickhouse-report
|
||||
- src: root/usr/bin/clickhouse-server
|
||||
dst: /usr/bin/clickhouse-server
|
||||
- src: clickhouse
|
||||
dst: /usr/bin/clickhouse-keeper-converter
|
||||
type: symlink
|
||||
# docs
|
||||
- src: ../AUTHORS
|
||||
dst: /usr/share/doc/clickhouse-server/AUTHORS
|
||||
|
||||
@ -4,6 +4,16 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
std::vector<UUID> EnabledRolesInfo::getCurrentRoles() const
|
||||
{
|
||||
return std::vector<UUID>{current_roles.begin(), current_roles.end()};
|
||||
}
|
||||
|
||||
std::vector<UUID> EnabledRolesInfo::getEnabledRoles() const
|
||||
{
|
||||
return std::vector<UUID>{enabled_roles.begin(), enabled_roles.end()};
|
||||
}
|
||||
|
||||
Strings EnabledRolesInfo::getCurrentRolesNames() const
|
||||
{
|
||||
Strings result;
|
||||
|
||||
@ -20,6 +20,9 @@ struct EnabledRolesInfo
|
||||
AccessRights access;
|
||||
SettingsProfileElements settings_from_enabled_roles;
|
||||
|
||||
std::vector<UUID> getCurrentRoles() const;
|
||||
std::vector<UUID> getEnabledRoles() const;
|
||||
|
||||
Strings getCurrentRolesNames() const;
|
||||
Strings getEnabledRolesNames() const;
|
||||
|
||||
|
||||
@ -61,7 +61,7 @@ private: // IAccessStorage implementations.
|
||||
bool areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials,
|
||||
const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const;
|
||||
|
||||
mutable std::recursive_mutex mutex;
|
||||
mutable std::recursive_mutex mutex; // Note: Reentrace possible by internal role lookup via access_control
|
||||
AccessControl & access_control;
|
||||
String ldap_server_name;
|
||||
LDAPClient::RoleSearchParamsList role_search_params;
|
||||
|
||||
@ -44,12 +44,12 @@ private:
|
||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||
|
||||
bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) TSA_REQUIRES(mutex);
|
||||
bool removeNoLock(const UUID & id, bool throw_if_not_exists) TSA_REQUIRES(mutex);
|
||||
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) TSA_REQUIRES(mutex);
|
||||
bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
||||
bool removeNoLock(const UUID & id, bool throw_if_not_exists);
|
||||
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
||||
|
||||
void removeAllExceptNoLock(const std::vector<UUID> & ids_to_keep) TSA_REQUIRES(mutex);
|
||||
void removeAllExceptNoLock(const boost::container::flat_set<UUID> & ids_to_keep) TSA_REQUIRES(mutex);
|
||||
void removeAllExceptNoLock(const std::vector<UUID> & ids_to_keep);
|
||||
void removeAllExceptNoLock(const boost::container::flat_set<UUID> & ids_to_keep);
|
||||
|
||||
struct Entry
|
||||
{
|
||||
@ -57,9 +57,9 @@ private:
|
||||
AccessEntityPtr entity;
|
||||
};
|
||||
|
||||
mutable std::mutex mutex;
|
||||
std::unordered_map<UUID, Entry> entries_by_id TSA_GUARDED_BY(mutex); /// We want to search entries both by ID and by the pair of name and type.
|
||||
std::unordered_map<String, Entry *> entries_by_name_and_type[static_cast<size_t>(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex);
|
||||
mutable std::recursive_mutex mutex; // Note: Reentrace possible via LDAPAccessStorage
|
||||
std::unordered_map<UUID, Entry> entries_by_id; /// We want to search entries both by ID and by the pair of name and type.
|
||||
std::unordered_map<String, Entry *> entries_by_name_and_type[static_cast<size_t>(AccessEntityType::MAX)];
|
||||
AccessChangesNotifier & changes_notifier;
|
||||
const bool backup_allowed = false;
|
||||
};
|
||||
|
||||
@ -205,7 +205,7 @@ class IColumn;
|
||||
M(Bool, allow_experimental_inverted_index, false, "If it is set to true, allow to use experimental inverted index.", 0) \
|
||||
\
|
||||
M(UInt64, mysql_max_rows_to_insert, 65536, "The maximum number of rows in MySQL batch insertion of the MySQL storage engine", 0) \
|
||||
M(Bool, use_mysql_types_in_show_columns, false, "Use MySQL converted types when connected via MySQL compatibility for show columns query", 0) \
|
||||
M(Bool, use_mysql_types_in_show_columns, false, "Show MySQL types in SHOW COLUMNS and system.columns", 0) \
|
||||
\
|
||||
M(UInt64, optimize_min_equality_disjunction_chain_length, 3, "The minimum length of the expression `expr = x1 OR ... expr = xN` for optimization ", 0) \
|
||||
\
|
||||
|
||||
@ -14,6 +14,8 @@
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Common/quoteString.h>
|
||||
|
||||
#include <boost/container/flat_set.hpp>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -60,10 +60,12 @@ namespace ErrorCodes
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
AsynchronousInsertQueue::InsertQuery::InsertQuery(const ASTPtr & query_, const Settings & settings_)
|
||||
AsynchronousInsertQueue::InsertQuery::InsertQuery(const ASTPtr & query_, const Settings & settings_, const std::optional<UUID> & user_id_, const std::vector<UUID> & current_roles_)
|
||||
: query(query_->clone())
|
||||
, query_str(queryToString(query))
|
||||
, settings(settings_)
|
||||
, user_id(user_id_)
|
||||
, current_roles(current_roles_)
|
||||
, hash(calculateHash())
|
||||
{
|
||||
}
|
||||
@ -72,6 +74,8 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery(const InsertQuery & other)
|
||||
: query(other.query->clone())
|
||||
, query_str(other.query_str)
|
||||
, settings(other.settings)
|
||||
, user_id(other.user_id)
|
||||
, current_roles(other.current_roles)
|
||||
, hash(other.hash)
|
||||
{
|
||||
}
|
||||
@ -83,6 +87,8 @@ AsynchronousInsertQueue::InsertQuery::operator=(const InsertQuery & other)
|
||||
{
|
||||
query = other.query->clone();
|
||||
query_str = other.query_str;
|
||||
user_id = other.user_id;
|
||||
current_roles = other.current_roles;
|
||||
settings = other.settings;
|
||||
hash = other.hash;
|
||||
}
|
||||
@ -95,6 +101,13 @@ UInt128 AsynchronousInsertQueue::InsertQuery::calculateHash() const
|
||||
SipHash siphash;
|
||||
query->updateTreeHash(siphash);
|
||||
|
||||
if (user_id)
|
||||
{
|
||||
siphash.update(*user_id);
|
||||
for (const auto & current_role : current_roles)
|
||||
siphash.update(current_role);
|
||||
}
|
||||
|
||||
for (const auto & setting : settings.allChanged())
|
||||
{
|
||||
/// We don't consider this setting because it is only for deduplication,
|
||||
@ -110,7 +123,7 @@ UInt128 AsynchronousInsertQueue::InsertQuery::calculateHash() const
|
||||
|
||||
bool AsynchronousInsertQueue::InsertQuery::operator==(const InsertQuery & other) const
|
||||
{
|
||||
return query_str == other.query_str && settings == other.settings;
|
||||
return query_str == other.query_str && user_id == other.user_id && current_roles == other.current_roles && settings == other.settings;
|
||||
}
|
||||
|
||||
AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && query_id_, const String & async_dedup_token_, MemoryTracker * user_memory_tracker_)
|
||||
@ -257,12 +270,9 @@ AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context)
|
||||
}
|
||||
}
|
||||
|
||||
if (auto quota = query_context->getQuota())
|
||||
quota->used(QuotaType::WRITTEN_BYTES, bytes.size());
|
||||
|
||||
auto entry = std::make_shared<InsertData::Entry>(std::move(bytes), query_context->getCurrentQueryId(), settings.insert_deduplication_token, CurrentThread::getUserMemoryTracker());
|
||||
|
||||
InsertQuery key{query, settings};
|
||||
InsertQuery key{query, settings, query_context->getUserID(), query_context->getCurrentRoles()};
|
||||
InsertDataPtr data_to_process;
|
||||
std::future<void> insert_future;
|
||||
|
||||
@ -469,6 +479,11 @@ try
|
||||
/// 'resetParser' doesn't work for parallel parsing.
|
||||
key.settings.set("input_format_parallel_parsing", false);
|
||||
insert_context->makeQueryContext();
|
||||
|
||||
/// Access rights must be checked for the user who executed the initial INSERT query.
|
||||
if (key.user_id)
|
||||
insert_context->setUser(*key.user_id, key.current_roles);
|
||||
|
||||
insert_context->setSettings(key.settings);
|
||||
|
||||
/// Set initial_query_id, because it's used in InterpreterInsertQuery for table lock.
|
||||
|
||||
@ -53,9 +53,11 @@ private:
|
||||
ASTPtr query;
|
||||
String query_str;
|
||||
Settings settings;
|
||||
std::optional<UUID> user_id;
|
||||
std::vector<UUID> current_roles;
|
||||
UInt128 hash;
|
||||
|
||||
InsertQuery(const ASTPtr & query_, const Settings & settings_);
|
||||
InsertQuery(const ASTPtr & query_, const Settings & settings_, const std::optional<UUID> & user_id_, const std::vector<UUID> & current_roles_);
|
||||
InsertQuery(const InsertQuery & other);
|
||||
InsertQuery & operator=(const InsertQuery & other);
|
||||
bool operator==(const InsertQuery & other) const;
|
||||
|
||||
@ -10,6 +10,8 @@
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/TTLCachePolicy.h>
|
||||
#include <Common/formatReadable.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <base/defines.h> /// chassert
|
||||
|
||||
@ -190,7 +192,7 @@ QueryCache::Writer::Writer(
|
||||
if (auto entry = cache.getWithKey(key); entry.has_value() && !IsStale()(entry->key))
|
||||
{
|
||||
skip_insert = true; /// Key already contained in cache and did not expire yet --> don't replace it
|
||||
LOG_TRACE(logger, "Skipped insert (non-stale entry found), query: {}", key.query_string);
|
||||
LOG_TRACE(logger, "Skipped insert because the cache contains a non-stale query result for query {}", doubleQuoteString(key.query_string));
|
||||
}
|
||||
}
|
||||
|
||||
@ -260,16 +262,17 @@ void QueryCache::Writer::finalizeWrite()
|
||||
|
||||
/// Check some reasons why the entry must not be cached:
|
||||
|
||||
if (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - query_start_time) < min_query_runtime)
|
||||
if (auto query_runtime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - query_start_time); query_runtime < min_query_runtime)
|
||||
{
|
||||
LOG_TRACE(logger, "Skipped insert (query not expensive enough), query: {}", key.query_string);
|
||||
LOG_TRACE(logger, "Skipped insert because the query is not expensive enough, query runtime: {} msec (minimum query runtime: {} msec), query: {}",
|
||||
query_runtime.count(), min_query_runtime.count(), doubleQuoteString(key.query_string));
|
||||
return;
|
||||
}
|
||||
|
||||
if (auto entry = cache.getWithKey(key); entry.has_value() && !IsStale()(entry->key))
|
||||
{
|
||||
/// Same check as in ctor because a parallel Writer could have inserted the current key in the meantime
|
||||
LOG_TRACE(logger, "Skipped insert (non-stale entry found), query: {}", key.query_string);
|
||||
LOG_TRACE(logger, "Skipped insert because the cache contains a non-stale query result for query {}", doubleQuoteString(key.query_string));
|
||||
return;
|
||||
}
|
||||
|
||||
@ -352,13 +355,14 @@ void QueryCache::Writer::finalizeWrite()
|
||||
|
||||
if ((new_entry_size_in_bytes > max_entry_size_in_bytes) || (new_entry_size_in_rows > max_entry_size_in_rows))
|
||||
{
|
||||
LOG_TRACE(logger, "Skipped insert (query result too big), new_entry_size_in_bytes: {} ({}), new_entry_size_in_rows: {} ({}), query: {}", new_entry_size_in_bytes, max_entry_size_in_bytes, new_entry_size_in_rows, max_entry_size_in_rows, key.query_string);
|
||||
LOG_TRACE(logger, "Skipped insert because the query result is too big, query result size: {} (maximum size: {}), query result size in rows: {} (maximum size: {}), query: {}",
|
||||
formatReadableSizeWithBinarySuffix(new_entry_size_in_bytes, 0), formatReadableSizeWithBinarySuffix(max_entry_size_in_bytes, 0), new_entry_size_in_rows, max_entry_size_in_rows, doubleQuoteString(key.query_string));
|
||||
return;
|
||||
}
|
||||
|
||||
cache.set(key, query_result);
|
||||
|
||||
LOG_TRACE(logger, "Stored query result, query: {}", key.query_string);
|
||||
LOG_TRACE(logger, "Stored query result of query {}", doubleQuoteString(key.query_string));
|
||||
|
||||
was_finalized = true;
|
||||
}
|
||||
@ -389,7 +393,7 @@ QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guar
|
||||
|
||||
if (!entry.has_value())
|
||||
{
|
||||
LOG_TRACE(logger, "No entry found for query {}", key.query_string);
|
||||
LOG_TRACE(logger, "No query result found for query {}", doubleQuoteString(key.query_string));
|
||||
return;
|
||||
}
|
||||
|
||||
@ -398,13 +402,13 @@ QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guar
|
||||
|
||||
if (!entry_key.is_shared && entry_key.user_name != key.user_name)
|
||||
{
|
||||
LOG_TRACE(logger, "Inaccessible entry found for query {}", key.query_string);
|
||||
LOG_TRACE(logger, "Inaccessible query result found for query {}", doubleQuoteString(key.query_string));
|
||||
return;
|
||||
}
|
||||
|
||||
if (IsStale()(entry_key))
|
||||
{
|
||||
LOG_TRACE(logger, "Stale entry found for query {}", key.query_string);
|
||||
LOG_TRACE(logger, "Stale query result found for query {}", doubleQuoteString(key.query_string));
|
||||
return;
|
||||
}
|
||||
|
||||
@ -442,7 +446,7 @@ QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guar
|
||||
buildSourceFromChunks(entry_key.header, std::move(decompressed_chunks), entry_mapped->totals, entry_mapped->extremes);
|
||||
}
|
||||
|
||||
LOG_TRACE(logger, "Entry found for query {}", key.query_string);
|
||||
LOG_TRACE(logger, "Query result found for query {}", doubleQuoteString(key.query_string));
|
||||
}
|
||||
|
||||
bool QueryCache::Reader::hasCacheEntryForKey() const
|
||||
|
||||
@ -1209,14 +1209,14 @@ void Context::setCurrentRolesDefault()
|
||||
setCurrentRoles(user->granted_roles.findGranted(user->default_roles));
|
||||
}
|
||||
|
||||
boost::container::flat_set<UUID> Context::getCurrentRoles() const
|
||||
std::vector<UUID> Context::getCurrentRoles() const
|
||||
{
|
||||
return getRolesInfo()->current_roles;
|
||||
return getRolesInfo()->getCurrentRoles();
|
||||
}
|
||||
|
||||
boost::container::flat_set<UUID> Context::getEnabledRoles() const
|
||||
std::vector<UUID> Context::getEnabledRoles() const
|
||||
{
|
||||
return getRolesInfo()->enabled_roles;
|
||||
return getRolesInfo()->getEnabledRoles();
|
||||
}
|
||||
|
||||
std::shared_ptr<const EnabledRolesInfo> Context::getRolesInfo() const
|
||||
@ -3418,7 +3418,9 @@ void Context::initializeSystemLogs()
|
||||
/// for example, system.filesystem_cache_log will be triggered by parts loading
|
||||
/// of any other table if it is stored on a disk with cache.
|
||||
callOnce(shared->system_logs_initializer, [&] {
|
||||
shared->system_logs = std::make_unique<SystemLogs>(getGlobalContext(), getConfigRef());
|
||||
auto system_logs = std::make_unique<SystemLogs>(getGlobalContext(), getConfigRef());
|
||||
auto lock = getGlobalLock();
|
||||
shared->system_logs = std::move(system_logs);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -28,7 +28,6 @@
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
@ -571,8 +570,8 @@ public:
|
||||
|
||||
void setCurrentRoles(const std::vector<UUID> & current_roles_);
|
||||
void setCurrentRolesDefault();
|
||||
boost::container::flat_set<UUID> getCurrentRoles() const;
|
||||
boost::container::flat_set<UUID> getEnabledRoles() const;
|
||||
std::vector<UUID> getCurrentRoles() const;
|
||||
std::vector<UUID> getEnabledRoles() const;
|
||||
std::shared_ptr<const EnabledRolesInfo> getRolesInfo() const;
|
||||
|
||||
void setCurrentProfile(const String & profile_name, bool check_constraints = true);
|
||||
|
||||
@ -1098,7 +1098,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
||||
{
|
||||
if (astContainsNonDeterministicFunctions(ast, context) && !settings.query_cache_store_results_of_queries_with_nondeterministic_functions)
|
||||
throw Exception(ErrorCodes::CANNOT_USE_QUERY_CACHE_WITH_NONDETERMINISTIC_FUNCTIONS,
|
||||
"Unable to cache the query result because the query contains a non-deterministic function. Use setting query_cache_store_results_of_queries_with_nondeterministic_functions = 1 to store the query result regardless.");
|
||||
"Unable to cache the query result because the query contains a non-deterministic function. Use setting `query_cache_store_results_of_queries_with_nondeterministic_functions = 1` to cache the query result regardless");
|
||||
|
||||
QueryCache::Key key(
|
||||
ast, res.pipeline.getHeader(),
|
||||
@ -1107,7 +1107,11 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
||||
settings.query_cache_compress_entries);
|
||||
|
||||
const size_t num_query_runs = query_cache->recordQueryRun(key);
|
||||
if (num_query_runs > settings.query_cache_min_query_runs)
|
||||
if (num_query_runs <= settings.query_cache_min_query_runs)
|
||||
{
|
||||
LOG_TRACE(&Poco::Logger::get("QueryCache"), "Skipped insert because the query ran {} times but the minimum required number of query runs to cache the query result is {}", num_query_runs, settings.query_cache_min_query_runs);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto query_cache_writer = std::make_shared<QueryCache::Writer>(query_cache->createWriter(
|
||||
key,
|
||||
|
||||
@ -1263,8 +1263,12 @@ HTTPRequestHandlerFactoryPtr createDynamicHandlerFactory(IServer & server,
|
||||
if (config.has(config_prefix + ".handler.content_type"))
|
||||
content_type_override = config.getString(config_prefix + ".handler.content_type");
|
||||
|
||||
auto factory = std::make_shared<HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>>(
|
||||
server, std::move(query_param_name), std::move(content_type_override));
|
||||
auto creator = [&server, query_param_name, content_type_override] () -> std::unique_ptr<DynamicQueryHandler>
|
||||
{
|
||||
return std::make_unique<DynamicQueryHandler>(server, query_param_name, content_type_override);
|
||||
};
|
||||
|
||||
auto factory = std::make_shared<HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>>(std::move(creator));
|
||||
|
||||
factory->addFiltersFromConfig(config, config_prefix);
|
||||
|
||||
@ -1335,25 +1339,40 @@ HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server,
|
||||
auto regex = getCompiledRegex(url_expression);
|
||||
if (capturingNamedQueryParam(analyze_receive_params, regex))
|
||||
{
|
||||
factory = std::make_shared<HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>>(
|
||||
server,
|
||||
std::move(analyze_receive_params),
|
||||
std::move(predefined_query),
|
||||
std::move(regex),
|
||||
std::move(headers_name_with_regex),
|
||||
std::move(content_type_override));
|
||||
auto creator = [
|
||||
&server,
|
||||
analyze_receive_params,
|
||||
predefined_query,
|
||||
regex,
|
||||
headers_name_with_regex,
|
||||
content_type_override]
|
||||
-> std::unique_ptr<PredefinedQueryHandler>
|
||||
{
|
||||
return std::make_unique<PredefinedQueryHandler>(
|
||||
server, analyze_receive_params, predefined_query, regex,
|
||||
headers_name_with_regex, content_type_override);
|
||||
};
|
||||
factory = std::make_shared<HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>>(std::move(creator));
|
||||
factory->addFiltersFromConfig(config, config_prefix);
|
||||
return factory;
|
||||
}
|
||||
}
|
||||
|
||||
factory = std::make_shared<HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>>(
|
||||
server,
|
||||
std::move(analyze_receive_params),
|
||||
std::move(predefined_query),
|
||||
CompiledRegexPtr{},
|
||||
std::move(headers_name_with_regex),
|
||||
std::move(content_type_override));
|
||||
auto creator = [
|
||||
&server,
|
||||
analyze_receive_params,
|
||||
predefined_query,
|
||||
headers_name_with_regex,
|
||||
content_type_override]
|
||||
-> std::unique_ptr<PredefinedQueryHandler>
|
||||
{
|
||||
return std::make_unique<PredefinedQueryHandler>(
|
||||
server, analyze_receive_params, predefined_query, CompiledRegexPtr{},
|
||||
headers_name_with_regex, content_type_override);
|
||||
};
|
||||
|
||||
factory = std::make_shared<HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>>(std::move(creator));
|
||||
|
||||
factory->addFiltersFromConfig(config, config_prefix);
|
||||
|
||||
return factory;
|
||||
|
||||
@ -119,17 +119,25 @@ HTTPRequestHandlerFactoryPtr createHandlerFactory(IServer & server, const Poco::
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "LOGICAL ERROR: Unknown HTTP handler factory name.");
|
||||
}
|
||||
|
||||
static const auto ping_response_expression = "Ok.\n";
|
||||
static const auto root_response_expression = "config://http_server_default_response";
|
||||
|
||||
void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server)
|
||||
{
|
||||
auto root_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, root_response_expression);
|
||||
auto root_creator = [&server]() -> std::unique_ptr<StaticRequestHandler>
|
||||
{
|
||||
constexpr auto root_response_expression = "config://http_server_default_response";
|
||||
return std::make_unique<StaticRequestHandler>(server, root_response_expression);
|
||||
};
|
||||
auto root_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(std::move(root_creator));
|
||||
root_handler->attachStrictPath("/");
|
||||
root_handler->allowGetAndHeadRequest();
|
||||
factory.addHandler(root_handler);
|
||||
|
||||
auto ping_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, ping_response_expression);
|
||||
auto ping_creator = [&server]() -> std::unique_ptr<StaticRequestHandler>
|
||||
{
|
||||
constexpr auto ping_response_expression = "Ok.\n";
|
||||
return std::make_unique<StaticRequestHandler>(server, ping_response_expression);
|
||||
};
|
||||
auto ping_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(std::move(ping_creator));
|
||||
ping_handler->attachStrictPath("/ping");
|
||||
ping_handler->allowGetAndHeadRequest();
|
||||
factory.addPathToHints("/ping");
|
||||
@ -167,7 +175,11 @@ void addDefaultHandlersFactory(
|
||||
{
|
||||
addCommonDefaultHandlersFactory(factory, server);
|
||||
|
||||
auto query_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>>(server, "query");
|
||||
auto dynamic_creator = [&server] () -> std::unique_ptr<DynamicQueryHandler>
|
||||
{
|
||||
return std::make_unique<DynamicQueryHandler>(server, "query");
|
||||
};
|
||||
auto query_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>>(std::move(dynamic_creator));
|
||||
query_handler->allowPostAndGetParamsAndOptionsRequest();
|
||||
factory.addHandler(query_handler);
|
||||
|
||||
@ -175,8 +187,12 @@ void addDefaultHandlersFactory(
|
||||
/// Otherwise it will be created separately, see createHandlerFactory(...).
|
||||
if (config.has("prometheus") && config.getInt("prometheus.port", 0) == 0)
|
||||
{
|
||||
auto prometheus_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>>(
|
||||
server, PrometheusMetricsWriter(config, "prometheus", async_metrics));
|
||||
PrometheusMetricsWriter writer(config, "prometheus", async_metrics);
|
||||
auto creator = [&server, writer] () -> std::unique_ptr<PrometheusRequestHandler>
|
||||
{
|
||||
return std::make_unique<PrometheusRequestHandler>(server, writer);
|
||||
};
|
||||
auto prometheus_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>>(std::move(creator));
|
||||
prometheus_handler->attachStrictPath(config.getString("prometheus.endpoint", "/metrics"));
|
||||
prometheus_handler->allowGetAndHeadRequest();
|
||||
factory.addHandler(prometheus_handler);
|
||||
|
||||
@ -25,18 +25,17 @@ class HandlingRuleHTTPHandlerFactory : public HTTPRequestHandlerFactory
|
||||
public:
|
||||
using Filter = std::function<bool(const HTTPServerRequest &)>;
|
||||
|
||||
template <typename... TArgs>
|
||||
explicit HandlingRuleHTTPHandlerFactory(TArgs &&... args)
|
||||
using Creator = std::function<std::unique_ptr<TEndpoint>()>;
|
||||
explicit HandlingRuleHTTPHandlerFactory(Creator && creator_)
|
||||
: creator(std::move(creator_))
|
||||
{}
|
||||
|
||||
explicit HandlingRuleHTTPHandlerFactory(IServer & server)
|
||||
{
|
||||
creator = [my_args = std::tuple<TArgs...>(std::forward<TArgs>(args) ...)]()
|
||||
{
|
||||
return std::apply([&](auto && ... endpoint_args)
|
||||
{
|
||||
return std::make_unique<TEndpoint>(std::forward<decltype(endpoint_args)>(endpoint_args)...);
|
||||
}, std::move(my_args));
|
||||
};
|
||||
creator = [&server]() -> std::unique_ptr<TEndpoint> { return std::make_unique<TEndpoint>(server); };
|
||||
}
|
||||
|
||||
|
||||
void addFilter(Filter cur_filter)
|
||||
{
|
||||
Filter prev_filter = filter;
|
||||
|
||||
@ -47,25 +47,31 @@ createPrometheusHandlerFactory(IServer & server,
|
||||
AsynchronousMetrics & async_metrics,
|
||||
const std::string & config_prefix)
|
||||
{
|
||||
auto factory = std::make_shared<HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>>(
|
||||
server, PrometheusMetricsWriter(config, config_prefix + ".handler", async_metrics));
|
||||
PrometheusMetricsWriter writer(config, config_prefix + ".handler", async_metrics);
|
||||
auto creator = [&server, writer]() -> std::unique_ptr<PrometheusRequestHandler>
|
||||
{
|
||||
return std::make_unique<PrometheusRequestHandler>(server, writer);
|
||||
};
|
||||
|
||||
auto factory = std::make_shared<HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>>(std::move(creator));
|
||||
factory->addFiltersFromConfig(config, config_prefix);
|
||||
return factory;
|
||||
}
|
||||
|
||||
HTTPRequestHandlerFactoryPtr
|
||||
createPrometheusMainHandlerFactory(IServer & server,
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
AsynchronousMetrics & async_metrics,
|
||||
const std::string & name)
|
||||
HTTPRequestHandlerFactoryPtr createPrometheusMainHandlerFactory(
|
||||
IServer & server, const Poco::Util::AbstractConfiguration & config, AsynchronousMetrics & async_metrics, const std::string & name)
|
||||
{
|
||||
auto factory = std::make_shared<HTTPRequestHandlerFactoryMain>(name);
|
||||
auto handler = std::make_shared<HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>>(
|
||||
server, PrometheusMetricsWriter(config, "prometheus", async_metrics));
|
||||
PrometheusMetricsWriter writer(config, "prometheus", async_metrics);
|
||||
auto creator = [&server, writer]() -> std::unique_ptr<PrometheusRequestHandler>
|
||||
{
|
||||
return std::make_unique<PrometheusRequestHandler>(server, writer);
|
||||
};
|
||||
|
||||
auto handler = std::make_shared<HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>>(std::move(creator));
|
||||
handler->attachStrictPath(config.getString("prometheus.endpoint", "/metrics"));
|
||||
handler->allowGetAndHeadRequest();
|
||||
factory->addHandler(handler);
|
||||
return factory;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -168,8 +168,13 @@ HTTPRequestHandlerFactoryPtr createStaticHandlerFactory(IServer & server,
|
||||
int status = config.getInt(config_prefix + ".handler.status", 200);
|
||||
std::string response_content = config.getRawString(config_prefix + ".handler.response_content", "Ok.\n");
|
||||
std::string response_content_type = config.getString(config_prefix + ".handler.content_type", "text/plain; charset=UTF-8");
|
||||
auto factory = std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(
|
||||
server, std::move(response_content), std::move(status), std::move(response_content_type));
|
||||
|
||||
auto creator = [&server, response_content, status, response_content_type]() -> std::unique_ptr<StaticRequestHandler>
|
||||
{
|
||||
return std::make_unique<StaticRequestHandler>(server, response_content, status, response_content_type);
|
||||
};
|
||||
|
||||
auto factory = std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(std::move(creator));
|
||||
|
||||
factory->addFiltersFromConfig(config, config_prefix);
|
||||
|
||||
|
||||
@ -132,18 +132,6 @@ protected:
|
||||
|
||||
bool check_access_for_columns = check_access_for_tables && !access->isGranted(AccessType::SHOW_COLUMNS, database_name, table_name);
|
||||
|
||||
auto get_type_name = [this](const IDataType& type) -> std::string
|
||||
{
|
||||
// Check if the use_mysql_types_in_show_columns setting is enabled and client is connected via MySQL protocol
|
||||
if (use_mysql_types && client_info_interface == DB::ClientInfo::Interface::MYSQL)
|
||||
{
|
||||
return type.getSQLCompatibleName();
|
||||
}
|
||||
else
|
||||
{
|
||||
return type.getName();
|
||||
}
|
||||
};
|
||||
size_t position = 0;
|
||||
for (const auto & column : columns)
|
||||
{
|
||||
@ -161,7 +149,7 @@ protected:
|
||||
if (columns_mask[src_index++])
|
||||
res_columns[res_index++]->insert(column.name);
|
||||
if (columns_mask[src_index++])
|
||||
res_columns[res_index++]->insert(get_type_name(*column.type));
|
||||
res_columns[res_index++]->insert(use_mysql_types ? (column.type->getSQLCompatibleName()) : (column.type->getName()));
|
||||
if (columns_mask[src_index++])
|
||||
res_columns[res_index++]->insert(position);
|
||||
|
||||
|
||||
@ -52,7 +52,7 @@ std::vector<size_t> TableFunctionS3::skipAnalysisForArguments(const QueryTreeNod
|
||||
return result;
|
||||
}
|
||||
|
||||
/// This is needed to avoid copy-pase. Because s3Cluster arguments only differ in additional argument (first) - cluster name
|
||||
/// This is needed to avoid copy-paste. Because s3Cluster arguments only differ in additional argument (first) - cluster name
|
||||
void TableFunctionS3::parseArgumentsImpl(ASTs & args, const ContextPtr & context)
|
||||
{
|
||||
if (auto named_collection = tryGetNamedCollectionWithOverrides(args, context))
|
||||
|
||||
@ -434,6 +434,7 @@ class ClickHouseCluster:
|
||||
self.with_net_trics = False
|
||||
self.with_redis = False
|
||||
self.with_cassandra = False
|
||||
self.with_ldap = False
|
||||
self.with_jdbc_bridge = False
|
||||
self.with_nginx = False
|
||||
self.with_hive = False
|
||||
@ -514,6 +515,13 @@ class ClickHouseCluster:
|
||||
self.cassandra_ip = None
|
||||
self.cassandra_id = self.get_instance_docker_id(self.cassandra_host)
|
||||
|
||||
# available when with_ldap == True
|
||||
self.ldap_host = "openldap"
|
||||
self.ldap_ip = None
|
||||
self.ldap_container = None
|
||||
self.ldap_port = 1389
|
||||
self.ldap_id = self.get_instance_docker_id(self.ldap_host)
|
||||
|
||||
# available when with_rabbitmq == True
|
||||
self.rabbitmq_host = "rabbitmq1"
|
||||
self.rabbitmq_ip = None
|
||||
@ -1396,6 +1404,23 @@ class ClickHouseCluster:
|
||||
]
|
||||
return self.base_cassandra_cmd
|
||||
|
||||
def setup_ldap_cmd(self, instance, env_variables, docker_compose_yml_dir):
|
||||
self.with_ldap = True
|
||||
env_variables["LDAP_EXTERNAL_PORT"] = str(self.ldap_port)
|
||||
self.base_cmd.extend(
|
||||
["--file", p.join(docker_compose_yml_dir, "docker_compose_ldap.yml")]
|
||||
)
|
||||
self.base_ldap_cmd = [
|
||||
"docker-compose",
|
||||
"--env-file",
|
||||
instance.env_file,
|
||||
"--project-name",
|
||||
self.project_name,
|
||||
"--file",
|
||||
p.join(docker_compose_yml_dir, "docker_compose_ldap.yml"),
|
||||
]
|
||||
return self.base_ldap_cmd
|
||||
|
||||
def setup_jdbc_bridge_cmd(self, instance, env_variables, docker_compose_yml_dir):
|
||||
self.with_jdbc_bridge = True
|
||||
env_variables["JDBC_DRIVER_LOGS"] = self.jdbc_driver_logs_dir
|
||||
@ -1483,6 +1508,7 @@ class ClickHouseCluster:
|
||||
with_minio=False,
|
||||
with_azurite=False,
|
||||
with_cassandra=False,
|
||||
with_ldap=False,
|
||||
with_jdbc_bridge=False,
|
||||
with_hive=False,
|
||||
with_coredns=False,
|
||||
@ -1583,6 +1609,7 @@ class ClickHouseCluster:
|
||||
with_hive=with_hive,
|
||||
with_coredns=with_coredns,
|
||||
with_cassandra=with_cassandra,
|
||||
with_ldap=with_ldap,
|
||||
allow_analyzer=allow_analyzer,
|
||||
server_bin_path=self.server_bin_path,
|
||||
odbc_bridge_bin_path=self.odbc_bridge_bin_path,
|
||||
@ -1807,6 +1834,11 @@ class ClickHouseCluster:
|
||||
)
|
||||
)
|
||||
|
||||
if with_ldap and not self.with_ldap:
|
||||
cmds.append(
|
||||
self.setup_ldap_cmd(instance, env_variables, docker_compose_yml_dir)
|
||||
)
|
||||
|
||||
if with_jdbc_bridge and not self.with_jdbc_bridge:
|
||||
cmds.append(
|
||||
self.setup_jdbc_bridge_cmd(
|
||||
@ -2489,6 +2521,32 @@ class ClickHouseCluster:
|
||||
|
||||
raise Exception("Can't wait Cassandra to start")
|
||||
|
||||
def wait_ldap_to_start(self, timeout=180):
|
||||
self.ldap_ip = self.get_instance_ip(self.ldap_host)
|
||||
self.ldap_container = self.get_docker_handle(self.ldap_id)
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
logging.info(
|
||||
f"Check LDAP Online {self.ldap_id} {self.ldap_ip} {self.ldap_port}"
|
||||
)
|
||||
self.exec_in_container(
|
||||
self.ldap_id,
|
||||
[
|
||||
"bash",
|
||||
"-c",
|
||||
f"/opt/bitnami/openldap/bin/ldapsearch -x -H ldap://{self.ldap_ip}:{self.ldap_port} -D cn=admin,dc=example,dc=org -w clickhouse -b dc=example,dc=org",
|
||||
],
|
||||
user="root",
|
||||
)
|
||||
logging.info("LDAP Online")
|
||||
return
|
||||
except Exception as ex:
|
||||
logging.warning("Can't connect to LDAP: %s", str(ex))
|
||||
time.sleep(1)
|
||||
|
||||
raise Exception("Can't wait LDAP to start")
|
||||
|
||||
def start(self):
|
||||
pytest_xdist_logging_to_separate_files.setup()
|
||||
logging.info("Running tests in {}".format(self.base_path))
|
||||
@ -2811,6 +2869,11 @@ class ClickHouseCluster:
|
||||
self.up_called = True
|
||||
self.wait_cassandra_to_start()
|
||||
|
||||
if self.with_ldap and self.base_ldap_cmd:
|
||||
subprocess_check_call(self.base_ldap_cmd + ["up", "-d"])
|
||||
self.up_called = True
|
||||
self.wait_ldap_to_start()
|
||||
|
||||
if self.with_jdbc_bridge and self.base_jdbc_bridge_cmd:
|
||||
os.makedirs(self.jdbc_driver_logs_dir)
|
||||
os.chmod(self.jdbc_driver_logs_dir, stat.S_IRWXU | stat.S_IRWXO)
|
||||
@ -3087,6 +3150,7 @@ class ClickHouseInstance:
|
||||
with_hive,
|
||||
with_coredns,
|
||||
with_cassandra,
|
||||
with_ldap,
|
||||
allow_analyzer,
|
||||
server_bin_path,
|
||||
odbc_bridge_bin_path,
|
||||
@ -3170,6 +3234,7 @@ class ClickHouseInstance:
|
||||
self.with_minio = with_minio
|
||||
self.with_azurite = with_azurite
|
||||
self.with_cassandra = with_cassandra
|
||||
self.with_ldap = with_ldap
|
||||
self.with_jdbc_bridge = with_jdbc_bridge
|
||||
self.with_hive = with_hive
|
||||
self.with_coredns = with_coredns
|
||||
|
||||
@ -0,0 +1,22 @@
|
||||
<clickhouse>
|
||||
<ldap_servers>
|
||||
<openldap>
|
||||
<host>openldap</host>
|
||||
<port>1389</port>
|
||||
<bind_dn>cn={user_name},ou=users,dc=example,dc=org</bind_dn>
|
||||
<enable_tls>no</enable_tls>
|
||||
</openldap>
|
||||
</ldap_servers>
|
||||
<user_directories>
|
||||
<ldap>
|
||||
<server>openldap</server>
|
||||
<role_mapping>
|
||||
<base_dn>dc=example,dc=org</base_dn>
|
||||
<scope>subtree</scope>
|
||||
<search_filter>(&(objectClass=groupOfNames)(member={bind_dn}))</search_filter>
|
||||
<attribute>cn</attribute>
|
||||
<prefix>clickhouse-</prefix>
|
||||
</role_mapping>
|
||||
</ldap>
|
||||
</user_directories>
|
||||
</clickhouse>
|
||||
95
tests/integration/test_ldap_external_user_directory/test.py
Normal file
95
tests/integration/test_ldap_external_user_directory/test.py
Normal file
@ -0,0 +1,95 @@
|
||||
import logging
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.test_tools import TSV
|
||||
|
||||
LDAP_ADMIN_BIND_DN = "cn=admin,dc=example,dc=org"
|
||||
LDAP_ADMIN_PASSWORD = "clickhouse"
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
instance = cluster.add_instance(
|
||||
"instance", main_configs=["configs/ldap_with_role_mapping.xml"], with_ldap=True
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def ldap_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
yield cluster
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def add_ldap_group(ldap_cluster, group_cn, member_cn):
|
||||
code, (stdout, stderr) = ldap_cluster.ldap_container.exec_run(
|
||||
[
|
||||
"sh",
|
||||
"-c",
|
||||
"""echo "dn: cn={group_cn},dc=example,dc=org
|
||||
objectClass: top
|
||||
objectClass: groupOfNames
|
||||
member: cn={member_cn},ou=users,dc=example,dc=org" | \
|
||||
ldapadd -H ldap://{host}:{port} -D "{admin_bind_dn}" -x -w {admin_password}
|
||||
""".format(
|
||||
host=ldap_cluster.ldap_host,
|
||||
port=ldap_cluster.ldap_port,
|
||||
admin_bind_dn=LDAP_ADMIN_BIND_DN,
|
||||
admin_password=LDAP_ADMIN_PASSWORD,
|
||||
group_cn=group_cn,
|
||||
member_cn=member_cn,
|
||||
),
|
||||
],
|
||||
demux=True,
|
||||
)
|
||||
logging.debug(
|
||||
f"test_ldap_external_user_directory code:{code} stdout:{stdout}, stderr:{stderr}"
|
||||
)
|
||||
assert code == 0
|
||||
|
||||
|
||||
def test_authentication_pass():
|
||||
assert instance.query(
|
||||
"select currentUser()", user="janedoe", password="qwerty"
|
||||
) == TSV([["janedoe"]])
|
||||
|
||||
|
||||
def test_authentication_fail():
|
||||
# User doesn't exist.
|
||||
assert "doesnotexist: Authentication failed" in instance.query_and_get_error(
|
||||
"SELECT currentUser()", user="doesnotexist"
|
||||
)
|
||||
|
||||
# Wrong password.
|
||||
assert "janedoe: Authentication failed" in instance.query_and_get_error(
|
||||
"SELECT currentUser()", user="janedoe", password="123"
|
||||
)
|
||||
|
||||
|
||||
def test_role_mapping(ldap_cluster):
|
||||
instance.query("CREATE ROLE role_1")
|
||||
instance.query("CREATE ROLE role_2")
|
||||
add_ldap_group(ldap_cluster, group_cn="clickhouse-role_1", member_cn="johndoe")
|
||||
add_ldap_group(ldap_cluster, group_cn="clickhouse-role_2", member_cn="johndoe")
|
||||
|
||||
assert instance.query(
|
||||
"select currentUser()", user="johndoe", password="qwertz"
|
||||
) == TSV([["johndoe"]])
|
||||
|
||||
assert instance.query(
|
||||
"select role_name from system.current_roles ORDER BY role_name",
|
||||
user="johndoe",
|
||||
password="qwertz",
|
||||
) == TSV([["role_1"], ["role_2"]])
|
||||
|
||||
instance.query("CREATE ROLE role_3")
|
||||
add_ldap_group(ldap_cluster, group_cn="clickhouse-role_3", member_cn="johndoe")
|
||||
# Check that non-existing role in ClickHouse is ignored during role update
|
||||
# See https://github.com/ClickHouse/ClickHouse/issues/54318
|
||||
add_ldap_group(ldap_cluster, group_cn="clickhouse-role_4", member_cn="johndoe")
|
||||
|
||||
assert instance.query(
|
||||
"select role_name from system.current_roles ORDER BY role_name",
|
||||
user="johndoe",
|
||||
password="qwertz",
|
||||
) == TSV([["role_1"], ["role_2"], ["role_3"]])
|
||||
@ -16,10 +16,13 @@ ${CLICKHOUSE_CLIENT} -q "CREATE ROLE r02247"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE USER u02247"
|
||||
${CLICKHOUSE_CLIENT} -q "GRANT ALL ON *.* TO r02247"
|
||||
${CLICKHOUSE_CLIENT} -q "GRANT r02247 to u02247"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE QUOTA q02247 FOR INTERVAL 100 YEAR MAX WRITTEN BYTES = 25 TO r02247"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE QUOTA q02247 FOR INTERVAL 100 YEAR MAX WRITTEN BYTES = 30 TO r02247"
|
||||
|
||||
# The value 'qwqw' means about 13 bytes are to be written, so the current quota (30 bytes) gives the ability to write 'qwqw' 2 times.
|
||||
${CLICKHOUSE_CLIENT} --user u02247 --async_insert 1 -q "INSERT INTO written_bytes_02247 VALUES ('qwqw')"
|
||||
#${CLICKHOUSE_CLIENT} --user u02247 -q "SHOW CURRENT QUOTA"
|
||||
${CLICKHOUSE_CLIENT} --user u02247 --async_insert 0 -q "INSERT INTO written_bytes_02247 VALUES ('qwqw')"
|
||||
#${CLICKHOUSE_CLIENT} --user u02247 -q "SHOW CURRENT QUOTA"
|
||||
${CLICKHOUSE_CLIENT} --user u02247 --async_insert 1 -q "INSERT INTO written_bytes_02247 VALUES ('qwqw')" 2>&1 | grep -m1 -o QUOTA_EXCEEDED
|
||||
${CLICKHOUSE_CLIENT} --user u02247 --async_insert 0 -q "INSERT INTO written_bytes_02247 VALUES ('qwqw')" 2>&1 | grep -m1 -o QUOTA_EXCEEDED
|
||||
|
||||
@ -30,7 +33,10 @@ ${CLICKHOUSE_CLIENT} -q "DROP QUOTA q02247"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE QUOTA q02247 FOR INTERVAL 100 YEAR MAX WRITTEN BYTES = 1000 TO r02247"
|
||||
${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE written_bytes_02247"
|
||||
|
||||
# Numbers from 0 to 50 means about 540 bytes are to be written, so the current quota (1000 bytes) is enough to do so.
|
||||
${CLICKHOUSE_CLIENT} --user u02247 -q "INSERT INTO written_bytes_02247 SELECT toString(number) FROM numbers(50)"
|
||||
|
||||
# Numbers from 0 to 100 means about 1090 bytes are to be written, so the current quota (1000 bytes total - 540 bytes already used) is NOT enough to do so.
|
||||
${CLICKHOUSE_CLIENT} --user u02247 -q "INSERT INTO written_bytes_02247 SELECT toString(number) FROM numbers(100)" 2>&1 | grep -m1 -o QUOTA_EXCEEDED
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT written_bytes > 100 FROM system.quotas_usage WHERE quota_name = 'q02247'"
|
||||
|
||||
@ -1,366 +1,148 @@
|
||||
Drop tables if they exist
|
||||
Create tab table
|
||||
Create pseudo-random database name
|
||||
Create tab duplicate table
|
||||
Run MySQL test
|
||||
field type null key default extra
|
||||
aggregate_function AggregateFunction(sum, Int32) NO NULL
|
||||
array_value Array(Int32) NO NULL
|
||||
boolean_value UInt8 NO NULL
|
||||
date32_value Date32 NO NULL
|
||||
date_value Date NO NULL
|
||||
datetime64_value DateTime64(3) NO NULL
|
||||
datetime_value DateTime NO NULL
|
||||
decimal_value Decimal(10, 2) NO NULL
|
||||
enum_value Enum8('apple' = 1, 'banana' = 2, 'orange' = 3) NO NULL
|
||||
fixed_string_value FixedString(10) NO NULL
|
||||
float32 Float32 NO NULL
|
||||
float64 Float64 NO NULL
|
||||
int128 Int128 NO NULL
|
||||
int16 Int16 NO NULL
|
||||
int256 Int256 NO NULL
|
||||
int32 Int32 NO NULL
|
||||
int64 Int64 NO NULL
|
||||
int8 Int8 NO NULL
|
||||
ipv4_value IPv4 NO NULL
|
||||
ipv6_value IPv6 NO NULL
|
||||
json_value Object('json') NO NULL
|
||||
low_cardinality LowCardinality(String) NO NULL
|
||||
low_cardinality_date LowCardinality(DateTime) NO NULL
|
||||
map_value Map(String, Int32) NO NULL
|
||||
nested.nested_int Array(Int32) NO NULL
|
||||
nested.nested_string Array(String) NO NULL
|
||||
nint32 Nullable(Int32) YES NULL
|
||||
nullable_value Nullable(Int32) YES NULL
|
||||
string_value String NO NULL
|
||||
tuple_value Tuple(Int32, String) NO NULL
|
||||
uint128 UInt128 NO NULL
|
||||
uint16 UInt16 NO NULL
|
||||
uint256 UInt256 NO NULL
|
||||
uint32 UInt32 NO NULL
|
||||
uint64 UInt64 NO PRI SOR NULL
|
||||
uint8 UInt8 NO NULL
|
||||
uuid_value UUID NO NULL
|
||||
field type null key default extra
|
||||
aggregate_function TEXT NO NULL
|
||||
array_value TEXT NO NULL
|
||||
boolean_value TINYINT UNSIGNED NO NULL
|
||||
date32_value DATE NO NULL
|
||||
date_value DATE NO NULL
|
||||
datetime64_value DATETIME NO NULL
|
||||
datetime_value DATETIME NO NULL
|
||||
decimal_value DECIMAL(10, 2) NO NULL
|
||||
enum_value ENUM('apple', 'banana', 'orange') NO NULL
|
||||
fixed_string_value TEXT NO NULL
|
||||
float32 FLOAT NO NULL
|
||||
float64 DOUBLE NO NULL
|
||||
int128 TEXT NO NULL
|
||||
int16 SMALLINT NO NULL
|
||||
int256 TEXT NO NULL
|
||||
int32 INTEGER NO NULL
|
||||
int64 BIGINT NO NULL
|
||||
int8 TINYINT NO NULL
|
||||
ipv4_value TEXT NO NULL
|
||||
ipv6_value TEXT NO NULL
|
||||
json_value JSON NO NULL
|
||||
low_cardinality BLOB NO NULL
|
||||
low_cardinality_date DATETIME NO NULL
|
||||
map_value JSON NO NULL
|
||||
nested.nested_int TEXT NO NULL
|
||||
nested.nested_string TEXT NO NULL
|
||||
nint32 INTEGER NO NULL
|
||||
nullable_value INTEGER NO NULL
|
||||
string_value BLOB NO NULL
|
||||
tuple_value JSON NO NULL
|
||||
uint128 TEXT NO NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL
|
||||
uint256 TEXT NO NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL
|
||||
uuid_value CHAR NO NULL
|
||||
field type null key default extra
|
||||
aggregate_function TEXT NO NULL
|
||||
array_value TEXT NO NULL
|
||||
boolean_value TINYINT UNSIGNED NO NULL
|
||||
date32_value DATE NO NULL
|
||||
date_value DATE NO NULL
|
||||
datetime64_value DATETIME NO NULL
|
||||
datetime_value DATETIME NO NULL
|
||||
decimal_value DECIMAL(10, 2) NO NULL
|
||||
enum_value ENUM('apple', 'banana', 'orange') NO NULL
|
||||
fixed_string_value TEXT NO NULL
|
||||
float32 FLOAT NO NULL
|
||||
float64 DOUBLE NO NULL
|
||||
int128 TEXT NO NULL
|
||||
int16 SMALLINT NO NULL
|
||||
int256 TEXT NO NULL
|
||||
int32 INTEGER NO NULL
|
||||
int64 BIGINT NO NULL
|
||||
int8 TINYINT NO NULL
|
||||
ipv4_value TEXT NO NULL
|
||||
ipv6_value TEXT NO NULL
|
||||
json_value JSON NO NULL
|
||||
low_cardinality BLOB NO NULL
|
||||
low_cardinality_date DATETIME NO NULL
|
||||
map_value JSON NO NULL
|
||||
nested.nested_int TEXT NO NULL
|
||||
nested.nested_string TEXT NO NULL
|
||||
nint32 INTEGER NO NULL
|
||||
nullable_value INTEGER NO NULL
|
||||
string_value BLOB NO NULL
|
||||
tuple_value JSON NO NULL
|
||||
uint128 TEXT NO NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL
|
||||
uint256 TEXT NO NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL
|
||||
uuid_value CHAR NO NULL
|
||||
field type null key default extra collation comment privileges
|
||||
aggregate_function TEXT NO NULL NULL
|
||||
array_value TEXT NO NULL NULL
|
||||
boolean_value TINYINT UNSIGNED NO NULL NULL
|
||||
date32_value DATE NO NULL NULL
|
||||
date_value DATE NO NULL NULL
|
||||
datetime64_value DATETIME NO NULL NULL
|
||||
datetime_value DATETIME NO NULL NULL
|
||||
decimal_value DECIMAL(10, 2) NO NULL NULL
|
||||
enum_value ENUM('apple', 'banana', 'orange') NO NULL NULL
|
||||
fixed_string_value TEXT NO NULL NULL
|
||||
float32 FLOAT NO NULL NULL
|
||||
float64 DOUBLE NO NULL NULL
|
||||
int128 TEXT NO NULL NULL
|
||||
int16 SMALLINT NO NULL NULL
|
||||
int256 TEXT NO NULL NULL
|
||||
int32 INTEGER NO NULL NULL
|
||||
int64 BIGINT NO NULL NULL
|
||||
int8 TINYINT NO NULL NULL
|
||||
ipv4_value TEXT NO NULL NULL
|
||||
ipv6_value TEXT NO NULL NULL
|
||||
json_value JSON NO NULL NULL
|
||||
low_cardinality BLOB NO NULL NULL
|
||||
low_cardinality_date DATETIME NO NULL NULL
|
||||
map_value JSON NO NULL NULL
|
||||
nested.nested_int TEXT NO NULL NULL
|
||||
nested.nested_string TEXT NO NULL NULL
|
||||
nint32 INTEGER NO NULL NULL
|
||||
nullable_value INTEGER NO NULL NULL
|
||||
string_value BLOB NO NULL NULL
|
||||
tuple_value JSON NO NULL NULL
|
||||
uint128 TEXT NO NULL NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL NULL
|
||||
uint256 TEXT NO NULL NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL NULL
|
||||
uuid_value CHAR NO NULL NULL
|
||||
field type null key default extra
|
||||
int128 TEXT NO NULL
|
||||
int16 SMALLINT NO NULL
|
||||
int256 TEXT NO NULL
|
||||
int32 INTEGER NO NULL
|
||||
int64 BIGINT NO NULL
|
||||
int8 TINYINT NO NULL
|
||||
nested.nested_int TEXT NO NULL
|
||||
nint32 INTEGER NO NULL
|
||||
uint128 TEXT NO NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL
|
||||
uint256 TEXT NO NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL
|
||||
field type null key default extra
|
||||
aggregate_function TEXT NO NULL
|
||||
array_value TEXT NO NULL
|
||||
boolean_value TINYINT UNSIGNED NO NULL
|
||||
date32_value DATE NO NULL
|
||||
date_value DATE NO NULL
|
||||
datetime64_value DATETIME NO NULL
|
||||
datetime_value DATETIME NO NULL
|
||||
decimal_value DECIMAL(10, 2) NO NULL
|
||||
enum_value ENUM('apple', 'banana', 'orange') NO NULL
|
||||
fixed_string_value TEXT NO NULL
|
||||
float32 FLOAT NO NULL
|
||||
float64 DOUBLE NO NULL
|
||||
ipv4_value TEXT NO NULL
|
||||
ipv6_value TEXT NO NULL
|
||||
json_value JSON NO NULL
|
||||
low_cardinality BLOB NO NULL
|
||||
low_cardinality_date DATETIME NO NULL
|
||||
map_value JSON NO NULL
|
||||
nested.nested_string TEXT NO NULL
|
||||
nullable_value INTEGER NO NULL
|
||||
string_value BLOB NO NULL
|
||||
tuple_value JSON NO NULL
|
||||
uuid_value CHAR NO NULL
|
||||
field type null key default extra
|
||||
int128 TEXT NO NULL
|
||||
int16 SMALLINT NO NULL
|
||||
int256 TEXT NO NULL
|
||||
int32 INTEGER NO NULL
|
||||
int64 BIGINT NO NULL
|
||||
int8 TINYINT NO NULL
|
||||
nested.nested_int TEXT NO NULL
|
||||
nint32 INTEGER NO NULL
|
||||
uint128 TEXT NO NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL
|
||||
uint256 TEXT NO NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL
|
||||
field type null key default extra
|
||||
aggregate_function TEXT NO NULL
|
||||
array_value TEXT NO NULL
|
||||
boolean_value TINYINT UNSIGNED NO NULL
|
||||
date32_value DATE NO NULL
|
||||
date_value DATE NO NULL
|
||||
datetime64_value DATETIME NO NULL
|
||||
datetime_value DATETIME NO NULL
|
||||
decimal_value DECIMAL(10, 2) NO NULL
|
||||
enum_value ENUM('apple', 'banana', 'orange') NO NULL
|
||||
fixed_string_value TEXT NO NULL
|
||||
float32 FLOAT NO NULL
|
||||
float64 DOUBLE NO NULL
|
||||
ipv4_value TEXT NO NULL
|
||||
ipv6_value TEXT NO NULL
|
||||
json_value JSON NO NULL
|
||||
low_cardinality BLOB NO NULL
|
||||
low_cardinality_date DATETIME NO NULL
|
||||
map_value JSON NO NULL
|
||||
nested.nested_string TEXT NO NULL
|
||||
nullable_value INTEGER NO NULL
|
||||
string_value BLOB NO NULL
|
||||
tuple_value JSON NO NULL
|
||||
uuid_value CHAR NO NULL
|
||||
field type null key default extra
|
||||
int128 TEXT NO NULL
|
||||
int16 SMALLINT NO NULL
|
||||
int256 TEXT NO NULL
|
||||
int32 INTEGER NO NULL
|
||||
int64 BIGINT NO NULL
|
||||
int8 TINYINT NO NULL
|
||||
nested.nested_int TEXT NO NULL
|
||||
nint32 INTEGER NO NULL
|
||||
uint128 TEXT NO NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL
|
||||
uint256 TEXT NO NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL
|
||||
field type null key default extra
|
||||
aggregate_function TEXT NO NULL
|
||||
field type null key default extra
|
||||
aggregate_function TEXT NO NULL
|
||||
array_value TEXT NO NULL
|
||||
boolean_value TINYINT UNSIGNED NO NULL
|
||||
date32_value DATE NO NULL
|
||||
date_value DATE NO NULL
|
||||
datetime64_value DATETIME NO NULL
|
||||
datetime_value DATETIME NO NULL
|
||||
decimal_value DECIMAL(10, 2) NO NULL
|
||||
enum_value ENUM('apple', 'banana', 'orange') NO NULL
|
||||
fixed_string_value TEXT NO NULL
|
||||
float32 FLOAT NO NULL
|
||||
float64 DOUBLE NO NULL
|
||||
int128 TEXT NO NULL
|
||||
int16 SMALLINT NO NULL
|
||||
int256 TEXT NO NULL
|
||||
int32 INTEGER NO NULL
|
||||
int64 BIGINT NO NULL
|
||||
int8 TINYINT NO NULL
|
||||
ipv4_value TEXT NO NULL
|
||||
ipv6_value TEXT NO NULL
|
||||
json_value JSON NO NULL
|
||||
low_cardinality BLOB NO NULL
|
||||
low_cardinality_date DATETIME NO NULL
|
||||
map_value JSON NO NULL
|
||||
nested.nested_int TEXT NO NULL
|
||||
nested.nested_string TEXT NO NULL
|
||||
nint32 INTEGER NO NULL
|
||||
nullable_value INTEGER NO NULL
|
||||
string_value BLOB NO NULL
|
||||
tuple_value JSON NO NULL
|
||||
uint128 TEXT NO NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL
|
||||
uint256 TEXT NO NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL
|
||||
uuid_value CHAR NO NULL
|
||||
field type null key default extra
|
||||
aggregate_function TEXT NO NULL
|
||||
array_value TEXT NO NULL
|
||||
boolean_value TINYINT UNSIGNED NO NULL
|
||||
date32_value DATE NO NULL
|
||||
date_value DATE NO NULL
|
||||
datetime64_value DATETIME NO NULL
|
||||
datetime_value DATETIME NO NULL
|
||||
decimal_value DECIMAL(10, 2) NO NULL
|
||||
enum_value ENUM('apple', 'banana', 'orange') NO NULL
|
||||
fixed_string_value TEXT NO NULL
|
||||
float32 FLOAT NO NULL
|
||||
float64 DOUBLE NO NULL
|
||||
int128 TEXT NO NULL
|
||||
int16 SMALLINT NO NULL
|
||||
int256 TEXT NO NULL
|
||||
int32 INTEGER NO NULL
|
||||
int64 BIGINT NO NULL
|
||||
int8 TINYINT NO NULL
|
||||
ipv4_value TEXT NO NULL
|
||||
ipv6_value TEXT NO NULL
|
||||
json_value JSON NO NULL
|
||||
low_cardinality BLOB NO NULL
|
||||
low_cardinality_date DATETIME NO NULL
|
||||
map_value JSON NO NULL
|
||||
nested.nested_int TEXT NO NULL
|
||||
nested.nested_string TEXT NO NULL
|
||||
nint32 INTEGER NO NULL
|
||||
nullable_value INTEGER NO NULL
|
||||
string_value BLOB NO NULL
|
||||
tuple_value JSON NO NULL
|
||||
uint128 TEXT NO NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL
|
||||
uint256 TEXT NO NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL
|
||||
uuid_value CHAR NO NULL
|
||||
field type null key default extra
|
||||
aggregate_function TEXT NO NULL
|
||||
array_value TEXT NO NULL
|
||||
boolean_value TINYINT UNSIGNED NO NULL
|
||||
date32_value DATE NO NULL
|
||||
date_value DATE NO NULL
|
||||
datetime64_value DATETIME NO NULL
|
||||
datetime_value DATETIME NO NULL
|
||||
decimal_value DECIMAL(10, 2) NO NULL
|
||||
enum_value ENUM('apple', 'banana', 'orange') NO NULL
|
||||
fixed_string_value TEXT NO NULL
|
||||
float32 FLOAT NO NULL
|
||||
float64 DOUBLE NO NULL
|
||||
int128 TEXT NO NULL
|
||||
int16 SMALLINT NO NULL
|
||||
int256 TEXT NO NULL
|
||||
int32 INTEGER NO NULL
|
||||
int64 BIGINT NO NULL
|
||||
int8 TINYINT NO NULL
|
||||
ipv4_value TEXT NO NULL
|
||||
ipv6_value TEXT NO NULL
|
||||
json_value JSON NO NULL
|
||||
low_cardinality BLOB NO NULL
|
||||
low_cardinality_date DATETIME NO NULL
|
||||
map_value JSON NO NULL
|
||||
nested.nested_int TEXT NO NULL
|
||||
nested.nested_string TEXT NO NULL
|
||||
nint32 INTEGER NO NULL
|
||||
nullable_value INTEGER NO NULL
|
||||
string_value BLOB NO NULL
|
||||
tuple_value JSON NO NULL
|
||||
uint128 TEXT NO NULL
|
||||
uint16 SMALLINT UNSIGNED NO NULL
|
||||
uint256 TEXT NO NULL
|
||||
uint32 INTEGER UNSIGNED NO NULL
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR NULL
|
||||
uint8 TINYINT UNSIGNED NO NULL
|
||||
uuid_value CHAR NO NULL
|
||||
aggregate_function AggregateFunction(sum, Int32) NO \N
|
||||
array_value Array(Int32) NO \N
|
||||
boolean_value UInt8 NO \N
|
||||
date32_value Date32 NO \N
|
||||
date_value Date NO \N
|
||||
datetime64_value DateTime64(3) NO \N
|
||||
datetime_value DateTime NO \N
|
||||
decimal_value Decimal(10, 2) NO \N
|
||||
enum_value Enum8(\'apple\' = 1, \'banana\' = 2, \'orange\' = 3) NO \N
|
||||
fixed_string_value FixedString(10) NO \N
|
||||
float32 Float32 NO \N
|
||||
float64 Float64 NO \N
|
||||
int128 Int128 NO \N
|
||||
int16 Int16 NO \N
|
||||
int256 Int256 NO \N
|
||||
int32 Int32 NO \N
|
||||
int64 Int64 NO \N
|
||||
int8 Int8 NO \N
|
||||
ipv4_value IPv4 NO \N
|
||||
ipv6_value IPv6 NO \N
|
||||
json_value Object(\'json\') NO \N
|
||||
low_cardinality LowCardinality(String) NO \N
|
||||
low_cardinality_date LowCardinality(DateTime) NO \N
|
||||
map_value Map(String, Int32) NO \N
|
||||
nested.nested_int Array(Int32) NO \N
|
||||
nested.nested_string Array(String) NO \N
|
||||
nint32 Nullable(Int32) YES \N
|
||||
nullable_value Nullable(Int32) YES \N
|
||||
string_value String NO \N
|
||||
tuple_value Tuple(Int32, String) NO \N
|
||||
uint128 UInt128 NO \N
|
||||
uint16 UInt16 NO \N
|
||||
uint256 UInt256 NO \N
|
||||
uint32 UInt32 NO \N
|
||||
uint64 UInt64 NO PRI SOR \N
|
||||
uint8 UInt8 NO \N
|
||||
uuid_value UUID NO \N
|
||||
aggregate_function TEXT NO \N
|
||||
array_value TEXT NO \N
|
||||
boolean_value TINYINT UNSIGNED NO \N
|
||||
date32_value DATE NO \N
|
||||
date_value DATE NO \N
|
||||
datetime64_value DATETIME NO \N
|
||||
datetime_value DATETIME NO \N
|
||||
decimal_value DECIMAL(10, 2) NO \N
|
||||
enum_value ENUM(\'apple\', \'banana\', \'orange\') NO \N
|
||||
fixed_string_value TEXT NO \N
|
||||
float32 FLOAT NO \N
|
||||
float64 DOUBLE NO \N
|
||||
int128 TEXT NO \N
|
||||
int16 SMALLINT NO \N
|
||||
int256 TEXT NO \N
|
||||
int32 INTEGER NO \N
|
||||
int64 BIGINT NO \N
|
||||
int8 TINYINT NO \N
|
||||
ipv4_value TEXT NO \N
|
||||
ipv6_value TEXT NO \N
|
||||
json_value JSON NO \N
|
||||
low_cardinality BLOB NO \N
|
||||
low_cardinality_date DATETIME NO \N
|
||||
map_value JSON NO \N
|
||||
nested.nested_int TEXT NO \N
|
||||
nested.nested_string TEXT NO \N
|
||||
nint32 INTEGER NO \N
|
||||
nullable_value INTEGER NO \N
|
||||
string_value BLOB NO \N
|
||||
tuple_value JSON NO \N
|
||||
uint128 TEXT NO \N
|
||||
uint16 SMALLINT UNSIGNED NO \N
|
||||
uint256 TEXT NO \N
|
||||
uint32 INTEGER UNSIGNED NO \N
|
||||
uint64 BIGINT UNSIGNED NO PRI SOR \N
|
||||
uint8 TINYINT UNSIGNED NO \N
|
||||
uuid_value CHAR NO \N
|
||||
uint8 UInt8
|
||||
uint16 UInt16
|
||||
uint32 UInt32
|
||||
uint64 UInt64
|
||||
uint128 UInt128
|
||||
uint256 UInt256
|
||||
int8 Int8
|
||||
int16 Int16
|
||||
int32 Int32
|
||||
int64 Int64
|
||||
int128 Int128
|
||||
int256 Int256
|
||||
nint32 Nullable(Int32)
|
||||
float32 Float32
|
||||
float64 Float64
|
||||
decimal_value Decimal(10, 2)
|
||||
boolean_value UInt8
|
||||
string_value String
|
||||
fixed_string_value FixedString(10)
|
||||
date_value Date
|
||||
date32_value Date32
|
||||
datetime_value DateTime
|
||||
datetime64_value DateTime64(3)
|
||||
json_value Object(\'json\')
|
||||
uuid_value UUID
|
||||
enum_value Enum8(\'apple\' = 1, \'banana\' = 2, \'orange\' = 3)
|
||||
low_cardinality LowCardinality(String)
|
||||
low_cardinality_date LowCardinality(DateTime)
|
||||
aggregate_function AggregateFunction(sum, Int32)
|
||||
array_value Array(Int32)
|
||||
map_value Map(String, Int32)
|
||||
tuple_value Tuple(Int32, String)
|
||||
nullable_value Nullable(Int32)
|
||||
ipv4_value IPv4
|
||||
ipv6_value IPv6
|
||||
nested.nested_int Array(Int32)
|
||||
nested.nested_string Array(String)
|
||||
uint8 TINYINT UNSIGNED
|
||||
uint16 SMALLINT UNSIGNED
|
||||
uint32 INTEGER UNSIGNED
|
||||
uint64 BIGINT UNSIGNED
|
||||
uint128 TEXT
|
||||
uint256 TEXT
|
||||
int8 TINYINT
|
||||
int16 SMALLINT
|
||||
int32 INTEGER
|
||||
int64 BIGINT
|
||||
int128 TEXT
|
||||
int256 TEXT
|
||||
nint32 INTEGER
|
||||
float32 FLOAT
|
||||
float64 DOUBLE
|
||||
decimal_value DECIMAL(10, 2)
|
||||
boolean_value TINYINT UNSIGNED
|
||||
string_value BLOB
|
||||
fixed_string_value TEXT
|
||||
date_value DATE
|
||||
date32_value DATE
|
||||
datetime_value DATETIME
|
||||
datetime64_value DATETIME
|
||||
json_value JSON
|
||||
uuid_value CHAR
|
||||
enum_value ENUM(\'apple\', \'banana\', \'orange\')
|
||||
low_cardinality BLOB
|
||||
low_cardinality_date DATETIME
|
||||
aggregate_function TEXT
|
||||
array_value TEXT
|
||||
map_value JSON
|
||||
tuple_value JSON
|
||||
nullable_value INTEGER
|
||||
ipv4_value TEXT
|
||||
ipv6_value TEXT
|
||||
nested.nested_int TEXT
|
||||
nested.nested_string TEXT
|
||||
|
||||
@ -1,153 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, no-parallel
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# This script tests the MySQL compatibility of the SHOW COLUMNS command in ClickHouse
|
||||
USER="default"
|
||||
PASSWORD=""
|
||||
HOST="127.0.0.1"
|
||||
PORT=9004
|
||||
|
||||
# First run the clickhouse test to create the ClickHouse Tables
|
||||
|
||||
echo "Drop tables if they exist"
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS tab"
|
||||
${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS database_123456789abcdef"
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS database_123456789abcdef.tab"
|
||||
|
||||
echo "Create tab table "
|
||||
${CLICKHOUSE_CLIENT} -n -q "
|
||||
SET allow_suspicious_low_cardinality_types=1;
|
||||
SET allow_experimental_object_type=1;
|
||||
CREATE TABLE tab
|
||||
(
|
||||
uint8 UInt8,
|
||||
uint16 UInt16,
|
||||
uint32 UInt32,
|
||||
uint64 UInt64,
|
||||
uint128 UInt128,
|
||||
uint256 UInt256,
|
||||
int8 Int8,
|
||||
int16 Int16,
|
||||
int32 Int32,
|
||||
int64 Int64,
|
||||
int128 Int128,
|
||||
int256 Int256,
|
||||
nint32 Nullable(Int32),
|
||||
float32 Float32,
|
||||
float64 Float64,
|
||||
decimal_value Decimal(10, 2),
|
||||
boolean_value UInt8,
|
||||
string_value String,
|
||||
fixed_string_value FixedString(10),
|
||||
date_value Date,
|
||||
date32_value Date32,
|
||||
datetime_value DateTime,
|
||||
datetime64_value DateTime64(3),
|
||||
json_value JSON,
|
||||
uuid_value UUID,
|
||||
enum_value Enum8('apple' = 1, 'banana' = 2, 'orange' = 3),
|
||||
low_cardinality LowCardinality(String),
|
||||
low_cardinality_date LowCardinality(DateTime),
|
||||
aggregate_function AggregateFunction(sum, Int32),
|
||||
array_value Array(Int32),
|
||||
map_value Map(String, Int32),
|
||||
tuple_value Tuple(Int32, String),
|
||||
nullable_value Nullable(Int32),
|
||||
ipv4_value IPv4,
|
||||
ipv6_value IPv6,
|
||||
nested Nested
|
||||
(
|
||||
nested_int Int32,
|
||||
nested_string String
|
||||
)
|
||||
) ENGINE = MergeTree
|
||||
ORDER BY uint64;
|
||||
"
|
||||
|
||||
|
||||
echo "Create pseudo-random database name"
|
||||
${CLICKHOUSE_CLIENT} --query "CREATE DATABASE database_123456789abcdef;"
|
||||
|
||||
echo "Create tab duplicate table"
|
||||
${CLICKHOUSE_CLIENT} -n -q "
|
||||
SET allow_suspicious_low_cardinality_types=1;
|
||||
SET allow_experimental_object_type =1;
|
||||
CREATE TABLE database_123456789abcdef.tab
|
||||
(
|
||||
uint8 UInt8,
|
||||
uint16 UInt16,
|
||||
uint32 UInt32,
|
||||
uint64 UInt64,
|
||||
uint128 UInt128,
|
||||
uint256 UInt256,
|
||||
int8 Int8,
|
||||
int16 Int16,
|
||||
int32 Int32,
|
||||
int64 Int64,
|
||||
int128 Int128,
|
||||
int256 Int256,
|
||||
nint32 Nullable(Int32),
|
||||
float32 Float32,
|
||||
float64 Float64,
|
||||
decimal_value Decimal(10, 2),
|
||||
boolean_value UInt8,
|
||||
string_value String,
|
||||
fixed_string_value FixedString(10),
|
||||
date_value Date,
|
||||
date32_value Date32,
|
||||
datetime_value DateTime,
|
||||
datetime64_value DateTime64(3),
|
||||
json_value JSON,
|
||||
uuid_value UUID,
|
||||
enum_value Enum8('apple' = 1, 'banana' = 2, 'orange' = 3),
|
||||
low_cardinality LowCardinality(String),
|
||||
low_cardinality_date LowCardinality(DateTime),
|
||||
aggregate_function AggregateFunction(sum, Int32),
|
||||
array_value Array(Int32),
|
||||
map_value Map(String, Int32),
|
||||
tuple_value Tuple(Int32, String),
|
||||
nullable_value Nullable(Int32),
|
||||
ipv4_value IPv4,
|
||||
ipv6_value IPv6,
|
||||
nested Nested
|
||||
(
|
||||
nested_int Int32,
|
||||
nested_string String
|
||||
)
|
||||
) ENGINE = MergeTree
|
||||
ORDER BY uint64;
|
||||
"
|
||||
|
||||
# Write sql to temp file
|
||||
TEMP_FILE=$(mktemp)
|
||||
|
||||
cat <<EOT > $TEMP_FILE
|
||||
SHOW COLUMNS FROM tab;
|
||||
SET use_mysql_types_in_show_columns=1;
|
||||
SHOW COLUMNS FROM tab;
|
||||
SHOW EXTENDED COLUMNS FROM tab;
|
||||
SHOW FULL COLUMNS FROM tab;
|
||||
SHOW COLUMNS FROM tab LIKE '%int%';
|
||||
SHOW COLUMNS FROM tab NOT LIKE '%int%';
|
||||
SHOW COLUMNS FROM tab ILIKE '%INT%';
|
||||
SHOW COLUMNS FROM tab NOT ILIKE '%INT%';
|
||||
SHOW COLUMNS FROM tab WHERE field LIKE '%int%';
|
||||
SHOW COLUMNS FROM tab LIMIT 1;
|
||||
SHOW COLUMNS FROM tab;
|
||||
SHOW COLUMNS FROM tab FROM database_123456789abcdef;
|
||||
SHOW COLUMNS FROM database_123456789abcdef.tab;
|
||||
DROP DATABASE database_123456789abcdef;
|
||||
DROP TABLE tab;
|
||||
EOT
|
||||
|
||||
# Now run the MySQL test script on the ClickHouse DB
|
||||
echo "Run MySQL test"
|
||||
MYSQL_PWD=$PASSWORD ${MYSQL_CLIENT} --user="$USER" --host="$HOST" --port="$PORT" < $TEMP_FILE
|
||||
|
||||
# Clean up the temp file
|
||||
rm $TEMP_FILE
|
||||
|
||||
@ -0,0 +1,63 @@
|
||||
-- Tags: no-fasttest, no-parallel
|
||||
-- no-fasttest: json type needs rapidjson library
|
||||
-- no-parallel: can't provide currentDatabase() to SHOW COLUMNS
|
||||
|
||||
-- Tests setting 'use_mysql_types_in_show_columns' in SHOW COLUMNS and SELECTs on system.columns
|
||||
|
||||
DROP TABLE IF EXISTS tab;
|
||||
|
||||
SET allow_suspicious_low_cardinality_types=1;
|
||||
SET allow_experimental_object_type=1;
|
||||
|
||||
CREATE TABLE tab
|
||||
(
|
||||
uint8 UInt8,
|
||||
uint16 UInt16,
|
||||
uint32 UInt32,
|
||||
uint64 UInt64,
|
||||
uint128 UInt128,
|
||||
uint256 UInt256,
|
||||
int8 Int8,
|
||||
int16 Int16,
|
||||
int32 Int32,
|
||||
int64 Int64,
|
||||
int128 Int128,
|
||||
int256 Int256,
|
||||
nint32 Nullable(Int32),
|
||||
float32 Float32,
|
||||
float64 Float64,
|
||||
decimal_value Decimal(10, 2),
|
||||
boolean_value UInt8,
|
||||
string_value String,
|
||||
fixed_string_value FixedString(10),
|
||||
date_value Date,
|
||||
date32_value Date32,
|
||||
datetime_value DateTime,
|
||||
datetime64_value DateTime64(3),
|
||||
json_value JSON,
|
||||
uuid_value UUID,
|
||||
enum_value Enum8('apple' = 1, 'banana' = 2, 'orange' = 3),
|
||||
low_cardinality LowCardinality(String),
|
||||
low_cardinality_date LowCardinality(DateTime),
|
||||
aggregate_function AggregateFunction(sum, Int32),
|
||||
array_value Array(Int32),
|
||||
map_value Map(String, Int32),
|
||||
tuple_value Tuple(Int32, String),
|
||||
nullable_value Nullable(Int32),
|
||||
ipv4_value IPv4,
|
||||
ipv6_value IPv6,
|
||||
nested Nested
|
||||
(
|
||||
nested_int Int32,
|
||||
nested_string String
|
||||
)
|
||||
) ENGINE = MergeTree
|
||||
ORDER BY uint64;
|
||||
|
||||
SHOW COLUMNS FROM tab SETTINGS use_mysql_types_in_show_columns = 0;
|
||||
SHOW COLUMNS FROM tab SETTINGS use_mysql_types_in_show_columns = 1;
|
||||
|
||||
SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = 'tab' SETTINGS use_mysql_types_in_show_columns = 0;
|
||||
SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = 'tab' SETTINGS use_mysql_types_in_show_columns = 1;
|
||||
|
||||
DROP TABLE tab;
|
||||
@ -0,0 +1,6 @@
|
||||
1 value_1
|
||||
2 value_2
|
||||
Not enough privileges
|
||||
Not enough privileges
|
||||
5 value_5
|
||||
6 value_6
|
||||
58
tests/queries/0_stateless/02885_async_insert_access_check_for_defaults.sh
Executable file
58
tests/queries/0_stateless/02885_async_insert_access_check_for_defaults.sh
Executable file
@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
restricted_user=${CLICKHOUSE_DATABASE}_restricted_user_$RANDOM$RANDOM
|
||||
|
||||
# The 'restricted_user' will not have access to the dictionary 'dict',
|
||||
# so they shouldn't be able to insert to a table with DEFAULT dictGet(dict, ...)
|
||||
|
||||
$CLICKHOUSE_CLIENT --multiquery <<EOF
|
||||
DROP USER IF EXISTS ${restricted_user};
|
||||
DROP TABLE IF EXISTS table_with_default;
|
||||
DROP DICTIONARY IF EXISTS dict;
|
||||
DROP TABLE IF EXISTS table_for_dict;
|
||||
|
||||
CREATE USER ${restricted_user};
|
||||
|
||||
CREATE TABLE table_for_dict (key UInt64, value String) ENGINE = Memory();
|
||||
|
||||
INSERT INTO table_for_dict VALUES (1, 'value_1'), (2, 'value_2'), (3, 'value_3'), (4, 'value_4'), (5, 'value_5'), (6, 'value_6');
|
||||
|
||||
CREATE DICTIONARY dict
|
||||
(
|
||||
key UInt64,
|
||||
value String
|
||||
)
|
||||
PRIMARY KEY key
|
||||
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' DB currentDatabase()))
|
||||
LIFETIME(1)
|
||||
LAYOUT(FLAT());
|
||||
|
||||
CREATE TABLE table_with_default(key UInt64, value String DEFAULT dictGetString(dict, 'value', key)) ENGINE = Memory();
|
||||
GRANT INSERT, SELECT ON table_with_default TO ${restricted_user};
|
||||
EOF
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "INSERT INTO table_with_default (key) VALUES (1)"
|
||||
$CLICKHOUSE_CLIENT --async_insert=1 --query "INSERT INTO table_with_default (key) VALUES (2)"
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM table_with_default WHERE key IN [1, 2] ORDER BY key"
|
||||
|
||||
$CLICKHOUSE_CLIENT --user "${restricted_user}" --query "INSERT INTO table_with_default (key) VALUES (3)" 2>&1 | grep -m 1 -oF "Not enough privileges"
|
||||
$CLICKHOUSE_CLIENT --user "${restricted_user}" --async_insert=1 --query "INSERT INTO table_with_default (key) VALUES (4)" 2>&1 | grep -m 1 -oF 'Not enough privileges'
|
||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM table_with_default WHERE key IN [3, 4] ORDER BY key"
|
||||
|
||||
# We give the 'restricted_user' access to the dictionary 'dict',
|
||||
# so now they should be able to insert to a table with DEFAULT dictGet(dict, ...)
|
||||
$CLICKHOUSE_CLIENT --query "GRANT dictGet ON dict TO ${restricted_user}"
|
||||
$CLICKHOUSE_CLIENT --user "${restricted_user}" --query "INSERT INTO table_with_default (key) VALUES (5)"
|
||||
$CLICKHOUSE_CLIENT --user "${restricted_user}" --async_insert=1 --query "INSERT INTO table_with_default (key) VALUES (6)"
|
||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM table_with_default WHERE key IN [5, 6] ORDER BY key"
|
||||
|
||||
$CLICKHOUSE_CLIENT --multiquery <<EOF
|
||||
DROP USER ${restricted_user};
|
||||
DROP TABLE table_with_default;
|
||||
DROP DICTIONARY dict;
|
||||
DROP TABLE table_for_dict;
|
||||
EOF
|
||||
Loading…
Reference in New Issue
Block a user