mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge branch 'master' into system-querylog-map
This commit is contained in:
sundyli
GitHub
commit
77194e4a5d
45
.pylintrc
Normal file
45
.pylintrc
Normal file
@ -0,0 +1,45 @@
|
||||
# vim: ft=config
|
||||
|
||||
[BASIC]
|
||||
max-module-lines=2000
|
||||
# due to SQL
|
||||
max-line-length=200
|
||||
# Drop/decrease them one day:
|
||||
max-branches=50
|
||||
max-nested-blocks=10
|
||||
max-statements=200
|
||||
|
||||
[FORMAT]
|
||||
ignore-long-lines = (# )?<?https?://\S+>?$
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
disable = bad-continuation,
|
||||
missing-docstring,
|
||||
bad-whitespace,
|
||||
too-few-public-methods,
|
||||
invalid-name,
|
||||
too-many-arguments,
|
||||
keyword-arg-before-vararg,
|
||||
too-many-locals,
|
||||
too-many-instance-attributes,
|
||||
cell-var-from-loop,
|
||||
fixme,
|
||||
too-many-public-methods,
|
||||
wildcard-import,
|
||||
unused-wildcard-import,
|
||||
singleton-comparison,
|
||||
# pytest.mark.parametrize is not callable (not-callable)
|
||||
not-callable,
|
||||
# https://github.com/PyCQA/pylint/issues/3882
|
||||
# [Python 3.9] Value 'Optional' is unsubscriptable (unsubscriptable-object) (also Union)
|
||||
unsubscriptable-object,
|
||||
# Drop them one day:
|
||||
redefined-outer-name,
|
||||
broad-except,
|
||||
bare-except,
|
||||
no-else-return,
|
||||
global-statement
|
||||
|
||||
[SIMILARITIES]
|
||||
# due to SQL
|
||||
min-similarity-lines=1000
|
||||
4
LICENSE
4
LICENSE
@ -1,4 +1,4 @@
|
||||
Copyright 2016-2020 Yandex LLC
|
||||
Copyright 2016-2021 Yandex LLC
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
@ -188,7 +188,7 @@ Copyright 2016-2020 Yandex LLC
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2016-2020 Yandex LLC
|
||||
Copyright 2016-2021 Yandex LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@ -851,7 +851,7 @@ public:
|
||||
}
|
||||
|
||||
/// Saturation can occur if 29 Feb is mapped to non-leap year.
|
||||
inline time_t addYears(time_t t, Int64 delta) const
|
||||
inline NO_SANITIZE_UNDEFINED time_t addYears(time_t t, Int64 delta) const
|
||||
{
|
||||
DayNum result_day = addYears(toDayNum(t), delta);
|
||||
|
||||
@ -863,7 +863,7 @@ public:
|
||||
return lut[result_day].date + time_offset;
|
||||
}
|
||||
|
||||
inline DayNum addYears(DayNum d, Int64 delta) const
|
||||
inline NO_SANITIZE_UNDEFINED DayNum addYears(DayNum d, Int64 delta) const
|
||||
{
|
||||
const Values & values = lut[d];
|
||||
|
||||
|
||||
@ -104,8 +104,3 @@ template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
|
||||
template <typename T>
|
||||
inline constexpr bool is_big_int_v = is_big_int<T>::value;
|
||||
|
||||
template <typename To, typename From>
|
||||
inline To bigint_cast(const From & x [[maybe_unused]])
|
||||
{
|
||||
return static_cast<To>(x);
|
||||
}
|
||||
|
||||
2
contrib/cassandra
vendored
2
contrib/cassandra
vendored
@ -1 +1 @@
|
||||
Subproject commit 9cbc1a806df5d40fddbf84533b9873542c6513d8
|
||||
Subproject commit b446d7eb68e6962f431e2b3771313bfe9a2bbd93
|
||||
@ -337,7 +337,7 @@ function run_tests
|
||||
01666_blns
|
||||
)
|
||||
|
||||
time clickhouse-test -j 8 --order=random --use-skip-list --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt"
|
||||
time clickhouse-test --hung-check -j 8 --order=random --use-skip-list --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt"
|
||||
|
||||
# substr is to remove semicolon after test name
|
||||
readarray -t FAILED_TESTS < <(awk '/\[ FAIL|TIMEOUT|ERROR \]/ { print substr($3, 1, length($3)-1) }' "$FASTTEST_OUTPUT/test_log.txt" | tee "$FASTTEST_OUTPUT/failed-parallel-tests.txt")
|
||||
@ -360,7 +360,7 @@ function run_tests
|
||||
|
||||
echo "Going to run again: ${FAILED_TESTS[*]}"
|
||||
|
||||
clickhouse-test --order=random --no-long --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_log.txt"
|
||||
clickhouse-test --hung-check --order=random --no-long --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_log.txt"
|
||||
else
|
||||
echo "No failed tests"
|
||||
fi
|
||||
|
||||
@ -189,14 +189,14 @@ case "$stage" in
|
||||
echo "failure" > status.txt
|
||||
if ! grep -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*" server.log > description.txt
|
||||
then
|
||||
echo "Lost connection to server. See the logs" > description.txt
|
||||
echo "Lost connection to server. See the logs." > description.txt
|
||||
fi
|
||||
else
|
||||
# Something different -- maybe the fuzzer itself died? Don't grep the
|
||||
# server log in this case, because we will find a message about normal
|
||||
# server termination (Received signal 15), which is confusing.
|
||||
echo "failure" > status.txt
|
||||
echo "Fuzzer failed ($fuzzer_exit_code). See the logs" > description.txt
|
||||
echo "Fuzzer failed ($fuzzer_exit_code). See the logs." > description.txt
|
||||
fi
|
||||
;&
|
||||
"report")
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#!/usr/bin/python3
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import clickhouse_driver
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# docker build -t yandex/clickhouse-style-test .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes shellcheck libxml2-utils git python3-pip && pip3 install codespell
|
||||
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes shellcheck libxml2-utils git python3-pip python3-pytest && pip3 install codespell
|
||||
|
||||
|
||||
CMD cd /ClickHouse/utils/check-style && \
|
||||
|
||||
17
docs/en/engines/database-engines/atomic.md
Normal file
17
docs/en/engines/database-engines/atomic.md
Normal file
@ -0,0 +1,17 @@
|
||||
---
|
||||
toc_priority: 32
|
||||
toc_title: Atomic
|
||||
---
|
||||
|
||||
|
||||
# Atomic {#atomic}
|
||||
|
||||
It is supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` queries. Atomic database engine is used by default.
|
||||
|
||||
## Creating a Database {#creating-a-database}
|
||||
|
||||
```sql
|
||||
CREATE DATABASE test ENGINE = Atomic;
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/engines/database_engines/atomic/) <!--hide-->
|
||||
@ -8,7 +8,7 @@ toc_title: Introduction
|
||||
|
||||
Database engines allow you to work with tables.
|
||||
|
||||
By default, ClickHouse uses its native database engine, which provides configurable [table engines](../../engines/table-engines/index.md) and an [SQL dialect](../../sql-reference/syntax.md).
|
||||
By default, ClickHouse uses database engine [Atomic](../../engines/database-engines/atomic.md). It is provides configurable [table engines](../../engines/table-engines/index.md) and an [SQL dialect](../../sql-reference/syntax.md).
|
||||
|
||||
You can also use the following database engines:
|
||||
|
||||
|
||||
@ -51,6 +51,23 @@ All other MySQL data types are converted into [String](../../sql-reference/data-
|
||||
|
||||
[Nullable](../../sql-reference/data-types/nullable.md) is supported.
|
||||
|
||||
## Global Variables Support {#global-variables-support}
|
||||
|
||||
For better compatibility you may address global variables in MySQL style, as `@@identifier`.
|
||||
|
||||
These variables are supported:
|
||||
- `version`
|
||||
- `max_allowed_packet`
|
||||
|
||||
!!! warning "Warning"
|
||||
By now these variables are stubs and don't correspond to anything.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT @@version;
|
||||
```
|
||||
|
||||
## Examples of Use {#examples-of-use}
|
||||
|
||||
Table in MySQL:
|
||||
|
||||
@ -114,6 +114,10 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage.
|
||||
- `_path` — Path to the file.
|
||||
- `_file` — Name of the file.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns)
|
||||
|
||||
## S3-related settings {#settings}
|
||||
|
||||
The following settings can be set before query execution or placed into configuration file.
|
||||
@ -124,8 +128,29 @@ The following settings can be set before query execution or placed into configur
|
||||
|
||||
Security consideration: if malicious user can specify arbitrary S3 URLs, `s3_max_redirects` must be set to zero to avoid [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery) attacks; or alternatively, `remote_host_filter` must be specified in server configuration.
|
||||
|
||||
**See Also**
|
||||
### Endpoint-based settings {#endpointsettings}
|
||||
|
||||
- [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns)
|
||||
The following settings can be specified in configuration file for given endpoint (which will be matched by exact prefix of a URL):
|
||||
|
||||
- `endpoint` — Mandatory. Specifies prefix of an endpoint.
|
||||
- `access_key_id` and `secret_access_key` — Optional. Specifies credentials to use with given endpoint.
|
||||
- `use_environment_credentials` — Optional, default value is `false`. If set to `true`, S3 client will try to obtain credentials from environment variables and Amazon EC2 metadata for given endpoint.
|
||||
- `header` — Optional, can be speficied multiple times. Adds specified HTTP header to a request to given endpoint.
|
||||
|
||||
This configuration also applies to S3 disks in `MergeTree` table engine family.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
<s3>
|
||||
<endpoint-name>
|
||||
<endpoint>https://storage.yandexcloud.net/my-test-bucket-768/</endpoint>
|
||||
<!-- <access_key_id>ACCESS_KEY_ID</access_key_id> -->
|
||||
<!-- <secret_access_key>SECRET_ACCESS_KEY</secret_access_key> -->
|
||||
<!-- <use_environment_credentials>false</use_environment_credentials> -->
|
||||
<!-- <header>Authorization: Bearer SOME-TOKEN</header> -->
|
||||
</endpoint-name>
|
||||
</s3>
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/table_engines/s3/) <!--hide-->
|
||||
|
||||
@ -254,7 +254,6 @@ ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID))
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
``` sql
|
||||
@ -450,7 +449,6 @@ ENGINE = CollapsingMergeTree(Sign)
|
||||
PARTITION BY toYYYYMM(StartDate)
|
||||
ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
You can execute those queries using the interactive mode of `clickhouse-client` (just launch it in a terminal without specifying a query in advance) or try some [alternative interface](../interfaces/index.md) if you want.
|
||||
|
||||
@ -27,6 +27,8 @@ We recommend using SQL-driven workflow. Both of the configuration methods work s
|
||||
!!! note "Warning"
|
||||
You can’t manage the same access entity by both configuration methods simultaneously.
|
||||
|
||||
To see all users, roles, profiles, etc. and all their grants use [SHOW ACCESS](../sql-reference/statements/show.md#show-access-statement) statement.
|
||||
|
||||
## Usage {#access-control-usage}
|
||||
|
||||
By default, the ClickHouse server provides the `default` user account which is not allowed using SQL-driven access control and account management but has all the rights and permissions. The `default` user account is used in any cases when the username is not defined, for example, at login from client or in distributed queries. In distributed query processing a default user account is used, if the configuration of the server or cluster doesn’t specify the [user and password](../engines/table-engines/special/distributed.md) properties.
|
||||
|
||||
@ -2489,7 +2489,6 @@ Possible values:
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
|
||||
## aggregate_functions_null_for_empty {#aggregate_functions_null_for_empty}
|
||||
|
||||
Enables or disables rewriting all aggregate functions in a query, adding [-OrNull](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-ornull) suffix to them. Enable it for SQL standard compatibility.
|
||||
@ -2523,7 +2522,6 @@ With `aggregate_functions_null_for_empty = 1` the result would be:
|
||||
└───────────────┴──────────────┘
|
||||
```
|
||||
|
||||
|
||||
## union_default_mode {#union-default-mode}
|
||||
|
||||
Sets a mode for combining `SELECT` query results. The setting is only used when shared with [UNION](../../sql-reference/statements/select/union.md) without explicitly specifying the `UNION ALL` or `UNION DISTINCT`.
|
||||
@ -2538,7 +2536,6 @@ Default value: `''`.
|
||||
|
||||
See examples in [UNION](../../sql-reference/statements/select/union.md).
|
||||
|
||||
|
||||
## data_type_default_nullable {#data_type_default_nullable}
|
||||
|
||||
Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable).
|
||||
@ -2550,7 +2547,6 @@ Possible values:
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
|
||||
## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold}
|
||||
|
||||
Enables special logic to perform merges on replicas.
|
||||
@ -2570,4 +2566,15 @@ High values for that threshold may lead to replication delays.
|
||||
|
||||
It can be useful when merges are CPU bounded not IO bounded (performing heavy data compression, calculating aggregate functions or default expressions that require a large amount of calculations, or just very high number of tiny merges).
|
||||
|
||||
## max_final_threads {#max-final-threads}
|
||||
|
||||
Sets the maximum number of parallel threads for the `SELECT` query data read phase with the [FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 or 1 — Disabled. `SELECT` queries are executed in a single thread.
|
||||
|
||||
Default value: `16`.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) <!-- hide -->
|
||||
|
||||
@ -413,4 +413,68 @@ Result:
|
||||
|
||||
- [log(x)](../../sql-reference/functions/math-functions.md#logx-lnx)
|
||||
|
||||
## sign(x) {#signx}
|
||||
|
||||
The `sign` function can extract the sign of a real number.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
sign(x)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x` — Values from `-∞` to `+∞`. Support all numeric types in ClickHouse.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- -1 for `x < 0`
|
||||
- 0 for `x = 0`
|
||||
- 1 for `x > 0`
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT sign(0);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─sign(0)─┐
|
||||
│ 0 │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT sign(1);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─sign(1)─┐
|
||||
│ 1 │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT sign(-1);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─sign(-1)─┐
|
||||
│ -1 │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/query_language/functions/math_functions/) <!--hide-->
|
||||
|
||||
@ -10,7 +10,7 @@ Changes roles.
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
@ -10,8 +10,8 @@ Changes row policy.
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table
|
||||
[RENAME TO new_name]
|
||||
ALTER [ROW] POLICY [IF EXISTS] name1 [ON CLUSTER cluster_name1] ON [database1.]table1 [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] ON [database2.]table2 [RENAME TO new_name2] ...]
|
||||
[AS {PERMISSIVE | RESTRICTIVE}]
|
||||
[FOR SELECT]
|
||||
[USING {condition | NONE}][,...]
|
||||
|
||||
@ -10,7 +10,7 @@ Changes settings profiles.
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER SETTINGS PROFILE [IF EXISTS] TO name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
@ -10,8 +10,8 @@ Changes ClickHouse user accounts.
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
||||
[IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
|
||||
[[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||
|
||||
@ -5,12 +5,12 @@ toc_title: ROLE
|
||||
|
||||
# CREATE ROLE {#create-role-statement}
|
||||
|
||||
Creates a new [role](../../../operations/access-rights.md#role-management). Role is a set of [privileges](../../../sql-reference/statements/grant.md#grant-privileges). A [user](../../../sql-reference/statements/create/user.md) assigned a role gets all the privileges of this role.
|
||||
Creates new [roles](../../../operations/access-rights.md#role-management). Role is a set of [privileges](../../../sql-reference/statements/grant.md#grant-privileges). A [user](../../../sql-reference/statements/create/user.md) assigned a role gets all the privileges of this role.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name
|
||||
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [, name2 ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
|
||||
@ -5,16 +5,17 @@ toc_title: ROW POLICY
|
||||
|
||||
# CREATE ROW POLICY {#create-row-policy-statement}
|
||||
|
||||
Creates a [filter for rows](../../../operations/access-rights.md#row-policy-management), which a user can read from a table.
|
||||
Creates [filters for rows](../../../operations/access-rights.md#row-policy-management), which a user can read from a table.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster_name] ON [db.]table
|
||||
CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1
|
||||
[, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2 ...]
|
||||
[AS {PERMISSIVE | RESTRICTIVE}]
|
||||
[FOR SELECT]
|
||||
[USING condition]
|
||||
[TO {role [,...] | ALL | ALL EXCEPT role [,...]}]
|
||||
[TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}]
|
||||
```
|
||||
|
||||
`ON CLUSTER` clause allows creating row policies on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md).
|
||||
|
||||
@ -5,12 +5,13 @@ toc_title: SETTINGS PROFILE
|
||||
|
||||
# CREATE SETTINGS PROFILE {#create-settings-profile-statement}
|
||||
|
||||
Creates a [settings profile](../../../operations/access-rights.md#settings-profiles-management) that can be assigned to a user or a role.
|
||||
Creates [settings profiles](../../../operations/access-rights.md#settings-profiles-management) that can be assigned to a user or a role.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name [ON CLUSTER cluster_name]
|
||||
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
|
||||
@ -39,13 +39,13 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine]
|
||||
|
||||
Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table.
|
||||
|
||||
## From a Table Function {#from-a-table-function}
|
||||
### From a Table Function {#from-a-table-function}
|
||||
|
||||
``` sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name AS table_function()
|
||||
```
|
||||
|
||||
Creates a table with the structure and data returned by a [table function](../../../sql-reference/table-functions/index.md#table-functions).
|
||||
Creates a table with the same result as that of the [table function](../../../sql-reference/table-functions/index.md#table-functions) specified. The created table will also work in the same way as the corresponding table function that was specified.
|
||||
|
||||
``` sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ...
|
||||
@ -260,3 +260,78 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name
|
||||
In most cases, temporary tables are not created manually, but when using external data for a query, or for distributed `(GLOBAL) IN`. For more information, see the appropriate sections
|
||||
|
||||
It’s possible to use tables with [ENGINE = Memory](../../../engines/table-engines/special/memory.md) instead of temporary tables.
|
||||
|
||||
## REPLACE TABLE {#replace-table-query}
|
||||
|
||||
'REPLACE' query allows you to update the table atomically.
|
||||
|
||||
!!!note "Note"
|
||||
This query is supported only for [Atomic](../../../engines/database-engines/atomic.md) database engine.
|
||||
|
||||
If you need to delete some data from a table, you can create a new table and fill it with a `SELECT` statement that doesn't retrieve unwanted data, then drop the old table and rename the new one:
|
||||
|
||||
```sql
|
||||
CREATE TABLE myNewTable AS myOldTable;
|
||||
INSERT INTO myNewTable SELECT * FROM myOldTable WHERE CounterID <12345;
|
||||
DROP TABLE myOldTable;
|
||||
RENAME TABLE myNewTable TO myOldTable;
|
||||
```
|
||||
|
||||
Instead of above, you can use the following:
|
||||
|
||||
```sql
|
||||
REPLACE TABLE myOldTable SELECT * FROM myOldTable WHERE CounterID <12345;
|
||||
```
|
||||
|
||||
### Syntax
|
||||
|
||||
{CREATE [OR REPLACE]|REPLACE} TABLE [db.]table_name
|
||||
|
||||
All syntax forms for `CREATE` query also work for this query. `REPLACE` for a non-existent table will cause an error.
|
||||
|
||||
### Examples:
|
||||
|
||||
Consider the table:
|
||||
|
||||
```sql
|
||||
CREATE DATABASE base ENGINE = Atomic;
|
||||
CREATE OR REPLACE TABLE base.t1 (n UInt64, s String) ENGINE = MergeTree ORDER BY n;
|
||||
INSERT INTO base.t1 VALUES (1, 'test');
|
||||
SELECT * FROM base.t1;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─n─┬─s────┐
|
||||
│ 1 │ test │
|
||||
└───┴──────┘
|
||||
```
|
||||
|
||||
Using `REPLACE` query to clear all data:
|
||||
|
||||
```sql
|
||||
CREATE OR REPLACE TABLE base.t1 (n UInt64, s Nullable(String)) ENGINE = MergeTree ORDER BY n;
|
||||
INSERT INTO base.t1 VALUES (2, null);
|
||||
SELECT * FROM base.t1;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─n─┬─s──┐
|
||||
│ 2 │ \N │
|
||||
└───┴────┘
|
||||
```
|
||||
|
||||
Using `REPLACE` query to change table structure:
|
||||
|
||||
```sql
|
||||
REPLACE TABLE base.t1 (n UInt64) ENGINE = MergeTree ORDER BY n;
|
||||
INSERT INTO base.t1 VALUES (3);
|
||||
SELECT * FROM base.t1;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─n─┐
|
||||
│ 3 │
|
||||
└───┘
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/create/table) <!--hide-->
|
||||
|
||||
@ -5,12 +5,13 @@ toc_title: USER
|
||||
|
||||
# CREATE USER {#create-user-statement}
|
||||
|
||||
Creates a [user account](../../../operations/access-rights.md#user-account-management).
|
||||
Creates [user accounts](../../../operations/access-rights.md#user-account-management).
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]
|
||||
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] ...]
|
||||
[IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
|
||||
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[DEFAULT ROLE role [,...]]
|
||||
@ -69,7 +70,7 @@ CREATE USER john DEFAULT ROLE role1, role2
|
||||
Create the user account `john` and make all his future roles default:
|
||||
|
||||
``` sql
|
||||
ALTER USER user DEFAULT ROLE ALL
|
||||
CREATE USER user DEFAULT ROLE ALL
|
||||
```
|
||||
|
||||
When some role is assigned to `john` in the future, it will become default automatically.
|
||||
@ -77,5 +78,5 @@ When some role is assigned to `john` in the future, it will become default autom
|
||||
Create the user account `john` and make all his future roles default excepting `role1` and `role2`:
|
||||
|
||||
``` sql
|
||||
ALTER USER john DEFAULT ROLE ALL EXCEPT role1, role2
|
||||
CREATE USER john DEFAULT ROLE ALL EXCEPT role1, role2
|
||||
```
|
||||
|
||||
@ -13,7 +13,7 @@ Basic query format:
|
||||
INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ...
|
||||
```
|
||||
|
||||
You can specify a list of columns to insert using the `(c1, c2, c3)`. You can also use an expression with column [matcher](../../sql-reference/statements/select/index.md#asterisk) such as `*` and/or [modifiers](../../sql-reference/statements/select/index.md#select-modifiers) such as [APPLY](../../sql-reference/statements/select/index.md#apply-modifier), [EXCEPT](../../sql-reference/statements/select/index.md#apply-modifier), [REPLACE](../../sql-reference/statements/select/index.md#replace-modifier).
|
||||
You can specify a list of columns to insert using the `(c1, c2, c3)`. You can also use an expression with column [matcher](../../sql-reference/statements/select/index.md#asterisk) such as `*` and/or [modifiers](../../sql-reference/statements/select/index.md#select-modifiers) such as [APPLY](../../sql-reference/statements/select/index.md#apply-modifier), [EXCEPT](../../sql-reference/statements/select/index.md#apply-modifier), [REPLACE](../../sql-reference/statements/select/index.md#replace-modifier).
|
||||
|
||||
For example, consider the table:
|
||||
|
||||
@ -30,7 +30,6 @@ CREATE TABLE insert_select_testtable
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY a
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
``` sql
|
||||
@ -55,7 +54,7 @@ SELECT * FROM insert_select_testtable;
|
||||
│ 1 │ a │ 1 │
|
||||
└───┴───┴───┘
|
||||
```
|
||||
|
||||
|
||||
In this example, we see that the second inserted row has `a` and `c` columns filled by the passed values, and `b` filled with value by default.
|
||||
|
||||
If a list of columns doesn't include all existing columns, the rest of the columns are filled with:
|
||||
|
||||
@ -25,6 +25,8 @@ It is applicable when selecting data from tables that use the [MergeTree](../../
|
||||
- [Replicated](../../../engines/table-engines/mergetree-family/replication.md) versions of `MergeTree` engines.
|
||||
- [View](../../../engines/table-engines/special/view.md), [Buffer](../../../engines/table-engines/special/buffer.md), [Distributed](../../../engines/table-engines/special/distributed.md), and [MaterializedView](../../../engines/table-engines/special/materializedview.md) engines that operate over other engines, provided they were created over `MergeTree`-engine tables.
|
||||
|
||||
Now `SELECT` queries with `FINAL` are executed in parallel and slightly faster. But there are drawbacks (see below). The [max_final_threads](../../../operations/settings/settings.md#max-final-threads) setting limits the number of threads used.
|
||||
|
||||
### Drawbacks {#drawbacks}
|
||||
|
||||
Queries that use `FINAL` are executed slightly slower than similar queries that don’t, because:
|
||||
|
||||
@ -231,7 +231,7 @@ Shows privileges for a user.
|
||||
### Syntax {#show-grants-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW GRANTS [FOR user]
|
||||
SHOW GRANTS [FOR user1 [, user2 ...]]
|
||||
```
|
||||
|
||||
If user is not specified, the query returns privileges for the current user.
|
||||
@ -245,7 +245,7 @@ Shows parameters that were used at a [user creation](../../sql-reference/stateme
|
||||
### Syntax {#show-create-user-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW CREATE USER [name | CURRENT_USER]
|
||||
SHOW CREATE USER [name1 [, name2 ...] | CURRENT_USER]
|
||||
```
|
||||
|
||||
## SHOW CREATE ROLE {#show-create-role-statement}
|
||||
@ -255,7 +255,7 @@ Shows parameters that were used at a [role creation](../../sql-reference/stateme
|
||||
### Syntax {#show-create-role-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW CREATE ROLE name
|
||||
SHOW CREATE ROLE name1 [, name2 ...]
|
||||
```
|
||||
|
||||
## SHOW CREATE ROW POLICY {#show-create-row-policy-statement}
|
||||
@ -265,7 +265,7 @@ Shows parameters that were used at a [row policy creation](../../sql-reference/s
|
||||
### Syntax {#show-create-row-policy-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW CREATE [ROW] POLICY name ON [database.]table
|
||||
SHOW CREATE [ROW] POLICY name ON [database1.]table1 [, [database2.]table2 ...]
|
||||
```
|
||||
|
||||
## SHOW CREATE QUOTA {#show-create-quota-statement}
|
||||
@ -275,7 +275,7 @@ Shows parameters that were used at a [quota creation](../../sql-reference/statem
|
||||
### Syntax {#show-create-quota-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW CREATE QUOTA [name | CURRENT]
|
||||
SHOW CREATE QUOTA [name1 [, name2 ...] | CURRENT]
|
||||
```
|
||||
|
||||
## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement}
|
||||
@ -285,7 +285,7 @@ Shows parameters that were used at a [settings profile creation](../../sql-refer
|
||||
### Syntax {#show-create-settings-profile-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW CREATE [SETTINGS] PROFILE name
|
||||
SHOW CREATE [SETTINGS] PROFILE name1 [, name2 ...]
|
||||
```
|
||||
|
||||
## SHOW USERS {#show-users-statement}
|
||||
@ -307,7 +307,6 @@ Returns a list of [roles](../../operations/access-rights.md#role-management). To
|
||||
``` sql
|
||||
SHOW [CURRENT|ENABLED] ROLES
|
||||
```
|
||||
|
||||
## SHOW PROFILES {#show-profiles-statement}
|
||||
|
||||
Returns a list of [setting profiles](../../operations/access-rights.md#settings-profiles-management). To view user accounts parameters, see the system table [settings_profiles](../../operations/system-tables/settings_profiles.md#system_tables-settings_profiles).
|
||||
@ -347,7 +346,15 @@ Returns a [quota](../../operations/quotas.md) consumption for all users or for c
|
||||
``` sql
|
||||
SHOW [CURRENT] QUOTA
|
||||
```
|
||||
## SHOW ACCESS {#show-access-statement}
|
||||
|
||||
Shows all [users](../../operations/access-rights.md#user-account-management), [roles](../../operations/access-rights.md#role-management), [profiles](../../operations/access-rights.md#settings-profiles-management), etc. and all their [grants](../../sql-reference/statements/grant.md#grant-privileges).
|
||||
|
||||
### Syntax {#show-access-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW ACCESS
|
||||
```
|
||||
## SHOW CLUSTER(s) {#show-cluster-statement}
|
||||
|
||||
Returns a list of clusters. All available clusters are listed in the [system.clusters](../../operations/system-tables/clusters.md) table.
|
||||
|
||||
17
docs/es/engines/database-engines/atomic.md
Normal file
17
docs/es/engines/database-engines/atomic.md
Normal file
@ -0,0 +1,17 @@
|
||||
---
|
||||
toc_priority: 32
|
||||
toc_title: Atomic
|
||||
---
|
||||
|
||||
|
||||
# Atomic {#atomic}
|
||||
|
||||
It is supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` queries. Atomic database engine is used by default.
|
||||
|
||||
## Creating a Database {#creating-a-database}
|
||||
|
||||
```sql
|
||||
CREATE DATABASE test ENGINE = Atomic;
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/engines/database_engines/atomic/) <!--hide-->
|
||||
@ -256,7 +256,6 @@ ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID))
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
``` sql
|
||||
@ -452,7 +451,6 @@ ENGINE = CollapsingMergeTree(Sign)
|
||||
PARTITION BY toYYYYMM(StartDate)
|
||||
ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
Puede ejecutar esas consultas utilizando el modo interactivo de `clickhouse-client` (simplemente ejecútelo en un terminal sin especificar una consulta por adelantado) o pruebe algunos [interfaz alternativa](../interfaces/index.md) Si quieres.
|
||||
|
||||
17
docs/fr/engines/database-engines/atomic.md
Normal file
17
docs/fr/engines/database-engines/atomic.md
Normal file
@ -0,0 +1,17 @@
|
||||
---
|
||||
toc_priority: 32
|
||||
toc_title: Atomic
|
||||
---
|
||||
|
||||
|
||||
# Atomic {#atomic}
|
||||
|
||||
It is supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` queries. Atomic database engine is used by default.
|
||||
|
||||
## Creating a Database {#creating-a-database}
|
||||
|
||||
```sql
|
||||
CREATE DATABASE test ENGINE = Atomic;
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/engines/database_engines/atomic/) <!--hide-->
|
||||
@ -256,7 +256,6 @@ ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID))
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
``` sql
|
||||
@ -452,7 +451,6 @@ ENGINE = CollapsingMergeTree(Sign)
|
||||
PARTITION BY toYYYYMM(StartDate)
|
||||
ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
Vous pouvez exécuter ces requêtes en utilisant le mode interactif de `clickhouse-client` (lancez - le simplement dans un terminal sans spécifier une requête à l'avance) ou essayez-en [interface de rechange](../interfaces/index.md) Si tu veux.
|
||||
|
||||
17
docs/ja/engines/database-engines/atomic.md
Normal file
17
docs/ja/engines/database-engines/atomic.md
Normal file
@ -0,0 +1,17 @@
|
||||
---
|
||||
toc_priority: 32
|
||||
toc_title: Atomic
|
||||
---
|
||||
|
||||
|
||||
# Atomic {#atomic}
|
||||
|
||||
It is supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` queries. Atomic database engine is used by default.
|
||||
|
||||
## Creating a Database {#creating-a-database}
|
||||
|
||||
```sql
|
||||
CREATE DATABASE test ENGINE = Atomic;
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/engines/database_engines/atomic/) <!--hide-->
|
||||
@ -262,7 +262,6 @@ ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID))
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
``` sql
|
||||
@ -458,7 +457,6 @@ ENGINE = CollapsingMergeTree(Sign)
|
||||
PARTITION BY toYYYYMM(StartDate)
|
||||
ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
これらのクエリは、`clickhouse-client` の対話型モード(事前にクエリを指定せずにターミナルで起動するだけです)を使って実行するか、[代替インターフェイス](../interfaces/index.md) で実行できます。
|
||||
|
||||
@ -51,6 +51,23 @@ ENGINE = MySQL('host:port', ['database' | database], 'user', 'password')
|
||||
|
||||
[Nullable](../../engines/database-engines/mysql.md) поддержан.
|
||||
|
||||
## Использование глобальных переменных {#global-variables-support}
|
||||
|
||||
Для лучшей совместимости к глобальным переменным можно обращаться в формате MySQL, как `@@identifier`.
|
||||
|
||||
Поддерживаются следующие переменные:
|
||||
- `version`
|
||||
- `max_allowed_packet`
|
||||
|
||||
!!! warning "Предупреждение"
|
||||
В настоящее время эти переменные реализованы только как "заглушки" и не содержат актуальных данных.
|
||||
|
||||
Пример:
|
||||
|
||||
``` sql
|
||||
SELECT @@version;
|
||||
```
|
||||
|
||||
## Примеры использования {#primery-ispolzovaniia}
|
||||
|
||||
Таблица в MySQL:
|
||||
|
||||
@ -254,7 +254,6 @@ ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID))
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
``` sql
|
||||
@ -450,7 +449,6 @@ ENGINE = CollapsingMergeTree(Sign)
|
||||
PARTITION BY toYYYYMM(StartDate)
|
||||
ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
You can execute those queries using the interactive mode of `clickhouse-client` (just launch it in a terminal without specifying a query in advance) or try some [alternative interface](../interfaces/index.md) if you want.
|
||||
|
||||
@ -28,6 +28,7 @@ ClickHouse поддерживает управление доступом на
|
||||
!!! note "Внимание"
|
||||
Нельзя одновременно использовать оба метода для управления одним и тем же объектом системы доступа.
|
||||
|
||||
Чтобы посмотреть список всех пользователей, ролей, профилей и пр., а также все привилегии, используйте запрос [SHOW ACCESS](../sql-reference/statements/show.md#show-access-statement).
|
||||
|
||||
## Использование {#access-control-usage}
|
||||
|
||||
|
||||
@ -2437,4 +2437,15 @@ SELECT SUM(-1), MAX(0) FROM system.one WHERE 0;
|
||||
|
||||
Эта настройка полезна, когда скорость слияния ограничивается мощностью процессора, а не скоростью операций ввода-вывода (при выполнении "тяжелого" сжатия данных, при расчете агрегатных функций или выражений по умолчанию, требующих большого объема вычислений, или просто при большом количестве мелких слияний).
|
||||
|
||||
## max_final_threads {#max-final-threads}
|
||||
|
||||
Устанавливает максимальное количество параллельных потоков для фазы чтения данных запроса `SELECT` с модификатором [FINAL](../../sql-reference/statements/select/from.md#select-from-final).
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- Положительное целое число.
|
||||
- 0 или 1 — настройка отключена. `SELECT` запросы выполняются в один поток.
|
||||
|
||||
Значение по умолчанию: `16`.
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) <!--hide-->
|
||||
|
||||
@ -5,14 +5,14 @@ toc_title: ROLE
|
||||
|
||||
# ALTER ROLE {#alter-role-statement}
|
||||
|
||||
Изменяет роль.
|
||||
Изменяет роли.
|
||||
|
||||
## Синтаксис {#alter-role-syntax}
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/role/) <!--hide-->
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/alter/role/) <!--hide-->
|
||||
|
||||
@ -7,15 +7,15 @@ toc_title: ROW POLICY
|
||||
|
||||
Изменяет политику доступа к строкам.
|
||||
|
||||
## Синтаксис {#alter-row-policy-syntax}
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table
|
||||
[RENAME TO new_name]
|
||||
ALTER [ROW] POLICY [IF EXISTS] name1 [ON CLUSTER cluster_name1] ON [database1.]table1 [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] ON [database2.]table2 [RENAME TO new_name2] ...]
|
||||
[AS {PERMISSIVE | RESTRICTIVE}]
|
||||
[FOR SELECT]
|
||||
[USING {condition | NONE}][,...]
|
||||
[TO {role [,...] | ALL | ALL EXCEPT role [,...]}]
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/row-policy/) <!--hide-->
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/alter/row-policy/) <!--hide-->
|
||||
@ -7,12 +7,12 @@ toc_title: SETTINGS PROFILE
|
||||
|
||||
Изменяет профили настроек.
|
||||
|
||||
## Синтаксис {#alter-settings-profile-syntax}
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/settings-profile) <!--hide-->
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/alter/settings-profile) <!--hide-->
|
||||
@ -5,21 +5,19 @@ toc_title: USER
|
||||
|
||||
# ALTER USER {#alter-user-statement}
|
||||
|
||||
Изменяет аккаунт пользователя ClickHouse.
|
||||
Изменяет аккаунты пользователей ClickHouse.
|
||||
|
||||
## Синтаксис {#alter-user-syntax}
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
||||
[IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
|
||||
[[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
## Описание {#alter-user-dscr}
|
||||
|
||||
Для выполнения `ALTER USER` необходима привилегия [ALTER USER](../grant.md#grant-access-management).
|
||||
|
||||
## Примеры {#alter-user-examples}
|
||||
|
||||
@ -5,18 +5,16 @@ toc_title: "\u0420\u043e\u043b\u044c"
|
||||
|
||||
# CREATE ROLE {#create-role-statement}
|
||||
|
||||
Создает [роль](../../../operations/access-rights.md#role-management).
|
||||
Создает [роли](../../../operations/access-rights.md#role-management). Роль — это набор [привилегий](../grant.md#grant-privileges). Пользователь, которому назначена роль, получает все привилегии этой роли.
|
||||
|
||||
### Синтаксис {#create-role-syntax}
|
||||
Синтаксис:
|
||||
|
||||
```sql
|
||||
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name
|
||||
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [, name2 ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
### Описание {#create-role-description}
|
||||
|
||||
Роль — это набор [привилегий](../grant.md#grant-privileges). Пользователь, которому назначена роль, получает все привилегии этой роли.
|
||||
## Управление ролями {#managing-roles}
|
||||
|
||||
Одному пользователю можно назначить несколько ролей. Пользователи могут применять назначенные роли в произвольных комбинациях с помощью выражения [SET ROLE](../misc.md#set-role-statement). Конечный объем привилегий — это комбинация всех привилегий всех примененных ролей. Если у пользователя имеются привилегии, присвоенные его аккаунту напрямую, они также прибавляются к привилегиям, присвоенным через роли.
|
||||
|
||||
@ -26,7 +24,7 @@ CREATE ROLE [IF NOT EXISTS | OR REPLACE] name
|
||||
|
||||
Для удаления роли используется выражение [DROP ROLE](../misc.md#drop-role-statement). Удаленная роль автоматически отзывается у всех пользователей, которым была назначена.
|
||||
|
||||
### Примеры {#create-role-examples}
|
||||
## Примеры {#create-role-examples}
|
||||
|
||||
```sql
|
||||
CREATE ROLE accountant;
|
||||
|
||||
@ -5,19 +5,22 @@ toc_title: "\u041f\u043e\u043b\u0438\u0442\u0438\u043a\u0430\u0020\u0434\u043e\u
|
||||
|
||||
# CREATE ROW POLICY {#create-row-policy-statement}
|
||||
|
||||
Создает [фильтр для строк](../../../operations/access-rights.md#row-policy-management), которые пользователь может прочесть из таблицы.
|
||||
Создает [фильтры для строк](../../../operations/access-rights.md#row-policy-management), которые пользователь может прочесть из таблицы.
|
||||
|
||||
### Синтаксис {#create-row-policy-syntax}
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster_name] ON [db.]table
|
||||
CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1
|
||||
[, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2 ...]
|
||||
[AS {PERMISSIVE | RESTRICTIVE}]
|
||||
[FOR SELECT]
|
||||
[USING condition]
|
||||
[TO {role [,...] | ALL | ALL EXCEPT role [,...]}]
|
||||
```
|
||||
|
||||
#### Секция AS {#create-row-policy-as}
|
||||
Секция `ON CLUSTER` позволяет создавать фильтры для строк на кластере, см. [Распределенные DDL запросы](../../../sql-reference/distributed-ddl.md).
|
||||
|
||||
## Секция AS {#create-row-policy-as}
|
||||
|
||||
С помощью данной секции можно создать политику разрешения или ограничения.
|
||||
|
||||
@ -27,16 +30,17 @@ CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster
|
||||
|
||||
Ограничительные политики применяются к строкам, прошедшим фильтр разрешительной политики. Если вы не зададите разрешительные политики, пользователь не сможет обращаться ни к каким строкам из таблицы.
|
||||
|
||||
#### Секция TO {#create-row-policy-to}
|
||||
## Секция TO {#create-row-policy-to}
|
||||
|
||||
В секции `TO` вы можете перечислить как роли, так и пользователей. Например, `CREATE ROW POLICY ... TO accountant, john@localhost`.
|
||||
|
||||
Ключевым словом `ALL` обозначаются все пользователи, включая текущего. Ключевые слова `ALL EXCEPT` позволяют исключить пользователей из списка всех пользователей. Например, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost`
|
||||
|
||||
### Примеры
|
||||
## Примеры
|
||||
|
||||
- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost`
|
||||
- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira`
|
||||
`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost`
|
||||
|
||||
`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira`
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/row-policy)
|
||||
<!--hide-->
|
||||
@ -5,16 +5,19 @@ toc_title: "\u041f\u0440\u043e\u0444\u0438\u043b\u044c\u0020\u043d\u0430\u0441\u
|
||||
|
||||
# CREATE SETTINGS PROFILE {#create-settings-profile-statement}
|
||||
|
||||
Создает [профиль настроек](../../../operations/access-rights.md#settings-profiles-management), который может быть присвоен пользователю или роли.
|
||||
Создает [профили настроек](../../../operations/access-rights.md#settings-profiles-management), которые могут быть присвоены пользователю или роли.
|
||||
|
||||
### Синтаксис {#create-settings-profile-syntax}
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]
|
||||
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
### Пример {#create-settings-profile-syntax}
|
||||
Секция `ON CLUSTER` позволяет создавать профили на кластере, см. [Распределенные DDL запросы](../../../sql-reference/distributed-ddl.md).
|
||||
|
||||
## Пример {#create-settings-profile-syntax}
|
||||
|
||||
Создать профиль настроек `max_memory_usage_profile`, который содержит значение и ограничения для настройки `max_memory_usage`. Присвоить профиль пользователю `robin`:
|
||||
|
||||
|
||||
@ -5,19 +5,20 @@ toc_title: "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u
|
||||
|
||||
# CREATE USER {#create-user-statement}
|
||||
|
||||
Создает [аккаунт пользователя](../../../operations/access-rights.md#user-account-management).
|
||||
Создает [аккаунты пользователей](../../../operations/access-rights.md#user-account-management).
|
||||
|
||||
### Синтаксис {#create-user-syntax}
|
||||
Синтаксис:
|
||||
|
||||
```sql
|
||||
CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]
|
||||
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
|
||||
[, name2 [ON CLUSTER cluster_name2] ...]
|
||||
[IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
|
||||
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[DEFAULT ROLE role [,...]]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
#### Идентификация
|
||||
## Идентификация
|
||||
|
||||
Существует несколько способов идентификации пользователя:
|
||||
|
||||
@ -28,7 +29,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]
|
||||
- `IDENTIFIED WITH double_sha1_password BY 'qwerty'`
|
||||
- `IDENTIFIED WITH double_sha1_hash BY 'hash'`
|
||||
|
||||
#### Пользовательский хост
|
||||
## Пользовательский хост
|
||||
|
||||
Пользовательский хост — это хост, с которого можно установить соединение с сервером ClickHouse. Хост задается в секции `HOST` следующими способами:
|
||||
|
||||
@ -49,7 +50,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]
|
||||
ClickHouse трактует конструкцию `user_name@'address'` как имя пользователя целиком. То есть технически вы можете создать несколько пользователей с одинаковыми `user_name`, но разными частями конструкции после `@`, но лучше так не делать.
|
||||
|
||||
|
||||
### Примеры {#create-user-examples}
|
||||
## Примеры {#create-user-examples}
|
||||
|
||||
|
||||
Создать аккаунт `mira`, защищенный паролем `qwerty`:
|
||||
@ -69,7 +70,7 @@ CREATE USER john DEFAULT ROLE role1, role2
|
||||
Создать аккаунт `john` и установить ролями по умолчанию все его будущие роли:
|
||||
|
||||
``` sql
|
||||
ALTER USER user DEFAULT ROLE ALL
|
||||
CREATE USER user DEFAULT ROLE ALL
|
||||
```
|
||||
|
||||
Когда роль будет назначена аккаунту `john`, она автоматически станет ролью по умолчанию.
|
||||
@ -77,7 +78,7 @@ ALTER USER user DEFAULT ROLE ALL
|
||||
Создать аккаунт `john` и установить ролями по умолчанию все его будущие роли, кроме `role1` и `role2`:
|
||||
|
||||
``` sql
|
||||
ALTER USER john DEFAULT ROLE ALL EXCEPT role1, role2
|
||||
CREATE USER john DEFAULT ROLE ALL EXCEPT role1, role2
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/user)
|
||||
|
||||
@ -13,7 +13,7 @@ toc_title: INSERT INTO
|
||||
INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ...
|
||||
```
|
||||
|
||||
Вы можете указать список столбцов для вставки, используя синтаксис `(c1, c2, c3)`. Также можно использовать выражение cо [звездочкой](../../sql-reference/statements/select/index.md#asterisk) и/или модификаторами, такими как `APPLY`, `EXCEPT`, `REPLACE`.
|
||||
Вы можете указать список столбцов для вставки, используя синтаксис `(c1, c2, c3)`. Также можно использовать выражение cо [звездочкой](../../sql-reference/statements/select/index.md#asterisk) и/или модификаторами, такими как `APPLY`, `EXCEPT`, `REPLACE`.
|
||||
|
||||
В качестве примера рассмотрим таблицу:
|
||||
|
||||
@ -30,13 +30,12 @@ SHOW CREATE insert_select_testtable
|
||||
`c` Int8
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY a
|
||||
SETTINGS index_granularity = 8192 │
|
||||
ORDER BY a │
|
||||
└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
``` sql
|
||||
INSERT INTO insert_select_testtable (*) VALUES (1, 'a', 1)
|
||||
INSERT INTO insert_select_testtable (*) VALUES (1, 'a', 1)
|
||||
```
|
||||
|
||||
Если вы хотите вставить данные во все столбцы, кроме 'b', вам нужно передать столько значений, сколько столбцов вы указали в скобках:
|
||||
|
||||
@ -25,6 +25,8 @@ toc_title: FROM
|
||||
- [Replicated](../../../engines/table-engines/mergetree-family/replication.md) варианты исполнения `MergeTree` движков.
|
||||
- [View](../../../engines/table-engines/special/view.md), [Buffer](../../../engines/table-engines/special/buffer.md), [Distributed](../../../engines/table-engines/special/distributed.md), и [MaterializedView](../../../engines/table-engines/special/materializedview.md), которые работают поверх других движков, если они созданы для таблиц с движками семейства `MergeTree`.
|
||||
|
||||
Теперь `SELECT` запросы с `FINAL` выполняются параллельно и, следовательно, немного быстрее. Но имеются серьезные недостатки при их использовании (смотрите ниже). Настройка [max_final_threads](../../../operations/settings/settings.md#max-final-threads) устанавливает максимальное количество потоков.
|
||||
|
||||
### Недостатки {#drawbacks}
|
||||
|
||||
Запросы, которые используют `FINAL` выполняются немного медленее, чем аналогичные запросы без него, потому что:
|
||||
|
||||
@ -249,11 +249,9 @@ SHOW GRANTS [FOR user]
|
||||
### Синтаксис {#show-create-user-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW CREATE USER [name | CURRENT_USER]
|
||||
SHOW CREATE USER [name1 [, name2 ...] | CURRENT_USER]
|
||||
```
|
||||
|
||||
|
||||
|
||||
## SHOW CREATE ROLE {#show-create-role-statement}
|
||||
|
||||
Выводит параметры, использованные при [создании роли](create/role.md#create-role-statement).
|
||||
@ -261,11 +259,9 @@ SHOW CREATE USER [name | CURRENT_USER]
|
||||
### Синтаксис {#show-create-role-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW CREATE ROLE name
|
||||
SHOW CREATE ROLE name1 [, name2 ...]
|
||||
```
|
||||
|
||||
|
||||
|
||||
## SHOW CREATE ROW POLICY {#show-create-row-policy-statement}
|
||||
|
||||
Выводит параметры, использованные при [создании политики доступа к строкам](create/row-policy.md#create-row-policy-statement).
|
||||
@ -273,10 +269,9 @@ SHOW CREATE ROLE name
|
||||
### Синтаксис {#show-create-row-policy-syntax}
|
||||
|
||||
```sql
|
||||
SHOW CREATE [ROW] POLICY name ON [database.]table
|
||||
SHOW CREATE [ROW] POLICY name ON [database1.]table1 [, [database2.]table2 ...]
|
||||
```
|
||||
|
||||
|
||||
## SHOW CREATE QUOTA {#show-create-quota-statement}
|
||||
|
||||
Выводит параметры, использованные при [создании квоты](create/quota.md#create-quota-statement).
|
||||
@ -284,10 +279,9 @@ SHOW CREATE [ROW] POLICY name ON [database.]table
|
||||
### Синтаксис {#show-create-row-policy-syntax}
|
||||
|
||||
```sql
|
||||
SHOW CREATE QUOTA [name | CURRENT]
|
||||
SHOW CREATE QUOTA [name1 [, name2 ...] | CURRENT]
|
||||
```
|
||||
|
||||
|
||||
## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement}
|
||||
|
||||
Выводит параметры, использованные при [создании профиля настроек](create/settings-profile.md#create-settings-profile-statement).
|
||||
@ -295,10 +289,9 @@ SHOW CREATE QUOTA [name | CURRENT]
|
||||
### Синтаксис {#show-create-row-policy-syntax}
|
||||
|
||||
```sql
|
||||
SHOW CREATE [SETTINGS] PROFILE name
|
||||
SHOW CREATE [SETTINGS] PROFILE name1 [, name2 ...]
|
||||
```
|
||||
|
||||
|
||||
## SHOW USERS {#show-users-statement}
|
||||
|
||||
Выводит список [пользовательских аккаунтов](../../operations/access-rights.md#user-account-management). Для просмотра параметров пользовательских аккаунтов, см. системную таблицу [system.users](../../operations/system-tables/users.md#system_tables-users).
|
||||
@ -359,4 +352,14 @@ SHOW QUOTAS
|
||||
SHOW [CURRENT] QUOTA
|
||||
```
|
||||
|
||||
## SHOW ACCESS {#show-access-statement}
|
||||
|
||||
Выводит список всех [пользователей](../../operations/access-rights.md#user-account-management), [ролей](../../operations/access-rights.md#role-management), [профилей](../../operations/access-rights.md#settings-profiles-management) и пр., а также все [привилегии](../../sql-reference/statements/grant.md#grant-privileges).
|
||||
|
||||
### Синтаксис {#show-access-syntax}
|
||||
|
||||
``` sql
|
||||
SHOW ACCESS
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/show/) <!--hide-->
|
||||
|
||||
17
docs/zh/engines/database-engines/atomic.md
Normal file
17
docs/zh/engines/database-engines/atomic.md
Normal file
@ -0,0 +1,17 @@
|
||||
---
|
||||
toc_priority: 32
|
||||
toc_title: Atomic
|
||||
---
|
||||
|
||||
|
||||
# Atomic {#atomic}
|
||||
|
||||
It is supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` queries. Atomic database engine is used by default.
|
||||
|
||||
## Creating a Database {#creating-a-database}
|
||||
|
||||
```sql
|
||||
CREATE DATABASE test ENGINE = Atomic;
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/engines/database_engines/atomic/) <!--hide-->
|
||||
@ -254,7 +254,6 @@ ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID))
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
``` sql
|
||||
@ -450,7 +449,6 @@ ENGINE = CollapsingMergeTree(Sign)
|
||||
PARTITION BY toYYYYMM(StartDate)
|
||||
ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
您可以使用`clickhouse-client`的交互模式执行这些查询（只需在终端中启动它，而不需要提前指定查询）。或者如果你愿意，可以尝试一些[替代接口](../interfaces/index.md)。
|
||||
|
||||
@ -494,11 +494,11 @@ SELECT now('Europe/Moscow');
|
||||
将时间向前取整半小时。
|
||||
此功能用于Yandex.Metrica，因为如果跟踪标记显示单个用户的连续综合浏览量在时间上严格超过此数量，则半小时是将会话分成两个会话的最短时间。这意味着（tag id，user id，time slot）可用于搜索相应会话中包含的综合浏览量。
|
||||
|
||||
## toYYYMM {#toyyyymm}
|
||||
## toYYYYMM {#toyyyymm}
|
||||
|
||||
将Date或DateTime转换为包含年份和月份编号的UInt32类型的数字（YYYY \* 100 + MM）。
|
||||
|
||||
## toYYYMMDD {#toyyyymmdd}
|
||||
## toYYYYMMDD {#toyyyymmdd}
|
||||
|
||||
将Date或DateTime转换为包含年份和月份编号的UInt32类型的数字（YYYY \* 10000 + MM \* 100 + DD）。
|
||||
|
||||
|
||||
@ -25,7 +25,6 @@ CREATE TABLE insert_select_testtable
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY a
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
``` sql
|
||||
|
||||
@ -321,9 +321,9 @@
|
||||
Parameters:
|
||||
host - LDAP server hostname or IP, this parameter is mandatory and cannot be empty.
|
||||
port - LDAP server port, default is 636 if enable_tls is set to true, 389 otherwise.
|
||||
auth_dn_prefix, auth_dn_suffix - prefix and suffix used to construct the DN to bind to.
|
||||
Effectively, the resulting DN will be constructed as auth_dn_prefix + escape(user_name) + auth_dn_suffix string.
|
||||
Note, that this implies that auth_dn_suffix should usually have comma ',' as its first non-space character.
|
||||
bind_dn - template used to construct the DN to bind to.
|
||||
The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual
|
||||
user name during each authentication attempt.
|
||||
verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed
|
||||
to be successfully authenticated for all consecutive requests without contacting the LDAP server.
|
||||
Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request.
|
||||
@ -344,8 +344,7 @@
|
||||
<my_ldap_server>
|
||||
<host>localhost</host>
|
||||
<port>636</port>
|
||||
<auth_dn_prefix>uid=</auth_dn_prefix>
|
||||
<auth_dn_suffix>,ou=users,dc=example,dc=com</auth_dn_suffix>
|
||||
<bind_dn>uid={user_name},ou=users,dc=example,dc=com</bind_dn>
|
||||
<verification_cooldown>300</verification_cooldown>
|
||||
<enable_tls>yes</enable_tls>
|
||||
<tls_minimum_protocol_version>tls1.2</tls_minimum_protocol_version>
|
||||
@ -375,9 +374,29 @@
|
||||
server - one of LDAP server names defined in 'ldap_servers' config section above.
|
||||
This parameter is mandatory and cannot be empty.
|
||||
roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server.
|
||||
If no roles are specified, user will not be able to perform any actions after authentication.
|
||||
If any of the listed roles is not defined locally at the time of authentication, the authenthication attempt
|
||||
will fail as if the provided password was incorrect.
|
||||
If no roles are specified here or assigned during role mapping (below), user will not be able to perform any
|
||||
actions after authentication.
|
||||
role_mapping - section with LDAP search parameters and mapping rules.
|
||||
When a user authenticates, while still bound to LDAP, an LDAP search is performed using search_filter and the
|
||||
name of the logged in user. For each entry found during that search, the value of the specified attribute is
|
||||
extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the
|
||||
value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by
|
||||
CREATE ROLE command.
|
||||
There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be
|
||||
applied.
|
||||
base_dn - template used to construct the base DN for the LDAP search.
|
||||
The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings
|
||||
of the template with the actual user name and bind DN during each LDAP search.
|
||||
scope - scope of the LDAP search.
|
||||
Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default).
|
||||
search_filter - template used to construct the search filter for the LDAP search.
|
||||
The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}'
|
||||
substrings of the template with the actual user name, bind DN, and base DN during each LDAP search.
|
||||
Note, that the special characters must be escaped properly in XML.
|
||||
attribute - attribute name whose values will be returned by the LDAP search.
|
||||
prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by
|
||||
the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated
|
||||
as local role names. Empty, by default.
|
||||
Example:
|
||||
<ldap>
|
||||
<server>my_ldap_server</server>
|
||||
@ -385,6 +404,13 @@
|
||||
<my_local_role1 />
|
||||
<my_local_role2 />
|
||||
</roles>
|
||||
<role_mapping>
|
||||
<base_dn>ou=groups,dc=example,dc=com</base_dn>
|
||||
<scope>subtree</scope>
|
||||
<search_filter>(&(objectClass=groupOfNames)(member={bind_dn}))</search_filter>
|
||||
<attribute>cn</attribute>
|
||||
<prefix>clickhouse_</prefix>
|
||||
</role_mapping>
|
||||
</ldap>
|
||||
-->
|
||||
</user_directories>
|
||||
|
||||
@ -33,13 +33,13 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const
|
||||
}
|
||||
|
||||
case SHA256_PASSWORD:
|
||||
throw Exception("Cannot get password double SHA1 for user with 'SHA256_PASSWORD' authentication.", ErrorCodes::BAD_ARGUMENTS);
|
||||
throw Exception("Cannot get password double SHA1 for user with 'SHA256_PASSWORD' authentication", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
case DOUBLE_SHA1_PASSWORD:
|
||||
return password_hash;
|
||||
|
||||
case LDAP_SERVER:
|
||||
throw Exception("Cannot get password double SHA1 for user with 'LDAP_SERVER' authentication.", ErrorCodes::BAD_ARGUMENTS);
|
||||
throw Exception("Cannot get password double SHA1 for user with 'LDAP_SERVER' authentication", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
case MAX_TYPE:
|
||||
break;
|
||||
|
||||
@ -6,6 +6,8 @@
|
||||
#include <Poco/SHA1Engine.h>
|
||||
#include <boost/algorithm/hex.hpp>
|
||||
#include <boost/algorithm/string/case_conv.hpp>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -19,6 +21,10 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
class ExternalAuthenticators;
|
||||
struct LDAPSearchParams;
|
||||
using LDAPSearchParamsList = std::vector<LDAPSearchParams>;
|
||||
using LDAPSearchResults = std::set<String>;
|
||||
using LDAPSearchResultsList = std::vector<LDAPSearchResults>;
|
||||
|
||||
/// Authentication type and encrypted password for checking when an user logins.
|
||||
class Authentication
|
||||
|
||||
@ -31,6 +31,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
|
||||
|
||||
const bool has_host = config.has(ldap_server_config + ".host");
|
||||
const bool has_port = config.has(ldap_server_config + ".port");
|
||||
const bool has_bind_dn = config.has(ldap_server_config + ".bind_dn");
|
||||
const bool has_auth_dn_prefix = config.has(ldap_server_config + ".auth_dn_prefix");
|
||||
const bool has_auth_dn_suffix = config.has(ldap_server_config + ".auth_dn_suffix");
|
||||
const bool has_verification_cooldown = config.has(ldap_server_config + ".verification_cooldown");
|
||||
@ -51,11 +52,19 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
|
||||
if (params.host.empty())
|
||||
throw Exception("Empty 'host' entry", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (has_auth_dn_prefix)
|
||||
params.auth_dn_prefix = config.getString(ldap_server_config + ".auth_dn_prefix");
|
||||
if (has_bind_dn)
|
||||
{
|
||||
if (has_auth_dn_prefix || has_auth_dn_suffix)
|
||||
throw Exception("Deprecated 'auth_dn_prefix' and 'auth_dn_suffix' entries cannot be used with 'bind_dn' entry", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (has_auth_dn_suffix)
|
||||
params.auth_dn_suffix = config.getString(ldap_server_config + ".auth_dn_suffix");
|
||||
params.bind_dn = config.getString(ldap_server_config + ".bind_dn");
|
||||
}
|
||||
else if (has_auth_dn_prefix || has_auth_dn_suffix)
|
||||
{
|
||||
const auto auth_dn_prefix = config.getString(ldap_server_config + ".auth_dn_prefix");
|
||||
const auto auth_dn_suffix = config.getString(ldap_server_config + ".auth_dn_suffix");
|
||||
params.bind_dn = auth_dn_prefix + "{user_name}" + auth_dn_suffix;
|
||||
}
|
||||
|
||||
if (has_verification_cooldown)
|
||||
params.verification_cooldown = std::chrono::seconds{config.getUInt64(ldap_server_config + ".verification_cooldown")};
|
||||
@ -168,7 +177,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur
|
||||
}
|
||||
}
|
||||
|
||||
bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const String & user_name, const String & password) const
|
||||
bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const String & user_name, const String & password,
|
||||
const LDAPSearchParamsList * search_params, LDAPSearchResultsList * search_results) const
|
||||
{
|
||||
std::optional<LDAPServerParams> params;
|
||||
std::size_t params_hash = 0;
|
||||
@ -184,7 +194,15 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S
|
||||
params = pit->second;
|
||||
params->user = user_name;
|
||||
params->password = password;
|
||||
params_hash = params->getCoreHash();
|
||||
|
||||
params->combineCoreHash(params_hash);
|
||||
if (search_params)
|
||||
{
|
||||
for (const auto & params_instance : *search_params)
|
||||
{
|
||||
params_instance.combineHash(params_hash);
|
||||
}
|
||||
}
|
||||
|
||||
// Check the cache, but only if the caching is enabled at all.
|
||||
if (params->verification_cooldown > std::chrono::seconds{0})
|
||||
@ -208,9 +226,19 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S
|
||||
// Check if we can safely "reuse" the result of the previous successful password verification.
|
||||
entry.last_successful_params_hash == params_hash &&
|
||||
last_check_period >= std::chrono::seconds{0} &&
|
||||
last_check_period <= params->verification_cooldown
|
||||
last_check_period <= params->verification_cooldown &&
|
||||
|
||||
// Ensure that search_params are compatible.
|
||||
(
|
||||
search_params == nullptr ?
|
||||
entry.last_successful_search_results.empty() :
|
||||
search_params->size() == entry.last_successful_search_results.size()
|
||||
)
|
||||
)
|
||||
{
|
||||
if (search_results)
|
||||
*search_results = entry.last_successful_search_results;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -227,7 +255,7 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S
|
||||
}
|
||||
|
||||
LDAPSimpleAuthClient client(params.value());
|
||||
const auto result = client.check();
|
||||
const auto result = client.authenticate(search_params, search_results);
|
||||
const auto current_check_timestamp = std::chrono::steady_clock::now();
|
||||
|
||||
// Update the cache, but only if this is the latest check and the server is still configured in a compatible way.
|
||||
@ -244,8 +272,18 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S
|
||||
new_params.user = user_name;
|
||||
new_params.password = password;
|
||||
|
||||
std::size_t new_params_hash = 0;
|
||||
new_params.combineCoreHash(new_params_hash);
|
||||
if (search_params)
|
||||
{
|
||||
for (const auto & params_instance : *search_params)
|
||||
{
|
||||
params_instance.combineHash(new_params_hash);
|
||||
}
|
||||
}
|
||||
|
||||
// If the critical server params have changed while we were checking the password, we discard the current result.
|
||||
if (params_hash != new_params.getCoreHash())
|
||||
if (params_hash != new_params_hash)
|
||||
return false;
|
||||
|
||||
auto & entry = ldap_server_caches[server][user_name];
|
||||
@ -253,8 +291,20 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S
|
||||
{
|
||||
entry.last_successful_params_hash = params_hash;
|
||||
entry.last_successful_authentication_timestamp = current_check_timestamp;
|
||||
|
||||
if (search_results)
|
||||
entry.last_successful_search_results = *search_results;
|
||||
else
|
||||
entry.last_successful_search_results.clear();
|
||||
}
|
||||
else if (entry.last_successful_params_hash != params_hash)
|
||||
else if (
|
||||
entry.last_successful_params_hash != params_hash ||
|
||||
(
|
||||
search_params == nullptr ?
|
||||
!entry.last_successful_search_results.empty() :
|
||||
search_params->size() != entry.last_successful_search_results.size()
|
||||
)
|
||||
)
|
||||
{
|
||||
// Somehow a newer check with different params/password succeeded, so the current result is obsolete and we discard it.
|
||||
return false;
|
||||
|
||||
@ -28,13 +28,15 @@ class ExternalAuthenticators
|
||||
public:
|
||||
void reset();
|
||||
void setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log);
|
||||
bool checkLDAPCredentials(const String & server, const String & user_name, const String & password) const;
|
||||
bool checkLDAPCredentials(const String & server, const String & user_name, const String & password,
|
||||
const LDAPSearchParamsList * search_params = nullptr, LDAPSearchResultsList * search_results = nullptr) const;
|
||||
|
||||
private:
|
||||
struct LDAPCacheEntry
|
||||
{
|
||||
std::size_t last_successful_params_hash = 0;
|
||||
std::chrono::steady_clock::time_point last_successful_authentication_timestamp;
|
||||
LDAPSearchResultsList last_successful_search_results;
|
||||
};
|
||||
|
||||
using LDAPServerCache = std::unordered_map<String, LDAPCacheEntry>; // user name -> cache entry
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
#include <Access/LDAPAccessStorage.h>
|
||||
#include <Access/AccessControlManager.h>
|
||||
#include <Access/ExternalAuthenticators.h>
|
||||
#include <Access/User.h>
|
||||
#include <Access/Role.h>
|
||||
#include <Access/LDAPClient.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <ext/scope_guard.h>
|
||||
@ -9,9 +11,11 @@
|
||||
#include <Poco/JSON/JSON.h>
|
||||
#include <Poco/JSON/Object.h>
|
||||
#include <Poco/JSON/Stringifier.h>
|
||||
#include <boost/container_hash/hash.hpp>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
#include <iterator>
|
||||
#include <sstream>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -44,95 +48,327 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m
|
||||
|
||||
const bool has_server = config.has(prefix_str + "server");
|
||||
const bool has_roles = config.has(prefix_str + "roles");
|
||||
const bool has_role_mapping = config.has(prefix_str + "role_mapping");
|
||||
|
||||
if (!has_server)
|
||||
throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS);
|
||||
throw Exception("Missing 'server' field for LDAP user directory", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
const auto ldap_server_cfg = config.getString(prefix_str + "server");
|
||||
if (ldap_server_cfg.empty())
|
||||
throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS);
|
||||
throw Exception("Empty 'server' field for LDAP user directory", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
std::set<String> roles_cfg;
|
||||
std::set<String> common_roles_cfg;
|
||||
if (has_roles)
|
||||
{
|
||||
Poco::Util::AbstractConfiguration::Keys role_names;
|
||||
config.keys(prefix_str + "roles", role_names);
|
||||
|
||||
// Currently, we only extract names of roles from the section names and assign them directly and unconditionally.
|
||||
roles_cfg.insert(role_names.begin(), role_names.end());
|
||||
common_roles_cfg.insert(role_names.begin(), role_names.end());
|
||||
}
|
||||
|
||||
LDAPSearchParamsList role_search_params_cfg;
|
||||
if (has_role_mapping)
|
||||
{
|
||||
Poco::Util::AbstractConfiguration::Keys all_keys;
|
||||
config.keys(prefix, all_keys);
|
||||
for (const auto & key : all_keys)
|
||||
{
|
||||
if (key != "role_mapping" && key.find("role_mapping[") != 0)
|
||||
continue;
|
||||
|
||||
const String rm_prefix = prefix_str + key;
|
||||
const String rm_prefix_str = rm_prefix + '.';
|
||||
role_search_params_cfg.emplace_back();
|
||||
auto & rm_params = role_search_params_cfg.back();
|
||||
|
||||
rm_params.base_dn = config.getString(rm_prefix_str + "base_dn", "");
|
||||
rm_params.search_filter = config.getString(rm_prefix_str + "search_filter", "");
|
||||
rm_params.attribute = config.getString(rm_prefix_str + "attribute", "cn");
|
||||
rm_params.prefix = config.getString(rm_prefix_str + "prefix", "");
|
||||
|
||||
auto scope = config.getString(rm_prefix_str + "scope", "subtree");
|
||||
boost::algorithm::to_lower(scope);
|
||||
if (scope == "base") rm_params.scope = LDAPSearchParams::Scope::BASE;
|
||||
else if (scope == "one_level") rm_params.scope = LDAPSearchParams::Scope::ONE_LEVEL;
|
||||
else if (scope == "subtree") rm_params.scope = LDAPSearchParams::Scope::SUBTREE;
|
||||
else if (scope == "children") rm_params.scope = LDAPSearchParams::Scope::CHILDREN;
|
||||
else
|
||||
throw Exception("Invalid value of 'scope' field in '" + key + "' section of LDAP user directory, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
}
|
||||
|
||||
access_control_manager = access_control_manager_;
|
||||
ldap_server = ldap_server_cfg;
|
||||
default_role_names.swap(roles_cfg);
|
||||
roles_of_interest.clear();
|
||||
role_search_params.swap(role_search_params_cfg);
|
||||
common_role_names.swap(common_roles_cfg);
|
||||
|
||||
external_role_hashes.clear();
|
||||
users_per_roles.clear();
|
||||
roles_per_users.clear();
|
||||
granted_role_names.clear();
|
||||
granted_role_ids.clear();
|
||||
|
||||
role_change_subscription = access_control_manager->subscribeForChanges<Role>(
|
||||
[this] (const UUID & id, const AccessEntityPtr & entity)
|
||||
{
|
||||
return this->processRoleChange(id, entity);
|
||||
}
|
||||
);
|
||||
|
||||
/// Update `roles_of_interests` with initial values.
|
||||
for (const auto & role_name : default_role_names)
|
||||
{
|
||||
if (auto role_id = access_control_manager->find<Role>(role_name))
|
||||
roles_of_interest.emplace(*role_id, role_name);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity)
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
const auto role = typeid_cast<std::shared_ptr<const Role>>(entity);
|
||||
const auto it = granted_role_names.find(id);
|
||||
|
||||
/// Update `roles_of_interests`.
|
||||
auto role = typeid_cast<std::shared_ptr<const Role>>(entity);
|
||||
bool need_to_update_users = false;
|
||||
|
||||
if (role && default_role_names.count(role->getName()))
|
||||
if (role) // Added or renamed a role.
|
||||
{
|
||||
/// If a role was created with one of the `default_role_names` or renamed to one of the `default_role_names`,
|
||||
/// then set `need_to_update_users`.
|
||||
need_to_update_users = roles_of_interest.insert_or_assign(id, role->getName()).second;
|
||||
const auto & new_role_name = role->getName();
|
||||
if (it != granted_role_names.end()) // Renamed a granted role.
|
||||
{
|
||||
const auto & old_role_name = it->second;
|
||||
if (new_role_name != old_role_name)
|
||||
{
|
||||
// Revoke the old role first, then grant the new role.
|
||||
applyRoleChangeNoLock(false /* revoke */, id, old_role_name);
|
||||
applyRoleChangeNoLock(true /* grant */, id, new_role_name);
|
||||
}
|
||||
}
|
||||
else // Added a role.
|
||||
{
|
||||
applyRoleChangeNoLock(true /* grant */, id, new_role_name);
|
||||
}
|
||||
}
|
||||
else // Removed a role.
|
||||
{
|
||||
if (it != granted_role_names.end()) // Removed a granted role.
|
||||
{
|
||||
const auto & old_role_name = it->second;
|
||||
applyRoleChangeNoLock(false /* revoke */, id, old_role_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LDAPAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name)
|
||||
{
|
||||
std::vector<UUID> user_ids;
|
||||
|
||||
// Build a list of ids of the relevant users.
|
||||
if (common_role_names.count(role_name))
|
||||
{
|
||||
user_ids = memory_storage.findAll<User>();
|
||||
}
|
||||
else
|
||||
{
|
||||
/// If a role was removed or renamed to a name which isn't contained in the `default_role_names`,
|
||||
/// then set `need_to_update_users`.
|
||||
need_to_update_users = roles_of_interest.erase(id) > 0;
|
||||
const auto it = users_per_roles.find(role_name);
|
||||
if (it != users_per_roles.end())
|
||||
{
|
||||
const auto & user_names = it->second;
|
||||
user_ids.reserve(user_names.size());
|
||||
|
||||
for (const auto & user_name : user_names)
|
||||
{
|
||||
if (const auto user_id = memory_storage.find<User>(user_name))
|
||||
user_ids.emplace_back(*user_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Update users which have been created.
|
||||
if (need_to_update_users)
|
||||
// Update the granted roles of the relevant users.
|
||||
if (!user_ids.empty())
|
||||
{
|
||||
auto update_func = [this] (const AccessEntityPtr & entity_) -> AccessEntityPtr
|
||||
auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_) -> AccessEntityPtr
|
||||
{
|
||||
if (auto user = typeid_cast<std::shared_ptr<const User>>(entity_))
|
||||
{
|
||||
auto changed_user = typeid_cast<std::shared_ptr<User>>(user->clone());
|
||||
auto & granted_roles = changed_user->granted_roles.roles;
|
||||
granted_roles.clear();
|
||||
boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end()));
|
||||
|
||||
if (grant)
|
||||
granted_roles.insert(role_id);
|
||||
else
|
||||
granted_roles.erase(role_id);
|
||||
|
||||
return changed_user;
|
||||
}
|
||||
return entity_;
|
||||
};
|
||||
memory_storage.update(memory_storage.findAll<User>(), update_func);
|
||||
|
||||
memory_storage.update(user_ids, update_func);
|
||||
}
|
||||
|
||||
// Actualize granted_role_* mappings.
|
||||
if (grant)
|
||||
{
|
||||
if (!user_ids.empty())
|
||||
{
|
||||
granted_role_names.insert_or_assign(role_id, role_name);
|
||||
granted_role_ids.insert_or_assign(role_name, role_id);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
granted_role_names.erase(role_id);
|
||||
granted_role_ids.erase(role_name);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LDAPAccessStorage::checkAllDefaultRoleNamesFoundNoLock() const
|
||||
void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPSearchResultsList & external_roles) const
|
||||
{
|
||||
boost::container::flat_set<std::string_view> role_names_of_interest;
|
||||
boost::range::copy(roles_of_interest | boost::adaptors::map_values, std::inserter(role_names_of_interest, role_names_of_interest.end()));
|
||||
const auto external_roles_hash = boost::hash<LDAPSearchResultsList>{}(external_roles);
|
||||
return assignRolesNoLock(user, external_roles, external_roles_hash);
|
||||
}
|
||||
|
||||
for (const auto & role_name : default_role_names)
|
||||
|
||||
void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPSearchResultsList & external_roles, const std::size_t external_roles_hash) const
|
||||
{
|
||||
const auto & user_name = user.getName();
|
||||
auto & granted_roles = user.granted_roles.roles;
|
||||
const auto local_role_names = mapExternalRolesNoLock(external_roles);
|
||||
|
||||
auto grant_role = [this, &user_name, &granted_roles] (const String & role_name, const bool common)
|
||||
{
|
||||
if (!role_names_of_interest.count(role_name))
|
||||
throwDefaultRoleNotFound(role_name);
|
||||
auto it = granted_role_ids.find(role_name);
|
||||
if (it == granted_role_ids.end())
|
||||
{
|
||||
if (const auto role_id = access_control_manager->find<Role>(role_name))
|
||||
{
|
||||
granted_role_names.insert_or_assign(*role_id, role_name);
|
||||
it = granted_role_ids.insert_or_assign(role_name, *role_id).first;
|
||||
}
|
||||
}
|
||||
|
||||
if (it != granted_role_ids.end())
|
||||
{
|
||||
const auto & role_id = it->second;
|
||||
granted_roles.insert(role_id);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_WARNING(getLogger(), "Unable to grant {} role '{}' to user '{}': role not found", (common ? "common" : "mapped"), role_name, user_name);
|
||||
}
|
||||
};
|
||||
|
||||
external_role_hashes.erase(user_name);
|
||||
granted_roles.clear();
|
||||
const auto old_role_names = std::move(roles_per_users[user_name]);
|
||||
|
||||
// Grant the common roles first.
|
||||
for (const auto & role_name : common_role_names)
|
||||
{
|
||||
grant_role(role_name, true /* common */);
|
||||
}
|
||||
|
||||
// Grant the mapped external roles and actualize users_per_roles mapping.
|
||||
// local_role_names allowed to overlap with common_role_names.
|
||||
for (const auto & role_name : local_role_names)
|
||||
{
|
||||
grant_role(role_name, false /* mapped */);
|
||||
users_per_roles[role_name].insert(user_name);
|
||||
}
|
||||
|
||||
// Cleanup users_per_roles and granted_role_* mappings.
|
||||
for (const auto & old_role_name : old_role_names)
|
||||
{
|
||||
if (local_role_names.count(old_role_name))
|
||||
continue;
|
||||
|
||||
const auto rit = users_per_roles.find(old_role_name);
|
||||
if (rit == users_per_roles.end())
|
||||
continue;
|
||||
|
||||
auto & user_names = rit->second;
|
||||
user_names.erase(user_name);
|
||||
|
||||
if (!user_names.empty())
|
||||
continue;
|
||||
|
||||
users_per_roles.erase(rit);
|
||||
|
||||
if (common_role_names.count(old_role_name))
|
||||
continue;
|
||||
|
||||
const auto iit = granted_role_ids.find(old_role_name);
|
||||
if (iit == granted_role_ids.end())
|
||||
continue;
|
||||
|
||||
const auto old_role_id = iit->second;
|
||||
granted_role_names.erase(old_role_id);
|
||||
granted_role_ids.erase(iit);
|
||||
}
|
||||
|
||||
// Actualize roles_per_users mapping and external_role_hashes cache.
|
||||
if (local_role_names.empty())
|
||||
roles_per_users.erase(user_name);
|
||||
else
|
||||
roles_per_users[user_name] = std::move(local_role_names);
|
||||
|
||||
external_role_hashes[user_name] = external_roles_hash;
|
||||
}
|
||||
|
||||
|
||||
void LDAPAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPSearchResultsList & external_roles) const
|
||||
{
|
||||
// No need to include common_role_names in this hash each time, since they don't change.
|
||||
const auto external_roles_hash = boost::hash<LDAPSearchResultsList>{}(external_roles);
|
||||
|
||||
// Map and grant the roles from scratch only if the list of external role has changed.
|
||||
const auto it = external_role_hashes.find(user_name);
|
||||
if (it != external_role_hashes.end() && it->second == external_roles_hash)
|
||||
return;
|
||||
|
||||
auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_) -> AccessEntityPtr
|
||||
{
|
||||
if (auto user = typeid_cast<std::shared_ptr<const User>>(entity_))
|
||||
{
|
||||
auto changed_user = typeid_cast<std::shared_ptr<User>>(user->clone());
|
||||
assignRolesNoLock(*changed_user, external_roles, external_roles_hash);
|
||||
return changed_user;
|
||||
}
|
||||
return entity_;
|
||||
};
|
||||
|
||||
memory_storage.update(id, update_func);
|
||||
}
|
||||
|
||||
|
||||
std::set<String> LDAPAccessStorage::mapExternalRolesNoLock(const LDAPSearchResultsList & external_roles) const
|
||||
{
|
||||
std::set<String> role_names;
|
||||
|
||||
if (external_roles.size() != role_search_params.size())
|
||||
throw Exception("Unable to map external roles", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
for (std::size_t i = 0; i < external_roles.size(); ++i)
|
||||
{
|
||||
const auto & external_role_set = external_roles[i];
|
||||
const auto & prefix = role_search_params[i].prefix;
|
||||
|
||||
for (const auto & external_role : external_role_set)
|
||||
{
|
||||
if (
|
||||
prefix.size() < external_role.size() &&
|
||||
external_role.compare(0, prefix.size(), prefix) == 0
|
||||
)
|
||||
{
|
||||
role_names.emplace(external_role, prefix.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return role_names;
|
||||
}
|
||||
|
||||
|
||||
bool LDAPAccessStorage::isPasswordCorrectLDAPNoLock(const String & user_name, const String & password,
|
||||
const ExternalAuthenticators & external_authenticators, LDAPSearchResultsList & search_results) const
|
||||
{
|
||||
return external_authenticators.checkLDAPCredentials(ldap_server, user_name, password, &role_search_params, &search_results);
|
||||
}
|
||||
|
||||
|
||||
@ -148,7 +384,37 @@ String LDAPAccessStorage::getStorageParamsJSON() const
|
||||
Poco::JSON::Object params_json;
|
||||
|
||||
params_json.set("server", ldap_server);
|
||||
params_json.set("roles", default_role_names);
|
||||
|
||||
Poco::JSON::Array common_role_names_json;
|
||||
for (const auto & role : common_role_names)
|
||||
{
|
||||
common_role_names_json.add(role);
|
||||
}
|
||||
params_json.set("roles", common_role_names_json);
|
||||
|
||||
Poco::JSON::Array role_mappings_json;
|
||||
for (const auto & role_mapping : role_search_params)
|
||||
{
|
||||
Poco::JSON::Object role_mapping_json;
|
||||
|
||||
role_mapping_json.set("base_dn", role_mapping.base_dn);
|
||||
role_mapping_json.set("search_filter", role_mapping.search_filter);
|
||||
role_mapping_json.set("attribute", role_mapping.attribute);
|
||||
role_mapping_json.set("prefix", role_mapping.prefix);
|
||||
|
||||
String scope;
|
||||
switch (role_mapping.scope)
|
||||
{
|
||||
case LDAPSearchParams::Scope::BASE: scope = "base"; break;
|
||||
case LDAPSearchParams::Scope::ONE_LEVEL: scope = "one_level"; break;
|
||||
case LDAPSearchParams::Scope::SUBTREE: scope = "subtree"; break;
|
||||
case LDAPSearchParams::Scope::CHILDREN: scope = "children"; break;
|
||||
}
|
||||
role_mapping_json.set("scope", scope);
|
||||
|
||||
role_mappings_json.add(role_mapping_json);
|
||||
}
|
||||
params_json.set("role_mappings", role_mappings_json);
|
||||
|
||||
std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
|
||||
oss.exceptions(std::ios::failbit);
|
||||
@ -251,17 +517,21 @@ bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const
|
||||
UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
LDAPSearchResultsList external_roles;
|
||||
auto id = memory_storage.find<User>(user_name);
|
||||
if (id)
|
||||
{
|
||||
auto user = memory_storage.read<User>(*id);
|
||||
|
||||
if (!isPasswordCorrectImpl(*user, password, external_authenticators))
|
||||
if (!isPasswordCorrectLDAPNoLock(user->getName(), password, external_authenticators, external_roles))
|
||||
throwInvalidPassword();
|
||||
|
||||
if (!isAddressAllowedImpl(*user, address))
|
||||
throwAddressNotAllowed(address);
|
||||
|
||||
// Just in case external_roles are changed. This will be no-op if they are not.
|
||||
updateAssignedRolesNoLock(*id, user_name, external_roles);
|
||||
|
||||
return *id;
|
||||
}
|
||||
else
|
||||
@ -272,16 +542,13 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw
|
||||
user->authentication = Authentication(Authentication::Type::LDAP_SERVER);
|
||||
user->authentication.setServerName(ldap_server);
|
||||
|
||||
if (!isPasswordCorrectImpl(*user, password, external_authenticators))
|
||||
if (!isPasswordCorrectLDAPNoLock(user->getName(), password, external_authenticators, external_roles))
|
||||
throwInvalidPassword();
|
||||
|
||||
if (!isAddressAllowedImpl(*user, address))
|
||||
throwAddressNotAllowed(address);
|
||||
|
||||
checkAllDefaultRoleNamesFoundNoLock();
|
||||
|
||||
auto & granted_roles = user->granted_roles.roles;
|
||||
boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end()));
|
||||
assignRolesNoLock(*user, external_roles);
|
||||
|
||||
return memory_storage.insert(user);
|
||||
}
|
||||
@ -303,18 +570,14 @@ UUID LDAPAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const
|
||||
user->authentication = Authentication(Authentication::Type::LDAP_SERVER);
|
||||
user->authentication.setServerName(ldap_server);
|
||||
|
||||
checkAllDefaultRoleNamesFoundNoLock();
|
||||
LDAPSearchResultsList external_roles;
|
||||
|
||||
auto & granted_roles = user->granted_roles.roles;
|
||||
boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end()));
|
||||
// TODO: mapped external roles are not available here. Without a password we can't authenticate and retrieve roles from LDAP server.
|
||||
|
||||
assignRolesNoLock(*user, external_roles);
|
||||
|
||||
return memory_storage.insert(user);
|
||||
}
|
||||
}
|
||||
|
||||
void LDAPAccessStorage::throwDefaultRoleNotFound(const String & role_name)
|
||||
{
|
||||
throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace Poco
|
||||
@ -20,6 +21,10 @@ namespace Poco
|
||||
namespace DB
|
||||
{
|
||||
class AccessControlManager;
|
||||
struct LDAPSearchParams;
|
||||
using LDAPSearchParamsList = std::vector<LDAPSearchParams>;
|
||||
using LDAPSearchResults = std::set<String>;
|
||||
using LDAPSearchResultsList = std::vector<LDAPSearchResults>;
|
||||
|
||||
/// Implementation of IAccessStorage which allows attaching users from a remote LDAP server.
|
||||
/// Currently, any user name will be treated as a name of an existing remote user,
|
||||
@ -58,15 +63,25 @@ private: // IAccessStorage implementations.
|
||||
private:
|
||||
void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix);
|
||||
void processRoleChange(const UUID & id, const AccessEntityPtr & entity);
|
||||
void checkAllDefaultRoleNamesFoundNoLock() const;
|
||||
|
||||
[[noreturn]] static void throwDefaultRoleNotFound(const String & role_name);
|
||||
void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name);
|
||||
void assignRolesNoLock(User & user, const LDAPSearchResultsList & external_roles) const;
|
||||
void assignRolesNoLock(User & user, const LDAPSearchResultsList & external_roles, const std::size_t external_roles_hash) const;
|
||||
void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPSearchResultsList & external_roles) const;
|
||||
std::set<String> mapExternalRolesNoLock(const LDAPSearchResultsList & external_roles) const;
|
||||
bool isPasswordCorrectLDAPNoLock(const String & user_name, const String & password,
|
||||
const ExternalAuthenticators & external_authenticators, LDAPSearchResultsList & search_results) const;
|
||||
|
||||
mutable std::recursive_mutex mutex;
|
||||
AccessControlManager * access_control_manager = nullptr;
|
||||
String ldap_server;
|
||||
std::set<String> default_role_names;
|
||||
std::map<UUID, String> roles_of_interest;
|
||||
LDAPSearchParamsList role_search_params;
|
||||
std::set<String> common_role_names; // role name that should be granted to all users at all times
|
||||
mutable std::map<String, std::size_t> external_role_hashes; // user name -> LDAPSearchResultsList hash (most recently retrieved and processed)
|
||||
mutable std::map<String, std::set<String>> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles)
|
||||
mutable std::map<String, std::set<String>> roles_per_users; // user name -> role names (...that should be granted to it; may but don't have to include common roles)
|
||||
mutable std::map<UUID, String> granted_role_names; // (currently granted) role id -> its name
|
||||
mutable std::map<String, UUID> granted_role_ids; // (currently granted) role name -> its id
|
||||
ext::scope_guard role_change_subscription;
|
||||
mutable MemoryAccessStorage memory_storage;
|
||||
};
|
||||
|
||||
@ -1,8 +1,14 @@
|
||||
#include <Access/LDAPClient.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <ext/scope_guard.h>
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <Poco/Logger.h>
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
|
||||
#include <mutex>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <cstring>
|
||||
|
||||
@ -63,38 +69,63 @@ namespace
|
||||
return dest;
|
||||
}
|
||||
|
||||
auto replacePlaceholders(const String & src, const std::vector<std::pair<String, String>> & pairs)
|
||||
{
|
||||
String dest = src;
|
||||
|
||||
for (const auto & pair : pairs)
|
||||
{
|
||||
const auto & placeholder = pair.first;
|
||||
const auto & value = pair.second;
|
||||
for (
|
||||
auto pos = dest.find(placeholder);
|
||||
pos != std::string::npos;
|
||||
pos = dest.find(placeholder, pos)
|
||||
)
|
||||
{
|
||||
dest.replace(pos, placeholder.size(), value);
|
||||
pos += value.size();
|
||||
}
|
||||
}
|
||||
|
||||
return dest;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void LDAPClient::diag(const int rc)
|
||||
void LDAPClient::diag(const int rc, String text)
|
||||
{
|
||||
std::scoped_lock lock(ldap_global_mutex);
|
||||
|
||||
if (rc != LDAP_SUCCESS)
|
||||
{
|
||||
String text;
|
||||
const char * raw_err_str = ldap_err2string(rc);
|
||||
|
||||
if (raw_err_str)
|
||||
text = raw_err_str;
|
||||
if (raw_err_str && *raw_err_str != '\0')
|
||||
{
|
||||
if (!text.empty())
|
||||
text += ": ";
|
||||
text += raw_err_str;
|
||||
}
|
||||
|
||||
if (handle)
|
||||
{
|
||||
String message;
|
||||
char * raw_message = nullptr;
|
||||
|
||||
SCOPE_EXIT({
|
||||
if (raw_message)
|
||||
{
|
||||
ldap_memfree(raw_message);
|
||||
raw_message = nullptr;
|
||||
}
|
||||
});
|
||||
|
||||
ldap_get_option(handle, LDAP_OPT_DIAGNOSTIC_MESSAGE, &raw_message);
|
||||
|
||||
if (raw_message)
|
||||
{
|
||||
message = raw_message;
|
||||
ldap_memfree(raw_message);
|
||||
raw_message = nullptr;
|
||||
}
|
||||
|
||||
if (!message.empty())
|
||||
if (raw_message && *raw_message != '\0')
|
||||
{
|
||||
if (!text.empty())
|
||||
text += ": ";
|
||||
text += message;
|
||||
text += raw_message;
|
||||
}
|
||||
}
|
||||
|
||||
@ -240,20 +271,20 @@ void LDAPClient::openConnection()
|
||||
{
|
||||
case LDAPServerParams::SASLMechanism::SIMPLE:
|
||||
{
|
||||
const String dn = params.auth_dn_prefix + escapeForLDAP(params.user) + params.auth_dn_suffix;
|
||||
const auto escaped_user_name = escapeForLDAP(params.user);
|
||||
const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} });
|
||||
|
||||
::berval cred;
|
||||
cred.bv_val = const_cast<char *>(params.password.c_str());
|
||||
cred.bv_len = params.password.size();
|
||||
|
||||
diag(ldap_sasl_bind_s(handle, dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr));
|
||||
diag(ldap_sasl_bind_s(handle, bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr));
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
{
|
||||
throw Exception("Unknown SASL mechanism", ErrorCodes::LDAP_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -268,13 +299,167 @@ void LDAPClient::closeConnection() noexcept
|
||||
handle = nullptr;
|
||||
}
|
||||
|
||||
bool LDAPSimpleAuthClient::check()
|
||||
LDAPSearchResults LDAPClient::search(const LDAPSearchParams & search_params)
|
||||
{
|
||||
std::scoped_lock lock(ldap_global_mutex);
|
||||
|
||||
LDAPSearchResults result;
|
||||
|
||||
int scope = 0;
|
||||
switch (search_params.scope)
|
||||
{
|
||||
case LDAPSearchParams::Scope::BASE: scope = LDAP_SCOPE_BASE; break;
|
||||
case LDAPSearchParams::Scope::ONE_LEVEL: scope = LDAP_SCOPE_ONELEVEL; break;
|
||||
case LDAPSearchParams::Scope::SUBTREE: scope = LDAP_SCOPE_SUBTREE; break;
|
||||
case LDAPSearchParams::Scope::CHILDREN: scope = LDAP_SCOPE_CHILDREN; break;
|
||||
}
|
||||
|
||||
const auto escaped_user_name = escapeForLDAP(params.user);
|
||||
const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} });
|
||||
const auto base_dn = replacePlaceholders(search_params.base_dn, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn} });
|
||||
const auto search_filter = replacePlaceholders(search_params.search_filter, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn}, {"{base_dn}", base_dn} });
|
||||
char * attrs[] = { const_cast<char *>(search_params.attribute.c_str()), nullptr };
|
||||
::timeval timeout = { params.search_timeout.count(), 0 };
|
||||
LDAPMessage* msgs = nullptr;
|
||||
|
||||
SCOPE_EXIT({
|
||||
if (msgs)
|
||||
{
|
||||
ldap_msgfree(msgs);
|
||||
msgs = nullptr;
|
||||
}
|
||||
});
|
||||
|
||||
diag(ldap_search_ext_s(handle, base_dn.c_str(), scope, search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs));
|
||||
|
||||
for (
|
||||
auto * msg = ldap_first_message(handle, msgs);
|
||||
msg != nullptr;
|
||||
msg = ldap_next_message(handle, msg)
|
||||
)
|
||||
{
|
||||
switch (ldap_msgtype(msg))
|
||||
{
|
||||
case LDAP_RES_SEARCH_ENTRY:
|
||||
{
|
||||
BerElement * ber = nullptr;
|
||||
|
||||
SCOPE_EXIT({
|
||||
if (ber)
|
||||
{
|
||||
ber_free(ber, 0);
|
||||
ber = nullptr;
|
||||
}
|
||||
});
|
||||
|
||||
for (
|
||||
auto * attr = ldap_first_attribute(handle, msg, &ber);
|
||||
attr != nullptr;
|
||||
attr = ldap_next_attribute(handle, msg, ber)
|
||||
)
|
||||
{
|
||||
SCOPE_EXIT({
|
||||
ldap_memfree(attr);
|
||||
attr = nullptr;
|
||||
});
|
||||
|
||||
if (search_params.attribute.empty() || boost::iequals(attr, search_params.attribute))
|
||||
{
|
||||
auto ** vals = ldap_get_values_len(handle, msg, attr);
|
||||
if (vals)
|
||||
{
|
||||
SCOPE_EXIT({
|
||||
ldap_value_free_len(vals);
|
||||
vals = nullptr;
|
||||
});
|
||||
|
||||
for (std::size_t i = 0; vals[i]; i++)
|
||||
{
|
||||
if (vals[i]->bv_val && vals[i]->bv_len > 0)
|
||||
result.emplace(vals[i]->bv_val, vals[i]->bv_len);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case LDAP_RES_SEARCH_REFERENCE:
|
||||
{
|
||||
char ** referrals = nullptr;
|
||||
diag(ldap_parse_reference(handle, msg, &referrals, nullptr, 0));
|
||||
|
||||
if (referrals)
|
||||
{
|
||||
SCOPE_EXIT({
|
||||
// ldap_value_free(referrals);
|
||||
ber_memvfree(reinterpret_cast<void **>(referrals));
|
||||
referrals = nullptr;
|
||||
});
|
||||
|
||||
for (std::size_t i = 0; referrals[i]; i++)
|
||||
{
|
||||
LOG_WARNING(&Poco::Logger::get("LDAPClient"), "Received reference during LDAP search but not following it: {}", referrals[i]);
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case LDAP_RES_SEARCH_RESULT:
|
||||
{
|
||||
int rc = LDAP_SUCCESS;
|
||||
char * matched_msg = nullptr;
|
||||
char * error_msg = nullptr;
|
||||
|
||||
diag(ldap_parse_result(handle, msg, &rc, &matched_msg, &error_msg, nullptr, nullptr, 0));
|
||||
|
||||
if (rc != LDAP_SUCCESS)
|
||||
{
|
||||
String message = "LDAP search failed";
|
||||
|
||||
const char * raw_err_str = ldap_err2string(rc);
|
||||
if (raw_err_str && *raw_err_str != '\0')
|
||||
{
|
||||
message += ": ";
|
||||
message += raw_err_str;
|
||||
}
|
||||
|
||||
if (error_msg && *error_msg != '\0')
|
||||
{
|
||||
message += ", ";
|
||||
message += error_msg;
|
||||
}
|
||||
|
||||
if (matched_msg && *matched_msg != '\0')
|
||||
{
|
||||
message += ", matching DN part: ";
|
||||
message += matched_msg;
|
||||
}
|
||||
|
||||
throw Exception(message, ErrorCodes::LDAP_ERROR);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case -1:
|
||||
throw Exception("Failed to process LDAP search message", ErrorCodes::LDAP_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
bool LDAPSimpleAuthClient::authenticate(const LDAPSearchParamsList * search_params, LDAPSearchResultsList * search_results)
|
||||
{
|
||||
if (params.user.empty())
|
||||
throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (!search_params != !search_results)
|
||||
throw Exception("Cannot return LDAP search results", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
// Silently reject authentication attempt if the password is empty as if it didn't match.
|
||||
if (params.password.empty())
|
||||
return false;
|
||||
@ -284,12 +469,32 @@ bool LDAPSimpleAuthClient::check()
|
||||
// Will throw on any error, including invalid credentials.
|
||||
openConnection();
|
||||
|
||||
// While connected, run search queries and save the results, if asked.
|
||||
if (search_params)
|
||||
{
|
||||
search_results->clear();
|
||||
search_results->reserve(search_params->size());
|
||||
|
||||
try
|
||||
{
|
||||
for (const auto & single_search_params : *search_params)
|
||||
{
|
||||
search_results->emplace_back(search(single_search_params));
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
search_results->clear();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#else // USE_LDAP
|
||||
|
||||
void LDAPClient::diag(const int)
|
||||
void LDAPClient::diag(const int, String)
|
||||
{
|
||||
throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
|
||||
}
|
||||
@ -303,7 +508,12 @@ void LDAPClient::closeConnection() noexcept
|
||||
{
|
||||
}
|
||||
|
||||
bool LDAPSimpleAuthClient::check()
|
||||
LDAPSearchResults LDAPClient::search(const LDAPSearchParams &)
|
||||
{
|
||||
throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
|
||||
}
|
||||
|
||||
bool LDAPSimpleAuthClient::authenticate(const LDAPSearchParamsList *, LDAPSearchResultsList *)
|
||||
{
|
||||
throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
|
||||
}
|
||||
|
||||
@ -30,9 +30,10 @@ public:
|
||||
LDAPClient & operator= (LDAPClient &&) = delete;
|
||||
|
||||
protected:
|
||||
MAYBE_NORETURN void diag(const int rc);
|
||||
MAYBE_NORETURN void diag(const int rc, String text = "");
|
||||
MAYBE_NORETURN void openConnection();
|
||||
void closeConnection() noexcept;
|
||||
LDAPSearchResults search(const LDAPSearchParams & search_params);
|
||||
|
||||
protected:
|
||||
const LDAPServerParams params;
|
||||
@ -46,7 +47,7 @@ class LDAPSimpleAuthClient
|
||||
{
|
||||
public:
|
||||
using LDAPClient::LDAPClient;
|
||||
bool check();
|
||||
bool authenticate(const LDAPSearchParamsList * search_params, LDAPSearchResultsList * search_results);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -5,11 +5,43 @@
|
||||
#include <boost/container_hash/hash.hpp>
|
||||
|
||||
#include <chrono>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
struct LDAPSearchParams
|
||||
{
|
||||
enum class Scope
|
||||
{
|
||||
BASE,
|
||||
ONE_LEVEL,
|
||||
SUBTREE,
|
||||
CHILDREN
|
||||
};
|
||||
|
||||
String base_dn;
|
||||
Scope scope = Scope::SUBTREE;
|
||||
String search_filter;
|
||||
String attribute = "cn";
|
||||
String prefix;
|
||||
|
||||
void combineHash(std::size_t & seed) const
|
||||
{
|
||||
boost::hash_combine(seed, base_dn);
|
||||
boost::hash_combine(seed, static_cast<int>(scope));
|
||||
boost::hash_combine(seed, search_filter);
|
||||
boost::hash_combine(seed, attribute);
|
||||
boost::hash_combine(seed, prefix);
|
||||
}
|
||||
};
|
||||
|
||||
using LDAPSearchParamsList = std::vector<LDAPSearchParams>;
|
||||
using LDAPSearchResults = std::set<String>;
|
||||
using LDAPSearchResultsList = std::vector<LDAPSearchResults>;
|
||||
|
||||
struct LDAPServerParams
|
||||
{
|
||||
enum class ProtocolVersion
|
||||
@ -64,9 +96,7 @@ struct LDAPServerParams
|
||||
|
||||
SASLMechanism sasl_mechanism = SASLMechanism::SIMPLE;
|
||||
|
||||
String auth_dn_prefix;
|
||||
String auth_dn_suffix;
|
||||
|
||||
String bind_dn;
|
||||
String user;
|
||||
String password;
|
||||
|
||||
@ -77,18 +107,13 @@ struct LDAPServerParams
|
||||
std::chrono::seconds search_timeout{20};
|
||||
std::uint32_t search_limit = 100;
|
||||
|
||||
std::size_t getCoreHash() const
|
||||
void combineCoreHash(std::size_t & seed) const
|
||||
{
|
||||
std::size_t seed = 0;
|
||||
|
||||
boost::hash_combine(seed, host);
|
||||
boost::hash_combine(seed, port);
|
||||
boost::hash_combine(seed, auth_dn_prefix);
|
||||
boost::hash_combine(seed, auth_dn_suffix);
|
||||
boost::hash_combine(seed, bind_dn);
|
||||
boost::hash_combine(seed, user);
|
||||
boost::hash_combine(seed, password);
|
||||
|
||||
return seed;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -229,7 +229,7 @@ public:
|
||||
{
|
||||
for (const auto & x : small)
|
||||
{
|
||||
if (!rb->contains(static_cast<Value>(x.getValue())))
|
||||
if (!r1.rb->contains(static_cast<Value>(x.getValue())))
|
||||
buffer.push_back(x.getValue());
|
||||
}
|
||||
|
||||
|
||||
@ -138,6 +138,7 @@ void Connection::connect(const ConnectionTimeouts & timeouts)
|
||||
|
||||
void Connection::disconnect()
|
||||
{
|
||||
maybe_compressed_out = nullptr;
|
||||
in = nullptr;
|
||||
last_input_packet_type.reset();
|
||||
out = nullptr; // can write to socket
|
||||
|
||||
@ -4,7 +4,6 @@
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/WeakHash.h>
|
||||
#include <Common/HashTable/Hash.h>
|
||||
#include <Core/BigInt.h>
|
||||
|
||||
#include <common/unaligned.h>
|
||||
#include <common/sort.h>
|
||||
|
||||
@ -37,33 +37,16 @@ namespace ErrorCodes
|
||||
template <typename T>
|
||||
StringRef ColumnVector<T>::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const
|
||||
{
|
||||
if constexpr (is_big_int_v<T>)
|
||||
{
|
||||
static constexpr size_t bytesize = BigInt<T>::size;
|
||||
char * pos = arena.allocContinue(bytesize, begin);
|
||||
return BigInt<T>::serialize(data[n], pos);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto * pos = arena.allocContinue(sizeof(T), begin);
|
||||
unalignedStore<T>(pos, data[n]);
|
||||
return StringRef(pos, sizeof(T));
|
||||
}
|
||||
auto * pos = arena.allocContinue(sizeof(T), begin);
|
||||
unalignedStore<T>(pos, data[n]);
|
||||
return StringRef(pos, sizeof(T));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const char * ColumnVector<T>::deserializeAndInsertFromArena(const char * pos)
|
||||
{
|
||||
if constexpr (is_big_int_v<T>)
|
||||
{
|
||||
data.emplace_back(BigInt<T>::deserialize(pos));
|
||||
return pos + BigInt<T>::size;
|
||||
}
|
||||
else
|
||||
{
|
||||
data.emplace_back(unalignedLoad<T>(pos));
|
||||
return pos + sizeof(T);
|
||||
}
|
||||
data.emplace_back(unalignedLoad<T>(pos));
|
||||
return pos + sizeof(T);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -299,18 +282,10 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
|
||||
new_col.data.resize(size);
|
||||
|
||||
size_t count = std::min(this->size(), size);
|
||||
if constexpr (is_POD)
|
||||
{
|
||||
memcpy(new_col.data.data(), data.data(), count * sizeof(data[0]));
|
||||
memcpy(new_col.data.data(), data.data(), count * sizeof(data[0]));
|
||||
|
||||
if (size > count)
|
||||
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(ValueType()), (size - count) * sizeof(ValueType));
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < count; i++)
|
||||
new_col.data[i] = data[i];
|
||||
}
|
||||
if (size > count)
|
||||
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(ValueType()), (size - count) * sizeof(ValueType));
|
||||
}
|
||||
|
||||
return res;
|
||||
@ -348,15 +323,7 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
|
||||
|
||||
size_t old_size = data.size();
|
||||
data.resize(old_size + length);
|
||||
if constexpr (is_POD)
|
||||
{
|
||||
memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0]));
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < length; i++)
|
||||
data[old_size + i] = src_vec.data[start + i];
|
||||
}
|
||||
memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0]));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -372,70 +339,52 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
|
||||
if (result_size_hint)
|
||||
res_data.reserve(result_size_hint > 0 ? result_size_hint : size);
|
||||
|
||||
if constexpr (is_POD)
|
||||
{
|
||||
const UInt8 * filt_pos = filt.data();
|
||||
const UInt8 * filt_end = filt_pos + size;
|
||||
const T * data_pos = data.data();
|
||||
const UInt8 * filt_pos = filt.data();
|
||||
const UInt8 * filt_end = filt_pos + size;
|
||||
const T * data_pos = data.data();
|
||||
|
||||
#ifdef __SSE2__
|
||||
/** A slightly more optimized version.
|
||||
* Based on the assumption that often pieces of consecutive values
|
||||
* completely pass or do not pass the filter.
|
||||
* Therefore, we will optimistically check the parts of `SIMD_BYTES` values.
|
||||
*/
|
||||
/** A slightly more optimized version.
|
||||
* Based on the assumption that often pieces of consecutive values
|
||||
* completely pass or do not pass the filter.
|
||||
* Therefore, we will optimistically check the parts of `SIMD_BYTES` values.
|
||||
*/
|
||||
|
||||
static constexpr size_t SIMD_BYTES = 16;
|
||||
const __m128i zero16 = _mm_setzero_si128();
|
||||
const UInt8 * filt_end_sse = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
|
||||
static constexpr size_t SIMD_BYTES = 16;
|
||||
const __m128i zero16 = _mm_setzero_si128();
|
||||
const UInt8 * filt_end_sse = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
|
||||
|
||||
while (filt_pos < filt_end_sse)
|
||||
while (filt_pos < filt_end_sse)
|
||||
{
|
||||
int mask = _mm_movemask_epi8(_mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(filt_pos)), zero16));
|
||||
|
||||
if (0 == mask)
|
||||
{
|
||||
int mask = _mm_movemask_epi8(_mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(filt_pos)), zero16));
|
||||
|
||||
if (0 == mask)
|
||||
{
|
||||
/// Nothing is inserted.
|
||||
}
|
||||
else if (0xFFFF == mask)
|
||||
{
|
||||
res_data.insert(data_pos, data_pos + SIMD_BYTES);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < SIMD_BYTES; ++i)
|
||||
if (filt_pos[i])
|
||||
res_data.push_back(data_pos[i]);
|
||||
}
|
||||
|
||||
filt_pos += SIMD_BYTES;
|
||||
data_pos += SIMD_BYTES;
|
||||
/// Nothing is inserted.
|
||||
}
|
||||
else if (0xFFFF == mask)
|
||||
{
|
||||
res_data.insert(data_pos, data_pos + SIMD_BYTES);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < SIMD_BYTES; ++i)
|
||||
if (filt_pos[i])
|
||||
res_data.push_back(data_pos[i]);
|
||||
}
|
||||
|
||||
filt_pos += SIMD_BYTES;
|
||||
data_pos += SIMD_BYTES;
|
||||
}
|
||||
#endif
|
||||
|
||||
while (filt_pos < filt_end)
|
||||
{
|
||||
if (*filt_pos)
|
||||
res_data.push_back(*data_pos);
|
||||
|
||||
++filt_pos;
|
||||
++data_pos;
|
||||
}
|
||||
}
|
||||
else
|
||||
while (filt_pos < filt_end)
|
||||
{
|
||||
const auto * filt_pos = filt.begin();
|
||||
const auto * filt_end = filt.end();
|
||||
auto data_pos = data.begin();
|
||||
if (*filt_pos)
|
||||
res_data.push_back(*data_pos);
|
||||
|
||||
while (filt_pos < filt_end)
|
||||
{
|
||||
if (*filt_pos)
|
||||
res_data.push_back(*data_pos);
|
||||
|
||||
++filt_pos;
|
||||
++data_pos;
|
||||
}
|
||||
++filt_pos;
|
||||
++data_pos;
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
#include <Columns/ColumnVectorHelper.h>
|
||||
#include <common/unaligned.h>
|
||||
#include <Core/Field.h>
|
||||
#include <Core/BigInt.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
|
||||
@ -107,10 +106,7 @@ private:
|
||||
|
||||
public:
|
||||
using ValueType = T;
|
||||
static constexpr bool is_POD = !is_big_int_v<T>;
|
||||
using Container = std::conditional_t<is_POD,
|
||||
PaddedPODArray<ValueType>,
|
||||
std::vector<ValueType>>;
|
||||
using Container = PaddedPODArray<ValueType>;
|
||||
|
||||
private:
|
||||
ColumnVector() {}
|
||||
@ -136,10 +132,7 @@ public:
|
||||
|
||||
void insertData(const char * pos, size_t) override
|
||||
{
|
||||
if constexpr (is_POD)
|
||||
data.emplace_back(unalignedLoad<T>(pos));
|
||||
else
|
||||
data.emplace_back(BigInt<T>::deserialize(pos));
|
||||
data.emplace_back(unalignedLoad<T>(pos));
|
||||
}
|
||||
|
||||
void insertDefault() override
|
||||
@ -149,18 +142,12 @@ public:
|
||||
|
||||
void insertManyDefaults(size_t length) override
|
||||
{
|
||||
if constexpr (is_POD)
|
||||
data.resize_fill(data.size() + length, T());
|
||||
else
|
||||
data.resize(data.size() + length, T());
|
||||
data.resize_fill(data.size() + length, T());
|
||||
}
|
||||
|
||||
void popBack(size_t n) override
|
||||
{
|
||||
if constexpr (is_POD)
|
||||
data.resize_assume_reserved(data.size() - n);
|
||||
else
|
||||
data.resize(data.size() - n);
|
||||
data.resize_assume_reserved(data.size() - n);
|
||||
}
|
||||
|
||||
StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
|
||||
@ -185,16 +172,12 @@ public:
|
||||
|
||||
size_t allocatedBytes() const override
|
||||
{
|
||||
if constexpr (is_POD)
|
||||
return data.allocated_bytes();
|
||||
else
|
||||
return data.capacity() * sizeof(data[0]);
|
||||
return data.allocated_bytes();
|
||||
}
|
||||
|
||||
void protect() override
|
||||
{
|
||||
if constexpr (is_POD)
|
||||
data.protect();
|
||||
data.protect();
|
||||
}
|
||||
|
||||
void insertValue(const T value)
|
||||
|
||||
@ -1,41 +0,0 @@
|
||||
#include <Common/DirectorySyncGuard.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Disks/IDisk.h>
|
||||
#include <fcntl.h> // O_RDWR
|
||||
|
||||
/// OSX does not have O_DIRECTORY
|
||||
#ifndef O_DIRECTORY
|
||||
#define O_DIRECTORY O_RDWR
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_FSYNC;
|
||||
}
|
||||
|
||||
DirectorySyncGuard::DirectorySyncGuard(const DiskPtr & disk_, const String & path)
|
||||
: disk(disk_)
|
||||
, fd(disk_->open(path, O_DIRECTORY))
|
||||
{}
|
||||
|
||||
DirectorySyncGuard::~DirectorySyncGuard()
|
||||
{
|
||||
try
|
||||
{
|
||||
#if defined(OS_DARWIN)
|
||||
if (fcntl(fd, F_FULLFSYNC, 0))
|
||||
throwFromErrno("Cannot fcntl(F_FULLFSYNC)", ErrorCodes::CANNOT_FSYNC);
|
||||
#endif
|
||||
disk->sync(fd);
|
||||
disk->close(fd);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -235,7 +235,7 @@ public:
|
||||
else if constexpr (std::is_same_v<T, UInt128>)
|
||||
throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED);
|
||||
else
|
||||
return bigint_cast<T>(x);
|
||||
return static_cast<T>(x);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <common/types.h>
|
||||
#include <Core/BigInt.h>
|
||||
#include <Common/UInt128.h>
|
||||
#include <common/unaligned.h>
|
||||
|
||||
|
||||
@ -18,7 +18,7 @@
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <Core/Defines.h>
|
||||
#include <Core/BigInt.h>
|
||||
|
||||
|
||||
#define ROTL(x, b) static_cast<UInt64>(((x) << (b)) | ((x) >> (64 - (b))))
|
||||
|
||||
@ -136,23 +136,11 @@ public:
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::enable_if_t<std::has_unique_object_representations_v<T>, void> update(const T & x)
|
||||
void update(const T & x)
|
||||
{
|
||||
update(reinterpret_cast<const char *>(&x), sizeof(x));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::enable_if_t<(std::is_floating_point_v<T> || std::is_same_v<T, CityHash_v1_0_2::uint128>), void> update(const T & x)
|
||||
{
|
||||
update(reinterpret_cast<const char *>(&x), sizeof(x));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::enable_if_t<is_big_int_v<T> && !std::has_unique_object_representations_v<T>, void> update(const T & x)
|
||||
{
|
||||
update(DB::BigInt<T>::serialize(x));
|
||||
}
|
||||
|
||||
void update(const std::string & x)
|
||||
{
|
||||
update(x.data(), x.length());
|
||||
@ -205,27 +193,13 @@ inline UInt64 sipHash64(const char * data, const size_t size)
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::enable_if_t<std::has_unique_object_representations_v<T>, UInt64> sipHash64(const T & x)
|
||||
UInt64 sipHash64(const T & x)
|
||||
{
|
||||
SipHash hash;
|
||||
hash.update(x);
|
||||
return hash.get64();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::enable_if_t<(std::is_floating_point_v<T> || (is_big_int_v<T> && !std::has_unique_object_representations_v<T>)), UInt64> sipHash64(const T & x)
|
||||
{
|
||||
SipHash hash;
|
||||
hash.update(x);
|
||||
return hash.get64();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::enable_if_t<DB::IsDecimalNumber<T>, UInt64> sipHash64(const T & x)
|
||||
{
|
||||
return sipHash64(x.value);
|
||||
}
|
||||
|
||||
inline UInt64 sipHash64(const std::string & s)
|
||||
{
|
||||
return sipHash64(s.data(), s.size());
|
||||
|
||||
@ -18,30 +18,30 @@ namespace zkutil
|
||||
void TestKeeperStorageDispatcher::processingThread()
|
||||
{
|
||||
setThreadName("TestKeeperSProc");
|
||||
try
|
||||
|
||||
while (!shutdown)
|
||||
{
|
||||
while (!shutdown)
|
||||
RequestInfo info;
|
||||
|
||||
UInt64 max_wait = UInt64(operation_timeout.totalMilliseconds());
|
||||
|
||||
if (requests_queue.tryPop(info, max_wait))
|
||||
{
|
||||
RequestInfo info;
|
||||
if (shutdown)
|
||||
break;
|
||||
|
||||
UInt64 max_wait = UInt64(operation_timeout.totalMilliseconds());
|
||||
|
||||
if (requests_queue.tryPop(info, max_wait))
|
||||
try
|
||||
{
|
||||
if (shutdown)
|
||||
break;
|
||||
|
||||
auto responses = storage.processRequest(info.request, info.session_id);
|
||||
for (const auto & response_for_session : responses)
|
||||
setResponse(response_for_session.session_id, response_for_session.response);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
finalize();
|
||||
}
|
||||
}
|
||||
|
||||
void TestKeeperStorageDispatcher::setResponse(int64_t session_id, const Coordination::ZooKeeperResponsePtr & response)
|
||||
|
||||
@ -37,7 +37,6 @@ SRCS(
|
||||
CurrentMetrics.cpp
|
||||
CurrentThread.cpp
|
||||
DNSResolver.cpp
|
||||
DirectorySyncGuard.cpp
|
||||
Dwarf.cpp
|
||||
Elf.cpp
|
||||
ErrorCodes.cpp
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <common/types.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Compression/ICompressionCodec.h>
|
||||
|
||||
|
||||
|
||||
@ -93,7 +93,7 @@ using bool_if_gt_int_vs_uint = std::enable_if_t<is_gt_int_vs_uint<TInt, TUInt>,
|
||||
template <typename TInt, typename TUInt>
|
||||
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
|
||||
{
|
||||
return bigint_cast<TInt>(a) > bigint_cast<TInt>(b);
|
||||
return static_cast<TInt>(a) > static_cast<TInt>(b);
|
||||
}
|
||||
|
||||
template <typename TInt, typename TUInt>
|
||||
@ -101,19 +101,19 @@ inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
|
||||
{
|
||||
using CastA = std::conditional_t<is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>, DB::UInt256, TInt>;
|
||||
|
||||
return bigint_cast<CastA>(a) > b;
|
||||
return static_cast<CastA>(a) > b;
|
||||
}
|
||||
|
||||
template <typename TInt, typename TUInt>
|
||||
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
|
||||
{
|
||||
return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
|
||||
return static_cast<TInt>(a) == static_cast<TInt>(b);
|
||||
}
|
||||
|
||||
template <typename TInt, typename TUInt>
|
||||
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
|
||||
{
|
||||
return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
|
||||
return static_cast<TInt>(a) == static_cast<TInt>(b);
|
||||
}
|
||||
|
||||
|
||||
@ -196,7 +196,7 @@ inline bool_if_safe_conversion<A, B> greaterOp(A a, B b)
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) > bigint_cast<CastB>(b);
|
||||
return static_cast<CastA>(a) > static_cast<CastB>(b);
|
||||
else
|
||||
return a > b;
|
||||
}
|
||||
@ -306,7 +306,7 @@ inline bool_if_safe_conversion<A, B> equalsOp(A a, B b)
|
||||
{
|
||||
using LargestType = std::conditional_t<(sizeof(A) > sizeof(B)) || ((sizeof(A) == sizeof(B)) && !std::is_same_v<A, DB::UInt128>), A, B>;
|
||||
|
||||
return bigint_cast<LargestType>(a) == bigint_cast<LargestType>(b);
|
||||
return static_cast<LargestType>(a) == static_cast<LargestType>(b);
|
||||
}
|
||||
|
||||
template <>
|
||||
@ -429,7 +429,7 @@ inline bool_if_safe_conversion<A, B> notEqualsOp(A a, B b)
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) != bigint_cast<CastB>(b);
|
||||
return static_cast<CastA>(a) != static_cast<CastB>(b);
|
||||
else
|
||||
return a != b;
|
||||
}
|
||||
@ -451,7 +451,7 @@ inline bool_if_safe_conversion<A, B> lessOp(A a, B b)
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) < bigint_cast<CastB>(b);
|
||||
return static_cast<CastA>(a) < static_cast<CastB>(b);
|
||||
else
|
||||
return a < b;
|
||||
}
|
||||
@ -475,7 +475,7 @@ inline bool_if_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) <= bigint_cast<CastB>(b);
|
||||
return static_cast<CastA>(a) <= static_cast<CastB>(b);
|
||||
else
|
||||
return a <= b;
|
||||
}
|
||||
@ -499,7 +499,7 @@ inline bool_if_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) >= bigint_cast<CastB>(b);
|
||||
return static_cast<CastA>(a) >= static_cast<CastB>(b);
|
||||
else
|
||||
return a >= b;
|
||||
}
|
||||
|
||||
@ -1,36 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <common/StringRef.h>
|
||||
#include <common/unaligned.h>
|
||||
#include <Core/Types.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
struct BigInt
|
||||
{
|
||||
static_assert(sizeof(T) == 32);
|
||||
static constexpr size_t size = 32;
|
||||
|
||||
static StringRef serialize(const T & x, char * pos)
|
||||
{
|
||||
unalignedStore<T>(pos, x);
|
||||
return StringRef(pos, size);
|
||||
}
|
||||
|
||||
static String serialize(const T & x)
|
||||
{
|
||||
String str(size, '\0');
|
||||
serialize(x, str.data());
|
||||
return str;
|
||||
}
|
||||
|
||||
static T deserialize(const char * pos)
|
||||
{
|
||||
return unalignedLoad<T>(pos);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@ -233,9 +233,9 @@ private:
|
||||
bool overflow = false;
|
||||
|
||||
if constexpr (sizeof(A) > sizeof(CompareInt))
|
||||
overflow |= (bigint_cast<A>(x) != a);
|
||||
overflow |= (static_cast<A>(x) != a);
|
||||
if constexpr (sizeof(B) > sizeof(CompareInt))
|
||||
overflow |= (bigint_cast<B>(y) != b);
|
||||
overflow |= (static_cast<B>(y) != b);
|
||||
if constexpr (is_unsigned_v<A>)
|
||||
overflow |= (x < 0);
|
||||
if constexpr (is_unsigned_v<B>)
|
||||
|
||||
@ -326,6 +326,7 @@ class IColumn;
|
||||
M(Bool, log_profile_events, true, "Log query performance statistics into the query_log and query_thread_log.", 0) \
|
||||
M(Bool, log_query_settings, true, "Log query settings into the query_log.", 0) \
|
||||
M(Bool, log_query_threads, true, "Log query threads into system.query_thread_log table. This setting have effect only when 'log_queries' is true.", 0) \
|
||||
M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \
|
||||
M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \
|
||||
M(Bool, enable_optimize_predicate_expression, 1, "If it is set to true, optimize predicates to subqueries.", 0) \
|
||||
M(Bool, enable_optimize_predicate_expression_to_final_subquery, 1, "Allow push predicate to final subquery.", 0) \
|
||||
@ -404,7 +405,7 @@ class IColumn;
|
||||
M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \
|
||||
M(Bool, optimize_trivial_insert_select, true, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \
|
||||
M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \
|
||||
M(Bool, enable_global_with_statement, false, "Propagate WITH statements to UNION queries and all subqueries", 0) \
|
||||
M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \
|
||||
M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \
|
||||
M(Bool, flatten_nested, true, "If true, columns of type Nested will be flatten to separate array columns instead of one array of tuples", 0) \
|
||||
M(Bool, asterisk_include_materialized_columns, false, "Include MATERIALIZED columns for wildcard query", 0) \
|
||||
|
||||
@ -158,7 +158,7 @@ struct Decimal
|
||||
return convertTo<typename U::NativeType>();
|
||||
}
|
||||
else
|
||||
return bigint_cast<U>(value);
|
||||
return static_cast<U>(value);
|
||||
}
|
||||
|
||||
const Decimal<T> & operator += (const T & x) { value += x; return *this; }
|
||||
|
||||
@ -546,7 +546,7 @@ ColumnPtr DataTypeNullable::getSubcolumn(const String & subcolumn_name, const IC
|
||||
{
|
||||
const auto & column_nullable = assert_cast<const ColumnNullable &>(column);
|
||||
if (subcolumn_name == "null")
|
||||
return column_nullable.getNullMapColumnPtr()->assumeMutable();
|
||||
return column_nullable.getNullMapColumnPtr();
|
||||
|
||||
return nested_data_type->getSubcolumn(subcolumn_name, column_nullable.getNestedColumn());
|
||||
}
|
||||
|
||||
@ -597,6 +597,7 @@ inline bool isEnum(const DataTypePtr & data_type) { return WhichDataType(data_ty
|
||||
inline bool isDecimal(const DataTypePtr & data_type) { return WhichDataType(data_type).isDecimal(); }
|
||||
inline bool isTuple(const DataTypePtr & data_type) { return WhichDataType(data_type).isTuple(); }
|
||||
inline bool isArray(const DataTypePtr & data_type) { return WhichDataType(data_type).isArray(); }
|
||||
inline bool isMap(const DataTypePtr & data_type) {return WhichDataType(data_type).isMap(); }
|
||||
|
||||
template <typename T>
|
||||
inline bool isUInt8(const T & data_type)
|
||||
|
||||
@ -218,7 +218,7 @@ using ResultOfGreatest = std::conditional_t<LeastGreatestSpecialCase<A, B>,
|
||||
template <typename T>
|
||||
static inline auto littleBits(const T & x)
|
||||
{
|
||||
return bigint_cast<UInt8>(x);
|
||||
return static_cast<UInt8>(x);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -38,10 +38,15 @@ DataTypePtr convertMySQLDataType(MultiEnum<MySQLDataTypesSupport> type_support,
|
||||
size_t precision,
|
||||
size_t scale)
|
||||
{
|
||||
// we expect mysql_data_type to be either "basic_type" or "type_with_params(param1, param2, ...)"
|
||||
// Mysql returns mysql_data_type as below:
|
||||
// 1. basic_type
|
||||
// 2. basic_type options
|
||||
// 3. type_with_params(param1, param2, ...)
|
||||
// 4. type_with_params(param1, param2, ...) options
|
||||
// The options can be unsigned, zerofill, or some other strings.
|
||||
auto data_type = std::string_view(mysql_data_type);
|
||||
const auto param_start_pos = data_type.find('(');
|
||||
const auto type_name = data_type.substr(0, param_start_pos);
|
||||
const auto type_end_pos = data_type.find_first_of(R"(( )"); // FIXME: fix style-check script instead
|
||||
const auto type_name = data_type.substr(0, type_end_pos);
|
||||
|
||||
DataTypePtr res;
|
||||
|
||||
|
||||
@ -44,6 +44,10 @@ static Context createQueryContext(const Context & global_context)
|
||||
Settings new_query_settings = global_context.getSettings();
|
||||
new_query_settings.insert_allow_materialized_columns = true;
|
||||
|
||||
/// To avoid call AST::format
|
||||
/// TODO: We need to implement the format function for MySQLAST
|
||||
new_query_settings.enable_global_with_statement = false;
|
||||
|
||||
Context query_context(global_context);
|
||||
query_context.setSettings(new_query_settings);
|
||||
CurrentThread::QueryScope query_scope(query_context);
|
||||
|
||||
@ -344,7 +344,9 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
e.addMessage("error parsing null_value");
|
||||
String dictionary_name = config.getString(".dictionary.name", "");
|
||||
e.addMessage("While parsing null_value for attribute with name " + name
|
||||
+ " in dictionary " + dictionary_name);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
@ -180,6 +180,32 @@ Names getPrimaryKeyColumns(const ASTExpressionList * primary_key)
|
||||
return result;
|
||||
}
|
||||
|
||||
void buildAttributeExpressionIfNeeded(
|
||||
AutoPtr<Document> doc,
|
||||
AutoPtr<Element> root,
|
||||
const ASTDictionaryAttributeDeclaration * dict_attr)
|
||||
{
|
||||
if (dict_attr->expression != nullptr)
|
||||
{
|
||||
AutoPtr<Element> expression_element(doc->createElement("expression"));
|
||||
|
||||
/// EXPRESSION PROPERTY should be expression or string
|
||||
String expression_str;
|
||||
if (const auto * literal = dict_attr->expression->as<ASTLiteral>();
|
||||
literal && literal->value.getType() == Field::Types::String)
|
||||
{
|
||||
expression_str = getFieldAsString(literal->value);
|
||||
}
|
||||
else
|
||||
expression_str = queryToString(dict_attr->expression);
|
||||
|
||||
|
||||
AutoPtr<Text> expression(doc->createTextNode(expression_str));
|
||||
expression_element->appendChild(expression);
|
||||
root->appendChild(expression_element);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transofrms single dictionary attribute to configuration
|
||||
* third_column UInt8 DEFAULT 2 EXPRESSION rand() % 100 * 77
|
||||
@ -217,25 +243,7 @@ void buildSingleAttribute(
|
||||
null_value_element->appendChild(null_value);
|
||||
attribute_element->appendChild(null_value_element);
|
||||
|
||||
if (dict_attr->expression != nullptr)
|
||||
{
|
||||
AutoPtr<Element> expression_element(doc->createElement("expression"));
|
||||
|
||||
/// EXPRESSION PROPERTY should be expression or string
|
||||
String expression_str;
|
||||
if (const auto * literal = dict_attr->expression->as<ASTLiteral>();
|
||||
literal && literal->value.getType() == Field::Types::String)
|
||||
{
|
||||
expression_str = getFieldAsString(literal->value);
|
||||
}
|
||||
else
|
||||
expression_str = queryToString(dict_attr->expression);
|
||||
|
||||
|
||||
AutoPtr<Text> expression(doc->createTextNode(expression_str));
|
||||
expression_element->appendChild(expression);
|
||||
attribute_element->appendChild(expression_element);
|
||||
}
|
||||
buildAttributeExpressionIfNeeded(doc, attribute_element, dict_attr);
|
||||
|
||||
if (dict_attr->hierarchical)
|
||||
{
|
||||
@ -286,6 +294,8 @@ void buildPrimaryKeyConfiguration(
|
||||
const Names & key_names,
|
||||
const ASTExpressionList * dictionary_attributes)
|
||||
{
|
||||
const auto & children = dictionary_attributes->children;
|
||||
|
||||
if (!complex)
|
||||
{
|
||||
if (key_names.size() != 1)
|
||||
@ -296,12 +306,16 @@ void buildPrimaryKeyConfiguration(
|
||||
root->appendChild(id_element);
|
||||
AutoPtr<Element> name_element(doc->createElement("name"));
|
||||
id_element->appendChild(name_element);
|
||||
AutoPtr<Text> name(doc->createTextNode(*key_names.begin()));
|
||||
|
||||
const ASTDictionaryAttributeDeclaration * dict_attr = children.front()->as<const ASTDictionaryAttributeDeclaration>();
|
||||
|
||||
AutoPtr<Text> name(doc->createTextNode(dict_attr->name));
|
||||
name_element->appendChild(name);
|
||||
|
||||
buildAttributeExpressionIfNeeded(doc, id_element, dict_attr);
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto & children = dictionary_attributes->children;
|
||||
if (children.size() < key_names.size())
|
||||
throw Exception(
|
||||
"Primary key fields count is more, than dictionary attributes count.", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
|
||||
|
||||
@ -175,24 +175,14 @@ void DiskDecorator::truncateFile(const String & path, size_t size)
|
||||
delegate->truncateFile(path, size);
|
||||
}
|
||||
|
||||
int DiskDecorator::open(const String & path, int flags) const
|
||||
{
|
||||
return delegate->open(path, flags);
|
||||
}
|
||||
|
||||
void DiskDecorator::close(int fd) const
|
||||
{
|
||||
delegate->close(fd);
|
||||
}
|
||||
|
||||
void DiskDecorator::sync(int fd) const
|
||||
{
|
||||
delegate->sync(fd);
|
||||
}
|
||||
|
||||
Executor & DiskDecorator::getExecutor()
|
||||
{
|
||||
return delegate->getExecutor();
|
||||
}
|
||||
|
||||
SyncGuardPtr DiskDecorator::getDirectorySyncGuard(const String & path) const
|
||||
{
|
||||
return delegate->getDirectorySyncGuard(path);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -48,11 +48,9 @@ public:
|
||||
void setReadOnly(const String & path) override;
|
||||
void createHardLink(const String & src_path, const String & dst_path) override;
|
||||
void truncateFile(const String & path, size_t size) override;
|
||||
int open(const String & path, int flags) const override;
|
||||
void close(int fd) const override;
|
||||
void sync(int fd) const override;
|
||||
const String getType() const override { return delegate->getType(); }
|
||||
Executor & getExecutor() override;
|
||||
SyncGuardPtr getDirectorySyncGuard(const String & path) const override;
|
||||
|
||||
protected:
|
||||
DiskPtr delegate;
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <Disks/LocalDirectorySyncGuard.h>
|
||||
|
||||
#include <IO/createReadBufferFromFileBase.h>
|
||||
#include <common/logger_useful.h>
|
||||
@ -20,10 +21,6 @@ namespace ErrorCodes
|
||||
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
|
||||
extern const int PATH_ACCESS_DENIED;
|
||||
extern const int INCORRECT_DISK_INDEX;
|
||||
extern const int FILE_DOESNT_EXIST;
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int CANNOT_FSYNC;
|
||||
extern const int CANNOT_CLOSE_FILE;
|
||||
extern const int CANNOT_TRUNCATE_FILE;
|
||||
extern const int CANNOT_UNLINK;
|
||||
extern const int CANNOT_RMDIR;
|
||||
@ -315,26 +312,9 @@ void DiskLocal::copy(const String & from_path, const std::shared_ptr<IDisk> & to
|
||||
IDisk::copy(from_path, to_disk, to_path); /// Copy files through buffers.
|
||||
}
|
||||
|
||||
int DiskLocal::open(const String & path, int flags) const
|
||||
SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const
|
||||
{
|
||||
String full_path = disk_path + path;
|
||||
int fd = ::open(full_path.c_str(), flags);
|
||||
if (-1 == fd)
|
||||
throwFromErrnoWithPath("Cannot open file " + full_path, full_path,
|
||||
errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
|
||||
return fd;
|
||||
}
|
||||
|
||||
void DiskLocal::close(int fd) const
|
||||
{
|
||||
if (-1 == ::close(fd))
|
||||
throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
|
||||
}
|
||||
|
||||
void DiskLocal::sync(int fd) const
|
||||
{
|
||||
if (-1 == ::fsync(fd))
|
||||
throw Exception("Cannot fsync", ErrorCodes::CANNOT_FSYNC);
|
||||
return std::make_unique<LocalDirectorySyncGuard>(disk_path + path);
|
||||
}
|
||||
|
||||
DiskPtr DiskLocalReservation::getDisk(size_t i) const
|
||||
|
||||
@ -98,14 +98,12 @@ public:
|
||||
|
||||
void createHardLink(const String & src_path, const String & dst_path) override;
|
||||
|
||||
int open(const String & path, int flags) const override;
|
||||
void close(int fd) const override;
|
||||
void sync(int fd) const override;
|
||||
|
||||
void truncateFile(const String & path, size_t size) override;
|
||||
|
||||
const String getType() const override { return "local"; }
|
||||
|
||||
SyncGuardPtr getDirectorySyncGuard(const String & path) const override;
|
||||
|
||||
private:
|
||||
bool tryReserve(UInt64 bytes);
|
||||
|
||||
|
||||
@ -436,21 +436,6 @@ void DiskMemory::setReadOnly(const String &)
|
||||
throw Exception("Method setReadOnly is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
int DiskMemory::open(const String & /*path*/, int /*flags*/) const
|
||||
{
|
||||
throw Exception("Method open is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
void DiskMemory::close(int /*fd*/) const
|
||||
{
|
||||
throw Exception("Method close is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
void DiskMemory::sync(int /*fd*/) const
|
||||
{
|
||||
throw Exception("Method sync is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
void DiskMemory::truncateFile(const String & path, size_t size)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
@ -89,10 +89,6 @@ public:
|
||||
|
||||
void createHardLink(const String & src_path, const String & dst_path) override;
|
||||
|
||||
int open(const String & path, int flags) const override;
|
||||
void close(int fd) const override;
|
||||
void sync(int fd) const override;
|
||||
|
||||
void truncateFile(const String & path, size_t size) override;
|
||||
|
||||
const String getType() const override { return "memory"; }
|
||||
|
||||
@ -76,4 +76,9 @@ void IDisk::truncateFile(const String &, size_t)
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Truncate operation is not implemented for disk of type {}", getType());
|
||||
}
|
||||
|
||||
SyncGuardPtr IDisk::getDirectorySyncGuard(const String & /* path */) const
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -57,6 +57,19 @@ public:
|
||||
|
||||
using SpacePtr = std::shared_ptr<Space>;
|
||||
|
||||
/**
|
||||
* A guard, that should synchronize file's or directory's state
|
||||
* with storage device (e.g. fsync in POSIX) in its destructor.
|
||||
*/
|
||||
class ISyncGuard
|
||||
{
|
||||
public:
|
||||
ISyncGuard() = default;
|
||||
virtual ~ISyncGuard() = default;
|
||||
};
|
||||
|
||||
using SyncGuardPtr = std::unique_ptr<ISyncGuard>;
|
||||
|
||||
/**
|
||||
* A unit of storage persisting data and metadata.
|
||||
* Abstract underlying storage technology.
|
||||
@ -174,15 +187,6 @@ public:
|
||||
/// Create hardlink from `src_path` to `dst_path`.
|
||||
virtual void createHardLink(const String & src_path, const String & dst_path) = 0;
|
||||
|
||||
/// Wrapper for POSIX open
|
||||
virtual int open(const String & path, int flags) const = 0;
|
||||
|
||||
/// Wrapper for POSIX close
|
||||
virtual void close(int fd) const = 0;
|
||||
|
||||
/// Wrapper for POSIX fsync
|
||||
virtual void sync(int fd) const = 0;
|
||||
|
||||
/// Truncate file to specified size.
|
||||
virtual void truncateFile(const String & path, size_t size);
|
||||
|
||||
@ -195,6 +199,9 @@ public:
|
||||
/// Returns executor to perform asynchronous operations.
|
||||
virtual Executor & getExecutor() { return *executor; }
|
||||
|
||||
/// Returns guard, that insures synchronization of directory metadata with storage device.
|
||||
virtual SyncGuardPtr getDirectorySyncGuard(const String & path) const;
|
||||
|
||||
private:
|
||||
std::unique_ptr<Executor> executor;
|
||||
};
|
||||
|
||||
50
src/Disks/LocalDirectorySyncGuard.cpp
Normal file
50
src/Disks/LocalDirectorySyncGuard.cpp
Normal file
@ -0,0 +1,50 @@
|
||||
#include <Disks/LocalDirectorySyncGuard.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Disks/IDisk.h>
|
||||
#include <fcntl.h> // O_RDWR
|
||||
|
||||
/// OSX does not have O_DIRECTORY
|
||||
#ifndef O_DIRECTORY
|
||||
#define O_DIRECTORY O_RDWR
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_FSYNC;
|
||||
extern const int FILE_DOESNT_EXIST;
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int CANNOT_CLOSE_FILE;
|
||||
}
|
||||
|
||||
LocalDirectorySyncGuard::LocalDirectorySyncGuard(const String & full_path)
|
||||
: fd(::open(full_path.c_str(), O_DIRECTORY))
|
||||
{
|
||||
if (-1 == fd)
|
||||
throwFromErrnoWithPath("Cannot open file " + full_path, full_path,
|
||||
errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
|
||||
}
|
||||
|
||||
LocalDirectorySyncGuard::~LocalDirectorySyncGuard()
|
||||
{
|
||||
try
|
||||
{
|
||||
#if defined(OS_DARWIN)
|
||||
if (fcntl(fd, F_FULLFSYNC, 0))
|
||||
throwFromErrno("Cannot fcntl(F_FULLFSYNC)", ErrorCodes::CANNOT_FSYNC);
|
||||
#endif
|
||||
if (-1 == ::fsync(fd))
|
||||
throw Exception("Cannot fsync", ErrorCodes::CANNOT_FSYNC);
|
||||
|
||||
if (-1 == ::close(fd))
|
||||
throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,7 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <Disks/IDisk.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -13,17 +12,16 @@ using DiskPtr = std::shared_ptr<IDisk>;
|
||||
/// It's used to keep descriptor open, while doing some operations with it, and do fsync at the end.
|
||||
/// Guaranties of sequence 'close-reopen-fsync' may depend on kernel version.
|
||||
/// Source: linux-fsdevel mailing-list https://marc.info/?l=linux-fsdevel&m=152535409207496
|
||||
class DirectorySyncGuard
|
||||
class LocalDirectorySyncGuard final : public ISyncGuard
|
||||
{
|
||||
public:
|
||||
/// NOTE: If you have already opened descriptor, it's preferred to use
|
||||
/// this constructor instead of constructor with path.
|
||||
DirectorySyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {}
|
||||
DirectorySyncGuard(const DiskPtr & disk_, const std::string & path);
|
||||
~DirectorySyncGuard();
|
||||
LocalDirectorySyncGuard(int fd_) : fd(fd_) {}
|
||||
LocalDirectorySyncGuard(const String & full_path);
|
||||
~LocalDirectorySyncGuard() override;
|
||||
|
||||
private:
|
||||
DiskPtr disk;
|
||||
int fd = -1;
|
||||
};
|
||||
|
||||
@ -36,7 +36,6 @@ namespace ErrorCodes
|
||||
extern const int CANNOT_SEEK_THROUGH_FILE;
|
||||
extern const int UNKNOWN_FORMAT;
|
||||
extern const int INCORRECT_DISK_INDEX;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int PATH_ACCESS_DENIED;
|
||||
extern const int CANNOT_DELETE_DIRECTORY;
|
||||
}
|
||||
@ -878,21 +877,6 @@ void DiskS3::setReadOnly(const String & path)
|
||||
metadata.save();
|
||||
}
|
||||
|
||||
int DiskS3::open(const String & /*path*/, int /*flags*/) const
|
||||
{
|
||||
throw Exception("Method open is not implemented for S3 disks", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
void DiskS3::close(int /*fd*/) const
|
||||
{
|
||||
throw Exception("Method close is not implemented for S3 disks", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
void DiskS3::sync(int /*fd*/) const
|
||||
{
|
||||
throw Exception("Method sync is not implemented for S3 disks", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
void DiskS3::shutdown()
|
||||
{
|
||||
/// This call stops any next retry attempts for ongoing S3 requests.
|
||||
|
||||
@ -105,10 +105,6 @@ public:
|
||||
|
||||
void setReadOnly(const String & path) override;
|
||||
|
||||
int open(const String & path, int flags) const override;
|
||||
void close(int fd) const override;
|
||||
void sync(int fd) const override;
|
||||
|
||||
const String getType() const override { return "s3"; }
|
||||
|
||||
void shutdown() override;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user