thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 07:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into zvonand-globs-small-fix

Browse Source
This commit is contained in:
Andrey Zvonov 2023-08-11 15:56:44 +02:00 committed by GitHub
commit 02c511a008
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
289 changed files with 2798 additions and 1657 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -208,9 +208,6 @@ option(OMIT_HEAVY_DEBUG_SYMBOLS
"Do not generate debugger info for heavy modules (ClickHouse functions and dictionaries, some contrib)"
${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT})
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
set(USE_DEBUG_HELPERS ON)
endif()
option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS})
option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF)

2
base/base/JSON.cpp
View File

@ -7,8 +7,6 @@
#include <base/find_symbols.h>
#include <base/preciseExp10.h>
#include <iostream>
#define JSON_MAX_DEPTH 100

2
base/base/wide_integer_impl.h
View File

@ -12,7 +12,6 @@
#include <tuple>
#include <limits>
#include <boost/multiprecision/cpp_bin_float.hpp>
#include <boost/math/special_functions/fpclassify.hpp>
// NOLINTBEGIN(*)
@ -22,6 +21,7 @@
#define CONSTEXPR_FROM_DOUBLE constexpr
using FromDoubleIntermediateType = long double;
#else
#include <boost/multiprecision/cpp_bin_float.hpp>
/// `wide_integer_from_builtin` can't be constexpr with non-literal `cpp_bin_float_double_extended`
#define CONSTEXPR_FROM_DOUBLE
using FromDoubleIntermediateType = boost::multiprecision::cpp_bin_float_double_extended;

1
base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp
View File

@ -19,7 +19,6 @@
#include "Poco/UTF16Encoding.h"
#include "Poco/Buffer.h"
#include "Poco/Exception.h"
#include <iostream>
using Poco::Buffer;

1
base/poco/Foundation/src/Task.cpp
View File

@ -16,7 +16,6 @@
#include "Poco/TaskManager.h"
#include "Poco/Exception.h"
#include <iostream>
#include <array>

1
base/poco/JSON/src/Object.cpp
View File

@ -14,7 +14,6 @@
#include "Poco/JSON/Object.h"
#include <iostream>
#include <sstream>
using Poco::Dynamic::Var;

1
base/poco/Net/src/HTTPClientSession.cpp
View File

@ -26,7 +26,6 @@
#include "Poco/CountingStream.h"
#include "Poco/RegularExpression.h"
#include <sstream>
#include <iostream>
using Poco::NumberFormatter;

2
docker/README.md
View File

@ -1,5 +1,5 @@
## ClickHouse Dockerfiles
This directory contain Dockerfiles for `clickhouse-client` and `clickhouse-server`. They are updated in each release.
This directory contain Dockerfiles for `clickhouse-server`. They are updated in each release.
Also there is bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`.

34
docker/client/Dockerfile
View File

@ -1,34 +0,0 @@
FROM ubuntu:18.04
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/"
ARG version=22.1.1.*
RUN apt-get update \
&& apt-get install --yes --no-install-recommends \
apt-transport-https \
ca-certificates \
dirmngr \
gnupg \
&& mkdir -p /etc/apt/sources.list.d \
&& apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 \
&& echo $repository > /etc/apt/sources.list.d/clickhouse.list \
&& apt-get update \
&& env DEBIAN_FRONTEND=noninteractive \
apt-get install --allow-unauthenticated --yes --no-install-recommends \
clickhouse-client=$version \
clickhouse-common-static=$version \
locales \
tzdata \
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf \
&& apt-get clean
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
ENTRYPOINT ["/usr/bin/clickhouse-client"]

7
docker/client/README.md
View File

@ -1,7 +0,0 @@
# ClickHouse Client Docker Image
For more information see [ClickHouse Server Docker Image](https://hub.docker.com/r/clickhouse/clickhouse-server/).
## License
View [license information](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) for the software contained in this image.

27
docker/packager/binary/Dockerfile
View File

@ -58,33 +58,6 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
rustup target add aarch64-apple-darwin && \
rustup target add powerpc64le-unknown-linux-gnu
# Create vendor cache for cargo.
#
# Note, that the config.toml for the root is used, you will not be able to
# install any other crates, except those which had been vendored (since if
# there is "replace-with" for some source, then cargo will not look to other
# remotes except this).
#
# Notes for the command itself:
# - --chown is required to preserve the rights
# - unstable-options for -C
# - chmod is required to fix the permissions, since builds are running from a different user
# - copy of the Cargo.lock is required for proper dependencies versions
# - cargo vendor --sync is requried to overcome [1] bug.
#
# [1]: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
COPY --chown=root:root /rust /rust/packages
RUN cargo -Z unstable-options -C /rust/packages vendor > $CARGO_HOME/config.toml && \
cp "$(rustc --print=sysroot)"/lib/rustlib/src/rust/Cargo.lock "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/ && \
cargo -Z unstable-options -C /rust/packages vendor --sync "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.toml && \
rm "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.lock && \
sed -i "s#\"vendor\"#\"/rust/vendor\"#" $CARGO_HOME/config.toml && \
cat $CARGO_HOME/config.toml && \
mv /rust/packages/vendor /rust/vendor && \
chmod -R o=r+X /rust/vendor && \
ls -R -l /rust/packages && \
rm -r /rust/packages
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
# A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \

1
docker/packager/binary/rust
View File

@ -1 +0,0 @@
../../../rust

12
docker/packager/packager
View File

@ -80,9 +80,11 @@ def run_docker_image_with_env(
output_dir: Path,
env_variables: List[str],
ch_root: Path,
cargo_cache_dir: Path,
ccache_dir: Optional[Path],
) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
cargo_cache_dir.mkdir(parents=True, exist_ok=True)
env_part = " -e ".join(env_variables)
if env_part:
@ -105,7 +107,7 @@ def run_docker_image_with_env(
cmd = (
f"docker run --network=host --user={user} --rm {ccache_mount}"
f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
f"{interactive} {image_name}"
f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}"
)
logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd)
@ -417,6 +419,13 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="if set, the build fails on errors writing cache to S3",
)
parser.add_argument(
"--cargo-cache-dir",
default=Path(os.getenv("CARGO_HOME", "") or Path.home() / ".cargo")
/ "registry",
type=dir_name,
help="a directory to preserve the rust cargo crates",
)
parser.add_argument("--force-build-image", action="store_true")
parser.add_argument("--version")
parser.add_argument("--official", action="store_true")
@ -497,6 +506,7 @@ def main() -> None:
args.output_dir,
env_prepared,
ch_root,
args.cargo_cache_dir,
args.ccache_dir,
)
logging.info("Output placed into %s", args.output_dir)

72
docker/test/performance-comparison/compare.sh
View File

@ -665,9 +665,8 @@ create view partial_query_times as select * from
-- Report for backward-incompatible ('partial') queries that we could only run on the new server (e.g.
-- queries with new functions added in the tested PR).
create table partial_queries_report engine File(TSV, 'report/partial-queries-report.tsv')
settings output_format_decimal_trailing_zeros = 1
as select toDecimal64(time_median, 3) time,
toDecimal64(time_stddev / time_median, 3) relative_time_stddev,
as select round(time_median, 3) time,
round(time_stddev / time_median, 3) relative_time_stddev,
test, query_index, query_display_name
from partial_query_times
join query_display_names using (test, query_index)
@ -739,28 +738,26 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv')
;
create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv')
settings output_format_decimal_trailing_zeros = 1
as with
-- server_time is sometimes reported as zero (if it's less than 1 ms),
-- so we have to work around this to not get an error about conversion
-- of NaN to decimal.
(left > right ? left / right : right / left) as times_change_float,
isFinite(times_change_float) as times_change_finite,
toDecimal64(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
round(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
times_change_finite
? (left > right ? '-' : '+') || toString(times_change_decimal) || 'x'
: '--' as times_change_str
select
toDecimal64(left, 3), toDecimal64(right, 3), times_change_str,
toDecimal64(diff, 3), toDecimal64(stat_threshold, 3),
round(left, 3), round(right, 3), times_change_str,
round(diff, 3), round(stat_threshold, 3),
changed_fail, test, query_index, query_display_name
from queries where changed_show order by abs(diff) desc;
create table unstable_queries_report engine File(TSV, 'report/unstable-queries.tsv')
settings output_format_decimal_trailing_zeros = 1
as select
toDecimal64(left, 3), toDecimal64(right, 3), toDecimal64(diff, 3),
toDecimal64(stat_threshold, 3), unstable_fail, test, query_index, query_display_name
round(left, 3), round(right, 3), round(diff, 3),
round(stat_threshold, 3), unstable_fail, test, query_index, query_display_name
from queries where unstable_show order by stat_threshold desc;
@ -789,11 +786,10 @@ create view total_speedup as
;
create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes.tsv')
settings output_format_decimal_trailing_zeros = 1
as with
(times_speedup >= 1
? '-' || toString(toDecimal64(times_speedup, 3)) || 'x'
: '+' || toString(toDecimal64(1 / times_speedup, 3)) || 'x')
? '-' || toString(round(times_speedup, 3)) || 'x'
: '+' || toString(round(1 / times_speedup, 3)) || 'x')
as times_speedup_str
select test, times_speedup_str, queries, bad, changed, unstable
-- Not sure what's the precedence of UNION ALL vs WHERE & ORDER BY, hence all
@ -817,11 +813,10 @@ create view total_client_time_per_query as select *
'test text, query_index int, client float, server float');
create table slow_on_client_report engine File(TSV, 'report/slow-on-client.tsv')
settings output_format_decimal_trailing_zeros = 1
as select client, server, toDecimal64(client/server, 3) p,
as select client, server, round(client/server, 3) p,
test, query_display_name
from total_client_time_per_query left join query_display_names using (test, query_index)
where p > toDecimal64(1.02, 3) order by p desc;
where p > round(1.02, 3) order by p desc;
create table wall_clock_time_per_test engine Memory as select *
from file('wall-clock-times.tsv', TSV, 'test text, real float, user float, system float');
@ -899,15 +894,14 @@ create view test_times_view_total as
;
create table test_times_report engine File(TSV, 'report/test-times.tsv')
settings output_format_decimal_trailing_zeros = 1
as select
test,
toDecimal64(real, 3),
toDecimal64(total_client_time, 3),
round(real, 3),
round(total_client_time, 3),
queries,
toDecimal64(query_max, 3),
toDecimal64(avg_real_per_query, 3),
toDecimal64(query_min, 3),
round(query_max, 3),
round(avg_real_per_query, 3),
round(query_min, 3),
runs
from (
select * from test_times_view
@ -919,21 +913,20 @@ create table test_times_report engine File(TSV, 'report/test-times.tsv')
-- report for all queries page, only main metric
create table all_tests_report engine File(TSV, 'report/all-queries.tsv')
settings output_format_decimal_trailing_zeros = 1
as with
-- server_time is sometimes reported as zero (if it's less than 1 ms),
-- so we have to work around this to not get an error about conversion
-- of NaN to decimal.
(left > right ? left / right : right / left) as times_change_float,
isFinite(times_change_float) as times_change_finite,
toDecimal64(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
round(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
times_change_finite
? (left > right ? '-' : '+') || toString(times_change_decimal) || 'x'
: '--' as times_change_str
select changed_fail, unstable_fail,
toDecimal64(left, 3), toDecimal64(right, 3), times_change_str,
toDecimal64(isFinite(diff) ? diff : 0, 3),
toDecimal64(isFinite(stat_threshold) ? stat_threshold : 0, 3),
round(left, 3), round(right, 3), times_change_str,
round(isFinite(diff) ? diff : 0, 3),
round(isFinite(stat_threshold) ? stat_threshold : 0, 3),
test, query_index, query_display_name
from queries order by test, query_index;
@ -1044,27 +1037,6 @@ create table unstable_run_traces engine File(TSVWithNamesAndTypes,
order by count() desc
;
create table metric_devation engine File(TSVWithNamesAndTypes,
'report/metric-deviation.$version.tsv')
settings output_format_decimal_trailing_zeros = 1
-- first goes the key used to split the file with grep
as select test, query_index, query_display_name,
toDecimal64(d, 3) d, q, metric
from (
select
test, query_index,
(q[3] - q[1])/q[2] d,
quantilesExact(0, 0.5, 1)(value) q, metric
from (select * from unstable_run_metrics
union all select * from unstable_run_traces
union all select * from unstable_run_metrics_2) mm
group by test, query_index, metric
having isFinite(d) and d > 0.5 and q[3] > 5
) metrics
left join query_display_names using (test, query_index)
order by test, query_index, d desc
;
create table stacks engine File(TSV, 'report/stacks.$version.tsv') as
select
-- first goes the key used to split the file with grep
@ -1173,9 +1145,8 @@ create table metrics engine File(TSV, 'metrics/metrics.tsv') as
-- Show metrics that have changed
create table changes engine File(TSV, 'metrics/changes.tsv')
settings output_format_decimal_trailing_zeros = 1
as select metric, left, right,
toDecimal64(diff, 3), toDecimal64(times_diff, 3)
round(diff, 3), round(times_diff, 3)
from (
select metric, median(left) as left, median(right) as right,
(right - left) / left diff,
@ -1226,7 +1197,6 @@ create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv')
'$SHA_TO_TEST' :: LowCardinality(String) AS commit_sha,
'${CLICKHOUSE_PERFORMANCE_COMPARISON_CHECK_NAME:-Performance}' :: LowCardinality(String) AS check_name,
'$(sed -n 's/.*<!--status: \(.*\)-->/\1/p' report.html)' :: LowCardinality(String) AS check_status,
-- TODO toDateTime() can't parse output of 'date', so no time for now.
(($(date +%s) - $CHPC_CHECK_START_TIMESTAMP) * 1000) :: UInt64 AS check_duration_ms,
fromUnixTimestamp($CHPC_CHECK_START_TIMESTAMP) check_start_time,
test_name :: LowCardinality(String) AS test_name ,

7
docs/en/operations/utilities/clickhouse-keeper-client.md
View File

@ -45,9 +45,14 @@ keeper foo bar
- `ls [path]` -- Lists the nodes for the given path (default: cwd)
- `cd [path]` -- Change the working path (default `.`)
- `set <path> <value> [version]` -- Updates the node's value. Only update if version matches (default: -1)
- `create <path> <value>` -- Creates new node
- `create <path> <value> [mode]` -- Creates new node with the set value
- `touch <path>` -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists
- `get <path>` -- Returns the node's value
- `remove <path>` -- Remove the node
- `rmr <path>` -- Recursively deletes path. Confirmation required
- `flwc <command>` -- Executes four-letter-word command
- `help` -- Prints this message
- `get_stat [path]` -- Returns the node's stat (default `.`)
- `find_super_nodes <threshold> [path]` -- Finds nodes with number of children larger than some threshold for the given path (default `.`)
- `delete_stale_backups` -- Deletes ClickHouse nodes used for backups that are now inactive
- `find_big_family [path] [n]` -- Returns the top n nodes with the biggest family in the subtree (default path = `.` and n = 10)

43
docs/en/sql-reference/functions/string-functions.md
View File

@ -729,6 +729,30 @@ Returns whether string `str` ends with `suffix`.
endsWith(str, suffix)
```
## endsWithUTF8
Returns whether string `str` ends with `suffix`, the difference between `endsWithUTF8` and `endsWith` is that `endsWithUTF8` match `str` and `suffix` by UTF-8 characters.
**Syntax**
```sql
endsWithUTF8(str, suffix)
```
**Example**
``` sql
SELECT endsWithUTF8('中国', '\xbd'), endsWith('中国', '\xbd')
```
Result:
```result
┌─endsWithUTF8('中国', '½')─┬─endsWith('中国', '½')─┐
│ 0 │ 1 │
└──────────────────────────┴──────────────────────┘
```
## startsWith
Returns whether string `str` starts with `prefix`.
@ -745,6 +769,25 @@ startsWith(str, prefix)
SELECT startsWith('Spider-Man', 'Spi');
```
## startsWithUTF8
Returns whether string `str` starts with `prefix`, the difference between `startsWithUTF8` and `startsWith` is that `startsWithUTF8` match `str` and `suffix` by UTF-8 characters.
**Example**
``` sql
SELECT startsWithUTF8('中国', '\xe4'), startsWith('中国', '\xe4')
```
Result:
```result
┌─startsWithUTF8('中国', '⥩─┬─startsWith('中国', '⥩─┐
│ 0 │ 1 │
└────────────────────────────┴────────────────────────┘
```
## trim
Removes the specified characters from the start or end of a string. If not specified otherwise, the function removes whitespace (ASCII-character 32).

107
docs/en/sql-reference/statements/check-table.md
View File

@ -5,38 +5,19 @@ sidebar_label: CHECK TABLE
title: "CHECK TABLE Statement"
---
The `CHECK TABLE` query in ClickHouse is used to perform a validation check on a specific table or its partitions. It ensures the integrity of the data by verifying the checksums and other internal data structures.
Checks if the data in the table is corrupted.
Particularly it compares actual file sizes with the expected values which are stored on the server. If the file sizes do not match the stored values, it means the data is corrupted. This can be caused, for example, by a system crash during query execution.
:::note
The `CHECK TABLE`` query may read all the data in the table and hold some resources, making it resource-intensive.
Consider the potential impact on performance and resource utilization before executing this query.
:::
## Syntax
The basic syntax of the query is as follows:
```sql
CHECK TABLE table_name [PARTITION partition_expression] [FORMAT format] [SETTINGS check_query_single_value_result = (0|1) [, other_settings] ]
``` sql
CHECK TABLE [db.]name [PARTITION partition_expr]
```
- `table_name`: Specifies the name of the table that you want to check.
- `partition_expression`: (Optional) If you want to check a specific partition of the table, you can use this expression to specify the partition.
- `FORMAT format`: (Optional) Allows you to specify the output format of the result.
- `SETTINGS`: (Optional) Allows additional settings.
- **`check_query_single_value_result`**: (Optional) This setting allows you to toggle between a detailed result (`0`) or a summarized result (`1`).
- Other settings (e.g. `max_threads` can be applied as well).
The `CHECK TABLE` query compares actual file sizes with the expected values which are stored on the server. If the file sizes do not match the stored values, it means the data is corrupted. This can be caused, for example, by a system crash during query execution.
The query response contains the `result` column with a single row. The row has a value of
[Boolean](../../sql-reference/data-types/boolean.md) type:
The query response depends on the value of contains `check_query_single_value_result` setting.
In case of `check_query_single_value_result = 1` only `result` column with a single row is returned. Value inside this row is `1` if the integrity check is passed and `0` if data is corrupted.
With `check_query_single_value_result = 0` the query returns the following columns:
- `part_path`: Indicates the path to the data part or file name.
- `is_passed`: Returns 1 if the check for this part is successful, 0 otherwise.
- `message`: Any additional messages related to the check, such as errors or success messages.
- 0 - The data in the table is corrupted.
- 1 - The data maintains integrity.
The `CHECK TABLE` query supports the following table engines:
@ -45,15 +26,30 @@ The `CHECK TABLE` query supports the following table engines:
- [StripeLog](../../engines/table-engines/log-family/stripelog.md)
- [MergeTree family](../../engines/table-engines/mergetree-family/mergetree.md)
Performed over the tables with another table engines causes an `NOT_IMPLEMETED` exception.
Performed over the tables with another table engines causes an exception.
Engines from the `*Log` family do not provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner.
## Examples
## Checking the MergeTree Family Tables
By default `CHECK TABLE` query shows the general table check status:
For `MergeTree` family engines, if [check_query_single_value_result](../../operations/settings/settings.md#check_query_single_value_result) = 0, the `CHECK TABLE` query shows a check status for every individual data part of a table on the local server.
```sql
SET check_query_single_value_result = 0;
CHECK TABLE test_table;
```
```text
┌─part_path─┬─is_passed─┬─message─┐
│ all_1_4_1 │ 1 │ │
│ all_1_4_2 │ 1 │ │
└───────────┴───────────┴─────────┘
```
If `check_query_single_value_result` = 1, the `CHECK TABLE` query shows the general table check status.
```sql
SET check_query_single_value_result = 1;
CHECK TABLE test_table;
```
@ -63,60 +59,11 @@ CHECK TABLE test_table;
└────────┘
```
If you want to see the check status for every individual data part you may use `check_query_single_value_result` setting.
Also, to check a specific partition of the table, you can use the `PARTITION` keyword.
```sql
CHECK TABLE t0 PARTITION ID '201003'
FORMAT PrettyCompactMonoBlock
SETTINGS check_query_single_value_result = 0
```
Output:
```text
┌─part_path────┬─is_passed─┬─message─┐
│ 201003_7_7_0 │ 1 │ │
│ 201003_3_3_0 │ 1 │ │
└──────────────┴───────────┴─────────┘
```
### Receiving a 'Corrupted' Result
:::warning
Disclaimer: The procedure described here, including the manual manipulating or removing files directly from the data directory, is for experimental or development environments only. Do **not** attempt this on a production server, as it may lead to data loss or other unintended consequences.
:::
Remove the existing checksum file:
```bash
rm /var/lib/clickhouse-server/data/default/t0/201003_3_3_0/checksums.txt
```
```sql
CHECK TABLE t0 PARTITION ID '201003'
FORMAT PrettyCompactMonoBlock
SETTINGS check_query_single_value_result = 0
Output:
```text
┌─part_path────┬─is_passed─┬─message──────────────────────────────────┐
│ 201003_7_7_0 │ 1 │ │
│ 201003_3_3_0 │ 1 │ Checksums recounted and written to disk. │
└──────────────┴───────────┴──────────────────────────────────────────┘
```
If the checksums.txt file is missing, it can be restored. It will be recalculated and rewritten during the execution of the CHECK TABLE command for the specific partition, and the status will still be reported as 'success.'"
## If the Data Is Corrupted
If the table is corrupted, you can copy the non-corrupted data to another table. To do this:
1. Create a new table with the same structure as damaged table. To do this execute the query `CREATE TABLE <new_table_name> AS <damaged_table_name>`.
2. Set the `max_threads` value to 1 to process the next query in a single thread. To do this run the query `SET max_threads = 1`.
2. Set the [max_threads](../../operations/settings/settings.md#settings-max_threads) value to 1 to process the next query in a single thread. To do this run the query `SET max_threads = 1`.
3. Execute the query `INSERT INTO <new_table_name> SELECT * FROM <damaged_table_name>`. This request copies the non-corrupted data from the damaged table to another table. Only the data before the corrupted part will be copied.
4. Restart the `clickhouse-client` to reset the `max_threads` value.

1
docs/en/sql-reference/statements/create/quota.md
View File

@ -11,6 +11,7 @@ Syntax:
``` sql
CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]
[IN access_storage_type]
[KEYED BY {user_name | ip_address | client_key | client_key,user_name | client_key,ip_address} | NOT KEYED]
[FOR [RANDOMIZED] INTERVAL number {second | minute | hour | day | week | month | quarter | year}
{MAX { {queries | query_selects | query_inserts | errors | result_rows | result_bytes | read_rows | read_bytes | execution_time} = number } [,...] |

1
docs/en/sql-reference/statements/create/role.md
View File

@ -11,6 +11,7 @@ Syntax:
``` sql
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...]
[IN access_storage_type]
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...]
```

1
docs/en/sql-reference/statements/create/row-policy.md
View File

@ -16,6 +16,7 @@ Syntax:
``` sql
CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1|db1.*
[, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2|db2.* ...]
[IN access_storage_type]
[FOR SELECT] USING condition
[AS {PERMISSIVE | RESTRICTIVE}]
[TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}]

1
docs/en/sql-reference/statements/create/settings-profile.md
View File

@ -12,6 +12,7 @@ Syntax:
``` sql
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
[IN access_storage_type]
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...]
```

1
docs/en/sql-reference/statements/create/user.md
View File

@ -14,6 +14,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'}]
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
[IN access_storage_type]
[DEFAULT ROLE role [,...]]
[DEFAULT DATABASE database | NONE]
[GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]

10
docs/en/sql-reference/statements/drop.md
View File

@ -49,7 +49,7 @@ Deletes a user.
Syntax:
``` sql
DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
```
## DROP ROLE
@ -59,7 +59,7 @@ Deletes a role. The deleted role is revoked from all the entities where it was a
Syntax:
``` sql
DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
```
## DROP ROW POLICY
@ -69,7 +69,7 @@ Deletes a row policy. Deleted row policy is revoked from all the entities where
Syntax:
``` sql
DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name]
DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
```
## DROP QUOTA
@ -79,7 +79,7 @@ Deletes a quota. The deleted quota is revoked from all the entities where it was
Syntax:
``` sql
DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
```
## DROP SETTINGS PROFILE
@ -89,7 +89,7 @@ Deletes a settings profile. The deleted settings profile is revoked from all the
Syntax:
``` sql
DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
```
## DROP VIEW

32
docs/en/sql-reference/statements/move.md Normal file
View File

@ -0,0 +1,32 @@
---
slug: /en/sql-reference/statements/move
sidebar_position: 54
sidebar_label: MOVE
---
# MOVE access entity statement
This statement allows to move an access entity from one access storage to another.
Syntax:
```sql
MOVE {USER, ROLE, QUOTA, SETTINGS PROFILE, ROW POLICY} name1 [, name2, ...] TO access_storage_type
```
Currently, there are five access storages in ClickHouse:
- `local_directory`
- `memory`
- `replicated`
- `users_xml` (ro)
- `ldap` (ro)
Examples:
```sql
MOVE USER test TO local_directory
```
```sql
MOVE ROLE test TO memory
```

32
docs/ru/getting-started/example-datasets/wikistat.md
View File

@ -1,32 +0,0 @@
---
slug: /ru/getting-started/example-datasets/wikistat
sidebar_position: 17
sidebar_label: WikiStat
---
# WikiStat {#wikistat}
См: http://dumps.wikimedia.org/other/pagecounts-raw/
Создание таблицы:
``` sql
CREATE TABLE wikistat
(
date Date,
time DateTime,
project String,
subproject String,
path String,
hits UInt64,
size UInt64
) ENGINE = MergeTree(date, (path, time), 8192);
```
Загрузка данных:
``` bash
$ for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt
$ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done
$ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done
```

1
docs/ru/getting-started/example-datasets/wikistat.md Symbolic link
View File

@ -0,0 +1 @@
../../../en/getting-started/example-datasets/wikistat.md

32
docs/zh/getting-started/example-datasets/wikistat.md
View File

@ -1,32 +0,0 @@
---
slug: /zh/getting-started/example-datasets/wikistat
sidebar_position: 17
sidebar_label: WikiStat
---
# WikiStat {#wikistat}
参考: http://dumps.wikimedia.org/other/pagecounts-raw/
创建表结构：
``` sql
CREATE TABLE wikistat
(
date Date,
time DateTime,
project String,
subproject String,
path String,
hits UInt64,
size UInt64
) ENGINE = MergeTree(date, (path, time), 8192);
```
加载数据：
``` bash
$ for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt
$ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done
$ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done
```

1
docs/zh/getting-started/example-datasets/wikistat.md Symbolic link
View File

@ -0,0 +1 @@
../../../en/getting-started/example-datasets/wikistat.md

2
programs/disks/ICommand.cpp
View File

@ -1,4 +1,6 @@
#include "ICommand.h"
#include <iostream>
namespace DB
{

199
programs/keeper-client/Commands.cpp
View File

@ -1,5 +1,6 @@
#include "Commands.h"
#include <queue>
#include "KeeperClient.h"
@ -24,8 +25,18 @@ void LSCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) con
else
path = client->cwd;
for (const auto & child : client->zookeeper->getChildren(path))
std::cout << child << " ";
auto children = client->zookeeper->getChildren(path);
std::sort(children.begin(), children.end());
bool need_space = false;
for (const auto & child : children)
{
if (std::exchange(need_space, true))
std::cout << " ";
std::cout << child;
}
std::cout << "\n";
}
@ -115,6 +126,21 @@ void CreateCommand::execute(const ASTKeeperQuery * query, KeeperClient * client)
static_cast<int>(query->args[2].safeGet<Int64>()));
}
bool TouchCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
{
String arg;
if (!parseKeeperPath(pos, expected, arg))
return false;
node->args.push_back(std::move(arg));
return true;
}
void TouchCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) const
{
client->zookeeper->createIfNotExists(client->getAbsolutePath(query->args[0].safeGet<String>()), "");
}
bool GetCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
{
String arg;
@ -130,6 +156,173 @@ void GetCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) co
std::cout << client->zookeeper->get(client->getAbsolutePath(query->args[0].safeGet<String>())) << "\n";
}
bool GetStatCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
{
String arg;
if (!parseKeeperPath(pos, expected, arg))
return true;
node->args.push_back(std::move(arg));
return true;
}
void GetStatCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) const
{
Coordination::Stat stat;
String path;
if (!query->args.empty())
path = client->getAbsolutePath(query->args[0].safeGet<String>());
else
path = client->cwd;
client->zookeeper->get(path, &stat);
std::cout << "cZxid = " << stat.czxid << "\n";
std::cout << "mZxid = " << stat.mzxid << "\n";
std::cout << "pZxid = " << stat.pzxid << "\n";
std::cout << "ctime = " << stat.ctime << "\n";
std::cout << "mtime = " << stat.mtime << "\n";
std::cout << "version = " << stat.version << "\n";
std::cout << "cversion = " << stat.cversion << "\n";
std::cout << "aversion = " << stat.aversion << "\n";
std::cout << "ephemeralOwner = " << stat.ephemeralOwner << "\n";
std::cout << "dataLength = " << stat.dataLength << "\n";
std::cout << "numChildren = " << stat.numChildren << "\n";
}
bool FindSuperNodes::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
{
ASTPtr threshold;
if (!ParserUnsignedInteger{}.parse(pos, threshold, expected))
return false;
node->args.push_back(threshold->as<ASTLiteral &>().value);
String path;
if (!parseKeeperPath(pos, expected, path))
path = ".";
node->args.push_back(std::move(path));
return true;
}
void FindSuperNodes::execute(const ASTKeeperQuery * query, KeeperClient * client) const
{
auto threshold = query->args[0].safeGet<UInt64>();
auto path = client->getAbsolutePath(query->args[1].safeGet<String>());
Coordination::Stat stat;
client->zookeeper->get(path, &stat);
if (stat.numChildren >= static_cast<Int32>(threshold))
{
std::cout << static_cast<String>(path) << "\t" << stat.numChildren << "\n";
return;
}
auto children = client->zookeeper->getChildren(path);
std::sort(children.begin(), children.end());
for (const auto & child : children)
{
auto next_query = *query;
next_query.args[1] = DB::Field(path / child);
execute(&next_query, client);
}
}
bool DeleteStaleBackups::parse(IParser::Pos & /* pos */, std::shared_ptr<ASTKeeperQuery> & /* node */, Expected & /* expected */) const
{
return true;
}
void DeleteStaleBackups::execute(const ASTKeeperQuery * /* query */, KeeperClient * client) const
{
client->askConfirmation(
"You are going to delete all inactive backups in /clickhouse/backups.",
[client]
{
fs::path backup_root = "/clickhouse/backups";
auto backups = client->zookeeper->getChildren(backup_root);
std::sort(backups.begin(), backups.end());
for (const auto & child : backups)
{
auto backup_path = backup_root / child;
std::cout << "Found backup " << backup_path << ", checking if it's active\n";
String stage_path = backup_path / "stage";
auto stages = client->zookeeper->getChildren(stage_path);
bool is_active = false;
for (const auto & stage : stages)
{
if (startsWith(stage, "alive"))
{
is_active = true;
break;
}
}
if (is_active)
{
std::cout << "Backup " << backup_path << " is active, not going to delete\n";
continue;
}
std::cout << "Backup " << backup_path << " is not active, deleting it\n";
client->zookeeper->removeRecursive(backup_path);
}
});
}
bool FindBigFamily::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
{
String path;
if (!parseKeeperPath(pos, expected, path))
path = ".";
node->args.push_back(std::move(path));
ASTPtr count;
if (ParserUnsignedInteger{}.parse(pos, count, expected))
node->args.push_back(count->as<ASTLiteral &>().value);
else
node->args.push_back(UInt64(10));
return true;
}
void FindBigFamily::execute(const ASTKeeperQuery * query, KeeperClient * client) const
{
auto path = client->getAbsolutePath(query->args[0].safeGet<String>());
auto n = query->args[1].safeGet<UInt64>();
std::vector<std::tuple<Int32, String>> result;
std::queue<fs::path> queue;
queue.push(path);
while (!queue.empty())
{
auto next_path = queue.front();
queue.pop();
auto children = client->zookeeper->getChildren(next_path);
std::transform(children.cbegin(), children.cend(), children.begin(), [&](const String & child) { return next_path / child; });
auto response = client->zookeeper->get(children);
for (size_t i = 0; i < response.size(); ++i)
{
result.emplace_back(response[i].stat.numChildren, children[i]);
queue.push(children[i]);
}
}
std::sort(result.begin(), result.end(), std::greater());
for (UInt64 i = 0; i < std::min(result.size(), static_cast<size_t>(n)); ++i)
std::cout << std::get<1>(result[i]) << "\t" << std::get<0>(result[i]) << "\n";
}
bool RMCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
{
String arg;
@ -170,7 +363,7 @@ bool HelpCommand::parse(IParser::Pos & /* pos */, std::shared_ptr<ASTKeeperQuery
void HelpCommand::execute(const ASTKeeperQuery * /* query */, KeeperClient * /* client */) const
{
for (const auto & pair : KeeperClient::commands)
std::cout << pair.second->getHelpMessage() << "\n";
std::cout << pair.second->generateHelpString() << "\n";
}
bool FourLetterWordCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const

89
programs/keeper-client/Commands.h
View File

@ -21,6 +21,12 @@ public:
virtual String getName() const = 0;
virtual ~IKeeperClientCommand() = default;
String generateHelpString() const
{
return fmt::vformat(getHelpMessage(), fmt::make_format_args(getName()));
}
};
using Command = std::shared_ptr<IKeeperClientCommand>;
@ -34,7 +40,7 @@ class LSCommand : public IKeeperClientCommand
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "ls [path] -- Lists the nodes for the given path (default: cwd)"; }
String getHelpMessage() const override { return "{} [path] -- Lists the nodes for the given path (default: cwd)"; }
};
class CDCommand : public IKeeperClientCommand
@ -45,7 +51,7 @@ class CDCommand : public IKeeperClientCommand
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "cd [path] -- Change the working path (default `.`)"; }
String getHelpMessage() const override { return "{} [path] -- Change the working path (default `.`)"; }
};
class SetCommand : public IKeeperClientCommand
@ -58,7 +64,7 @@ class SetCommand : public IKeeperClientCommand
String getHelpMessage() const override
{
return "set <path> <value> [version] -- Updates the node's value. Only update if version matches (default: -1)";
return "{} <path> <value> [version] -- Updates the node's value. Only update if version matches (default: -1)";
}
};
@ -70,7 +76,18 @@ class CreateCommand : public IKeeperClientCommand
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "create <path> <value> -- Creates new node"; }
String getHelpMessage() const override { return "{} <path> <value> [mode] -- Creates new node with the set value"; }
};
class TouchCommand : public IKeeperClientCommand
{
String getName() const override { return "touch"; }
bool parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const override;
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "{} <path> -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists"; }
};
class GetCommand : public IKeeperClientCommand
@ -81,9 +98,63 @@ class GetCommand : public IKeeperClientCommand
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "get <path> -- Returns the node's value"; }
String getHelpMessage() const override { return "{} <path> -- Returns the node's value"; }
};
class GetStatCommand : public IKeeperClientCommand
{
String getName() const override { return "get_stat"; }
bool parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const override;
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "{} [path] -- Returns the node's stat (default `.`)"; }
};
class FindSuperNodes : public IKeeperClientCommand
{
String getName() const override { return "find_super_nodes"; }
bool parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const override;
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override
{
return "{} <threshold> [path] -- Finds nodes with number of children larger than some threshold for the given path (default `.`)";
}
};
class DeleteStaleBackups : public IKeeperClientCommand
{
String getName() const override { return "delete_stale_backups"; }
bool parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const override;
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override
{
return "{} -- Deletes ClickHouse nodes used for backups that are now inactive";
}
};
class FindBigFamily : public IKeeperClientCommand
{
String getName() const override { return "find_big_family"; }
bool parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const override;
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override
{
return "{} [path] [n] -- Returns the top n nodes with the biggest family in the subtree (default path = `.` and n = 10)";
}
};
class RMCommand : public IKeeperClientCommand
{
String getName() const override { return "rm"; }
@ -92,7 +163,7 @@ class RMCommand : public IKeeperClientCommand
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "remove <path> -- Remove the node"; }
String getHelpMessage() const override { return "{} <path> -- Remove the node"; }
};
class RMRCommand : public IKeeperClientCommand
@ -103,7 +174,7 @@ class RMRCommand : public IKeeperClientCommand
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "rmr <path> -- Recursively deletes path. Confirmation required"; }
String getHelpMessage() const override { return "{} <path> -- Recursively deletes path. Confirmation required"; }
};
class HelpCommand : public IKeeperClientCommand
@ -114,7 +185,7 @@ class HelpCommand : public IKeeperClientCommand
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "help -- Prints this message"; }
String getHelpMessage() const override { return "{} -- Prints this message"; }
};
class FourLetterWordCommand : public IKeeperClientCommand
@ -125,7 +196,7 @@ class FourLetterWordCommand : public IKeeperClientCommand
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
String getHelpMessage() const override { return "flwc <command> -- Executes four-letter-word command"; }
String getHelpMessage() const override { return "{} <command> -- Executes four-letter-word command"; }
};
}

5
programs/keeper-client/KeeperClient.cpp
View File

@ -176,7 +176,12 @@ void KeeperClient::initialize(Poco::Util::Application & /* self */)
std::make_shared<CDCommand>(),
std::make_shared<SetCommand>(),
std::make_shared<CreateCommand>(),
std::make_shared<TouchCommand>(),
std::make_shared<GetCommand>(),
std::make_shared<GetStatCommand>(),
std::make_shared<FindSuperNodes>(),
std::make_shared<DeleteStaleBackups>(),
std::make_shared<FindBigFamily>(),
std::make_shared<RMCommand>(),
std::make_shared<RMRCommand>(),
std::make_shared<HelpCommand>(),

1
programs/keeper/CMakeLists.txt
View File

@ -57,6 +57,7 @@ if (BUILD_STANDALONE_KEEPER)
${CMAKE_CURRENT_SOURCE_DIR}/../../src/IO/ReadBuffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTPPathHints.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/KeeperTCPHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/TCPServer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/NotFoundHandler.cpp

2
programs/keeper/Keeper.cpp
View File

@ -150,7 +150,7 @@ int Keeper::run()
}
if (config().hasOption("version"))
{
std::cout << DBMS_NAME << " keeper version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
std::cout << VERSION_NAME << " keeper version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
return 0;
}

2
programs/server/Server.cpp
View File

@ -389,7 +389,7 @@ int Server::run()
}
if (config().hasOption("version"))
{
std::cout << DBMS_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
std::cout << VERSION_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
return 0;
}
return Application::run(); // NOLINT

2
programs/server/config.xml
View File

@ -317,7 +317,7 @@
<concurrent_threads_soft_limit_ratio_to_cores>0</concurrent_threads_soft_limit_ratio_to_cores>
<!-- Maximum number of concurrent queries. -->
<max_concurrent_queries>100</max_concurrent_queries>
<max_concurrent_queries>1000</max_concurrent_queries>
<!-- Maximum memory usage (resident set size) for server process.
Zero value or unset means default. Default is "max_server_memory_usage_to_ram_ratio" of available physical RAM.

1
rust/.cargo/config.toml.in
View File

@ -5,6 +5,7 @@ CXXFLAGS = "@RUST_CXXFLAGS@"
[build]
rustflags = @RUSTFLAGS@
rustdocflags = @RUSTFLAGS@
@RUSTCWRAPPER@
[unstable]
@RUST_CARGO_BUILD_STD@

4
rust/.dockerignore
View File

@ -1,4 +0,0 @@
# Just in case ignore any cargo stuff (and just in case someone will run this
# docker build locally with build context using folder root):
target
vendor

4
rust/.gitignore vendored
View File

@ -1,4 +0,0 @@
# This is for tar --exclude-vcs-ignores (and just in case someone will run
# docker build locally with build context created via tar):
target
vendor

7
rust/CMakeLists.txt
View File

@ -14,6 +14,13 @@ macro(configure_rustc)
set(RUST_CFLAGS "${RUST_CFLAGS} --sysroot ${CMAKE_SYSROOT}")
endif()
if(CCACHE_EXECUTABLE MATCHES "/sccache$")
message(STATUS "Using RUSTC_WRAPPER: ${CCACHE_EXECUTABLE}")
set(RUSTCWRAPPER "rustc-wrapper = \"${CCACHE_EXECUTABLE}\"")
else()
set(RUSTCWRAPPER "")
endif()
set(RUSTFLAGS "[]")
set(RUST_CARGO_BUILD_STD "")
# For more info: https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html#memorysanitizer

12
src/Access/AccessControl.cpp
View File

@ -418,7 +418,7 @@ void AccessControl::addStoragesFromUserDirectoriesConfig(
String type = key_in_user_directories;
if (size_t bracket_pos = type.find('['); bracket_pos != String::npos)
type.resize(bracket_pos);
if ((type == "users_xml") || (type == "users_config"))
if ((type == "users.xml") || (type == "users_config"))
type = UsersConfigAccessStorage::STORAGE_TYPE;
else if ((type == "local") || (type == "local_directory"))
type = DiskAccessStorage::STORAGE_TYPE;
@ -528,12 +528,14 @@ scope_guard AccessControl::subscribeForChanges(const std::vector<UUID> & ids, co
return changes_notifier->subscribeForChanges(ids, handler);
}
std::optional<UUID> AccessControl::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
bool AccessControl::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
{
auto id = MultipleAccessStorage::insertImpl(entity, replace_if_exists, throw_if_exists);
if (id)
if (MultipleAccessStorage::insertImpl(id, entity, replace_if_exists, throw_if_exists))
{
changes_notifier->sendNotifications();
return id;
return true;
}
return false;
}
bool AccessControl::removeImpl(const UUID & id, bool throw_if_not_exists)

2
src/Access/AccessControl.h
View File

@ -232,7 +232,7 @@ private:
class CustomSettingsPrefixes;
class PasswordComplexityRules;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;

16
src/Access/DiskAccessStorage.cpp
View File

@ -498,20 +498,10 @@ std::optional<std::pair<String, AccessEntityType>> DiskAccessStorage::readNameWi
}
std::optional<UUID> DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
UUID id = generateRandomID();
if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true))
return id;
return std::nullopt;
}
bool DiskAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk)
bool DiskAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
std::lock_guard lock{mutex};
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, write_on_disk);
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk = */ true);
}
@ -745,7 +735,7 @@ void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
{
for (const auto & [id, entity] : my_entities)
insertWithID(id, entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true);
insert(id, entity, replace_if_exists, throw_if_exists);
});
}

5
src/Access/DiskAccessStorage.h
View File

@ -13,7 +13,7 @@ class AccessChangesNotifier;
class DiskAccessStorage : public IAccessStorage
{
public:
static constexpr char STORAGE_TYPE[] = "local directory";
static constexpr char STORAGE_TYPE[] = "local_directory";
DiskAccessStorage(const String & storage_name_, const String & directory_path_, AccessChangesNotifier & changes_notifier_, bool readonly_, bool allow_backup_);
~DiskAccessStorage() override;
@ -39,7 +39,7 @@ private:
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
@ -53,7 +53,6 @@ private:
void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS;
void stopListsWritingThread();
bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk);
bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) TSA_REQUIRES(mutex);
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
bool removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);

62
src/Access/IAccessStorage.cpp
View File

@ -93,6 +93,17 @@ String IAccessStorage::readName(const UUID & id) const
}
bool IAccessStorage::exists(const std::vector<UUID> & ids) const
{
for (const auto & id : ids)
{
if (!exists(id))
return false;
}
return true;
}
std::optional<String> IAccessStorage::readName(const UUID & id, bool throw_if_not_exists) const
{
if (auto name_and_type = readNameWithType(id, throw_if_not_exists))
@ -167,38 +178,69 @@ UUID IAccessStorage::insert(const AccessEntityPtr & entity)
return *insert(entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true);
}
std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
{
return insertImpl(entity, replace_if_exists, throw_if_exists);
auto id = generateRandomID();
if (insert(id, entity, replace_if_exists, throw_if_exists))
return id;
return std::nullopt;
}
bool IAccessStorage::insert(const DB::UUID & id, const DB::AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
{
return insertImpl(id, entity, replace_if_exists, throw_if_exists);
}
std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists, bool throw_if_exists)
{
return insert(multiple_entities, /* ids = */ {}, replace_if_exists, throw_if_exists);
}
std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & multiple_entities, const std::vector<UUID> & ids, bool replace_if_exists, bool throw_if_exists)
{
assert(ids.empty() || (multiple_entities.size() == ids.size()));
if (multiple_entities.empty())
return {};
if (multiple_entities.size() == 1)
{
if (auto id = insert(multiple_entities[0], replace_if_exists, throw_if_exists))
return {*id};
UUID id;
if (!ids.empty())
id = ids[0];
else
id = generateRandomID();
if (insert(id, multiple_entities[0], replace_if_exists, throw_if_exists))
return {id};
return {};
}
std::vector<AccessEntityPtr> successfully_inserted;
try
{
std::vector<UUID> ids;
for (const auto & entity : multiple_entities)
std::vector<UUID> new_ids;
for (size_t i = 0; i < multiple_entities.size(); ++i)
{
if (auto id = insertImpl(entity, replace_if_exists, throw_if_exists))
const auto & entity = multiple_entities[i];
UUID id;
if (!ids.empty())
id = ids[i];
else
id = generateRandomID();
if (insert(id, entity, replace_if_exists, throw_if_exists))
{
successfully_inserted.push_back(entity);
ids.push_back(*id);
new_ids.push_back(id);
}
}
return ids;
return new_ids;
}
catch (Exception & e)
{
@ -244,7 +286,7 @@ std::vector<UUID> IAccessStorage::insertOrReplace(const std::vector<AccessEntity
}
std::optional<UUID> IAccessStorage::insertImpl(const AccessEntityPtr & entity, bool, bool)
bool IAccessStorage::insertImpl(const UUID &, const AccessEntityPtr & entity, bool, bool)
{
if (isReadOnly())
throwReadonlyCannotInsert(entity->getType(), entity->getName());

28
src/Access/IAccessStorage.h
View File

@ -3,6 +3,8 @@
#include <Access/IAccessEntity.h>
#include <Core/Types.h>
#include <Core/UUID.h>
#include <Parsers/IParser.h>
#include <Parsers/parseIdentifierOrStringLiteral.h>
#include <functional>
#include <optional>
#include <vector>
@ -92,6 +94,7 @@ public:
/// Returns whether there is an entity with such identifier in the storage.
virtual bool exists(const UUID & id) const = 0;
bool exists(const std::vector<UUID> & ids) const;
/// Reads an entity. Throws an exception if not found.
template <typename EntityClassT = IAccessEntity>
@ -100,6 +103,9 @@ public:
template <typename EntityClassT = IAccessEntity>
std::shared_ptr<const EntityClassT> read(const String & name, bool throw_if_not_exists = true) const;
template <typename EntityClassT = IAccessEntity>
std::vector<AccessEntityPtr> read(const std::vector<UUID> & ids, bool throw_if_not_exists = true) const;
/// Reads an entity. Returns nullptr if not found.
template <typename EntityClassT = IAccessEntity>
std::shared_ptr<const EntityClassT> tryRead(const UUID & id) const;
@ -128,7 +134,9 @@ public:
/// Throws an exception if the specified name already exists.
UUID insert(const AccessEntityPtr & entity);
std::optional<UUID> insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
bool insert(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists = false, bool throw_if_exists = true);
std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, const std::vector<UUID> & ids, bool replace_if_exists = false, bool throw_if_exists = true);
/// Inserts an entity to the storage. Returns ID of a new entry in the storage.
std::optional<UUID> tryInsert(const AccessEntityPtr & entity);
@ -179,7 +187,7 @@ protected:
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const;
virtual std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
virtual bool removeImpl(const UUID & id, bool throw_if_not_exists);
virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
virtual std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const;
@ -240,6 +248,19 @@ std::shared_ptr<const EntityClassT> IAccessStorage::read(const String & name, bo
}
template <typename EntityClassT>
std::vector<AccessEntityPtr> IAccessStorage::read(const std::vector<UUID> & ids, bool throw_if_not_exists) const
{
std::vector<AccessEntityPtr> result;
result.reserve(ids.size());
for (const auto & id : ids)
result.push_back(read<EntityClassT>(id, throw_if_not_exists));
return result;
}
template <typename EntityClassT>
std::shared_ptr<const EntityClassT> IAccessStorage::tryRead(const UUID & id) const
{
@ -265,4 +286,9 @@ std::vector<std::pair<UUID, std::shared_ptr<const EntityClassT>>> IAccessStorage
return entities;
}
inline bool parseAccessStorageName(IParser::Pos & pos, Expected & expected, String & storage_name)
{
return parseIdentifierOrStringLiteral(pos, expected, storage_name);
}
}

14
src/Access/MemoryAccessStorage.cpp
View File

@ -63,17 +63,7 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id, bool throw_if_not
}
std::optional<UUID> MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
UUID id = generateRandomID();
if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
return id;
return std::nullopt;
}
bool MemoryAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
bool MemoryAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
std::lock_guard lock{mutex};
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists);
@ -300,7 +290,7 @@ void MemoryAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
{
for (const auto & [id, entity] : my_entities)
insertWithID(id, entity, replace_if_exists, throw_if_exists);
insert(id, entity, replace_if_exists, throw_if_exists);
});
}

8
src/Access/MemoryAccessStorage.h
View File

@ -6,6 +6,7 @@
#include <memory>
#include <mutex>
#include <unordered_map>
#include <boost/container/flat_set.hpp>
namespace DB
@ -22,11 +23,6 @@ public:
const char * getStorageType() const override { return STORAGE_TYPE; }
/// Inserts an entity with a specified ID.
/// If `replace_if_exists == true` it can replace an existing entry with such ID and also remove an existing entry
/// with such name & type.
bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists);
/// Removes all entities except the specified list `ids_to_keep`.
/// The function skips IDs not contained in the storage.
void removeAllExcept(const std::vector<UUID> & ids_to_keep);
@ -44,7 +40,7 @@ private:
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;

97
src/Access/MultipleAccessStorage.cpp
View File

@ -16,6 +16,7 @@ namespace ErrorCodes
{
extern const int ACCESS_ENTITY_ALREADY_EXISTS;
extern const int ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND;
extern const int ACCESS_ENTITY_NOT_FOUND;
}
using Storage = IAccessStorage;
@ -178,6 +179,91 @@ ConstStoragePtr MultipleAccessStorage::getStorage(const UUID & id) const
return const_cast<MultipleAccessStorage *>(this)->getStorage(id);
}
StoragePtr MultipleAccessStorage::findStorageByName(const DB::String & storage_name)
{
auto storages = getStoragesInternal();
for (const auto & storage : *storages)
{
if (storage->getStorageName() == storage_name)
return storage;
}
return nullptr;
}
ConstStoragePtr MultipleAccessStorage::findStorageByName(const DB::String & storage_name) const
{
return const_cast<MultipleAccessStorage *>(this)->findStorageByName(storage_name);
}
StoragePtr MultipleAccessStorage::getStorageByName(const DB::String & storage_name)
{
auto storage = findStorageByName(storage_name);
if (storage)
return storage;
throw Exception(ErrorCodes::ACCESS_ENTITY_NOT_FOUND, "Access storage with name {} is not found", storage_name);
}
ConstStoragePtr MultipleAccessStorage::getStorageByName(const DB::String & storage_name) const
{
return const_cast<MultipleAccessStorage *>(this)->getStorageByName(storage_name);
}
StoragePtr MultipleAccessStorage::findExcludingStorage(AccessEntityType type, const DB::String & name, DB::MultipleAccessStorage::StoragePtr exclude) const
{
auto storages = getStoragesInternal();
for (const auto & storage : *storages)
{
if (storage == exclude)
continue;
if (storage->find(type, name))
return storage;
}
return nullptr;
}
void MultipleAccessStorage::moveAccessEntities(const std::vector<UUID> & ids, const String & source_storage_name, const String & destination_storage_name)
{
auto source_storage = getStorageByName(source_storage_name);
auto destination_storage = getStorageByName(destination_storage_name);
auto to_move = source_storage->read(ids);
bool need_rollback = false;
try
{
source_storage->remove(ids);
need_rollback = true;
destination_storage->insert(to_move, ids);
}
catch (Exception & e)
{
String message;
bool need_comma = false;
for (const auto & entity : to_move)
{
if (std::exchange(need_comma, true))
message += ", ";
message += entity->formatTypeWithName();
}
e.addMessage("while moving {} from {} to {}", message, source_storage_name, destination_storage_name);
if (need_rollback)
source_storage->insert(to_move, ids);
throw;
}
}
AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
{
if (auto storage = findStorage(id))
@ -245,7 +331,7 @@ void MultipleAccessStorage::reload(ReloadMode reload_mode)
}
std::optional<UUID> MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
bool MultipleAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
{
std::shared_ptr<IAccessStorage> storage_for_insertion;
@ -268,13 +354,14 @@ std::optional<UUID> MultipleAccessStorage::insertImpl(const AccessEntityPtr & en
getStorageName());
}
auto id = storage_for_insertion->insert(entity, replace_if_exists, throw_if_exists);
if (id)
if (storage_for_insertion->insert(id, entity, replace_if_exists, throw_if_exists))
{
std::lock_guard lock{mutex};
ids_cache.set(*id, storage_for_insertion);
ids_cache.set(id, storage_for_insertion);
return true;
}
return id;
return false;
}

14
src/Access/MultipleAccessStorage.h
View File

@ -41,6 +41,16 @@ public:
ConstStoragePtr getStorage(const UUID & id) const;
StoragePtr getStorage(const UUID & id);
ConstStoragePtr findStorageByName(const String & storage_name) const;
StoragePtr findStorageByName(const String & storage_name);
ConstStoragePtr getStorageByName(const String & storage_name) const;
StoragePtr getStorageByName(const String & storage_name);
/// Search for an access entity storage, excluding one. Returns nullptr if not found.
StoragePtr findExcludingStorage(AccessEntityType type, const String & name, StoragePtr exclude) const;
void moveAccessEntities(const std::vector<UUID> & ids, const String & source_storage_name, const String & destination_storage_name);
bool exists(const UUID & id) const override;
bool isBackupAllowed() const override;
@ -53,7 +63,7 @@ protected:
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override;
@ -65,6 +75,8 @@ private:
std::shared_ptr<const Storages> nested_storages TSA_GUARDED_BY(mutex);
mutable CacheBase<UUID, Storage> ids_cache TSA_GUARDED_BY(mutex);
mutable std::mutex mutex;
mutable std::mutex move_mutex;
};
}

16
src/Access/ReplicatedAccessStorage.cpp
View File

@ -108,17 +108,7 @@ static void retryOnZooKeeperUserError(size_t attempts, Func && function)
}
}
std::optional<UUID> ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
const UUID id = generateRandomID();
if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
return id;
return std::nullopt;
}
bool ReplicatedAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
bool ReplicatedAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
const AccessEntityTypeInfo type_info = AccessEntityTypeInfo::get(new_entity->getType());
const String & name = new_entity->getName();
@ -619,7 +609,7 @@ AccessEntityPtr ReplicatedAccessStorage::tryReadEntityFromZooKeeper(const zkutil
void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntityPtr & entity)
{
LOG_DEBUG(getLogger(), "Setting id {} to entity named {}", toString(id), entity->getName());
memory_storage.insertWithID(id, entity, /* replace_if_exists= */ true, /* throw_if_exists= */ false);
memory_storage.insert(id, entity, /* replace_if_exists= */ true, /* throw_if_exists= */ false);
}
@ -711,7 +701,7 @@ void ReplicatedAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
{
for (const auto & [id, entity] : my_entities)
insertWithID(id, entity, replace_if_exists, throw_if_exists);
insert(id, entity, replace_if_exists, throw_if_exists);
});
}

3
src/Access/ReplicatedAccessStorage.h
View File

@ -46,11 +46,10 @@ private:
std::unique_ptr<ThreadFromGlobalPool> watching_thread;
std::shared_ptr<ConcurrentBoundedQueue<UUID>> watched_queue;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists);
bool insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists);
bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);

2
src/Access/UsersConfigAccessStorage.h
View File

@ -20,7 +20,7 @@ class UsersConfigAccessStorage : public IAccessStorage
{
public:
static constexpr char STORAGE_TYPE[] = "users.xml";
static constexpr char STORAGE_TYPE[] = "users_xml";
UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_, bool allow_backup_);
~UsersConfigAccessStorage() override;

1
src/AggregateFunctions/tests/gtest_ranks.cpp
View File

@ -2,7 +2,6 @@
#include <IO/ReadBufferFromString.h>
#include <Common/PODArray.h>
#include <AggregateFunctions/StatCommon.h>
#include <iostream>
#include <gtest/gtest.h>

3
src/Analyzer/Passes/FuseFunctionsPass.cpp
View File

@ -14,6 +14,9 @@
#include <Analyzer/FunctionNode.h>
#include <Analyzer/HashUtils.h>
#include <numeric>
namespace DB
{

1
src/Analyzer/examples/query_analyzer.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
int main(int argc, char ** argv)
{

3
src/CMakeLists.txt
View File

@ -19,6 +19,7 @@ message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION} ${VERSI
include (configure_config.cmake)
configure_file (Common/config.h.in ${CONFIG_INCLUDE_PATH}/config.h)
configure_file (Common/config_version.h.in ${CONFIG_INCLUDE_PATH}/config_version.h)
configure_file (Common/config_version.cpp.in ${CONFIG_INCLUDE_PATH}/config_version.cpp)
if (USE_DEBUG_HELPERS)
get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES)
@ -150,7 +151,7 @@ else()
message(STATUS "StorageFileLog is only supported on Linux")
endif ()
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
list (APPEND clickhouse_common_io_sources ${CONFIG_INCLUDE_PATH}/config_version.cpp)
list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/FunctionsLogical.cpp Functions/indexHint.cpp)
list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/FunctionsLogical.h Functions/indexHint.h)

2
src/Client/ClientBase.cpp
View File

@ -2505,7 +2505,7 @@ void ClientBase::clearTerminal()
void ClientBase::showClientVersion()
{
std::cout << DBMS_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
std::cout << VERSION_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
}
namespace

6
src/Client/Connection.cpp
View File

@ -280,9 +280,9 @@ void Connection::sendHello()
"Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters");
writeVarUInt(Protocol::Client::Hello, *out);
writeStringBinary((DBMS_NAME " ") + client_name, *out);
writeVarUInt(DBMS_VERSION_MAJOR, *out);
writeVarUInt(DBMS_VERSION_MINOR, *out);
writeStringBinary((VERSION_NAME " ") + client_name, *out);
writeVarUInt(VERSION_MAJOR, *out);
writeVarUInt(VERSION_MINOR, *out);
// NOTE For backward compatibility of the protocol, client cannot send its version_patch.
writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out);
writeStringBinary(default_database, *out);

1
src/Client/ConnectionParameters.cpp
View File

@ -1,6 +1,5 @@
#include "ConnectionParameters.h"
#include <fstream>
#include <iostream>
#include <Core/Defines.h>
#include <Core/Protocol.h>
#include <Core/Types.h>

1
src/Client/ConnectionString.cpp
View File

@ -6,7 +6,6 @@
#include <Poco/URI.h>
#include <array>
#include <iostream>
#include <string>
#include <unordered_map>

4
src/Columns/ColumnObject.cpp
View File

@ -2,17 +2,17 @@
#include <Columns/ColumnObject.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnSparse.h>
#include <DataTypes/ObjectUtils.h>
#include <DataTypes/getLeastSupertype.h>
#include <DataTypes/DataTypeNothing.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/NestedUtils.h>
#include <Interpreters/castColumn.h>
#include <Interpreters/convertFieldToType.h>
#include <Common/HashTable/HashSet.h>
#include <Processors/Transforms/ColumnGathererTransform.h>
#include <numeric>
namespace DB
{

1
src/Columns/tests/gtest_column_sparse.cpp
View File

@ -10,6 +10,7 @@
#include <Common/FieldVisitors.h>
using namespace DB;
static pcg64 rng(randomSeed());

1
src/Common/Config/configReadClient.cpp
View File

@ -3,7 +3,6 @@
#include <Poco/Util/LayeredConfiguration.h>
#include "ConfigProcessor.h"
#include <filesystem>
#include <iostream>
#include <base/types.h>
namespace fs = std::filesystem;

1
src/Common/CounterInFile.h
View File

@ -4,7 +4,6 @@
#include <sys/file.h>
#include <string>
#include <iostream>
#include <mutex>
#include <filesystem>

1
src/Common/DateLUTImpl.cpp
View File

@ -10,7 +10,6 @@
#include <chrono>
#include <cstring>
#include <memory>
#include <iostream>
/// Embedded timezones.

2
src/Common/EventNotifier.h
View File

@ -7,11 +7,11 @@
#include <map>
#include <memory>
#include <utility>
#include <iostream>
#include <base/types.h>
#include <Common/HashTable/Hash.h>
namespace DB
{

1
src/Common/FST.cpp
View File

@ -1,7 +1,6 @@
#include "FST.h"
#include <algorithm>
#include <cassert>
#include <iostream>
#include <memory>
#include <vector>
#include <Common/Exception.h>

42
src/Common/FileChecker.cpp
View File

@ -82,35 +82,33 @@ size_t FileChecker::getTotalSize() const
}
FileChecker::DataValidationTasksPtr FileChecker::getDataValidationTasks()
CheckResults FileChecker::check() const
{
return std::make_unique<DataValidationTasks>(map);
}
CheckResult FileChecker::checkNextEntry(DataValidationTasksPtr & check_data_tasks, bool & has_nothing_to_do) const
{
String name;
size_t expected_size;
bool is_finished = check_data_tasks->next(name, expected_size);
if (is_finished)
{
has_nothing_to_do = true;
if (map.empty())
return {};
}
String path = parentPath(files_info_path) + name;
bool exists = fileReallyExists(path);
auto real_size = exists ? getRealFileSize(path) : 0; /// No race condition assuming no one else is working with these files.
CheckResults results;
if (real_size != expected_size)
for (const auto & name_size : map)
{
String failure_message = exists
? ("Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(expected_size))
: ("File " + path + " doesn't exist");
return CheckResult(name, false, failure_message);
const String & name = name_size.first;
String path = parentPath(files_info_path) + name;
bool exists = fileReallyExists(path);
auto real_size = exists ? getRealFileSize(path) : 0; /// No race condition assuming no one else is working with these files.
if (real_size != name_size.second)
{
String failure_message = exists
? ("Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second))
: ("File " + path + " doesn't exist");
results.emplace_back(name, false, failure_message);
break;
}
results.emplace_back(name, true, "");
}
return CheckResult(name, true, "");
return results;
}
void FileChecker::repair()

37
src/Common/FileChecker.h
View File

@ -3,7 +3,6 @@
#include <Storages/CheckResults.h>
#include <map>
#include <base/types.h>
#include <mutex>
namespace Poco { class Logger; }
@ -29,11 +28,7 @@ public:
bool empty() const { return map.empty(); }
/// Check the files whose parameters are specified in sizes.json
/// See comment in IStorage::checkDataNext
struct DataValidationTasks;
using DataValidationTasksPtr = std::unique_ptr<DataValidationTasks>;
DataValidationTasksPtr getDataValidationTasks();
CheckResult checkNextEntry(DataValidationTasksPtr & check_data_tasks, bool & has_nothing_to_do) const;
CheckResults check() const;
/// Truncate files that have excessive size to the expected size.
/// Throw exception if the file size is less than expected.
@ -46,36 +41,6 @@ public:
/// Returns total size of all files.
size_t getTotalSize() const;
struct DataValidationTasks
{
DataValidationTasks(const std::map<String, size_t> & map_)
: map(map_), it(map.begin())
{}
bool next(String & out_name, size_t & out_size)
{
std::lock_guard lock(mutex);
if (it == map.end())
return true;
out_name = it->first;
out_size = it->second;
++it;
return false;
}
size_t size() const
{
std::lock_guard lock(mutex);
return std::distance(it, map.end());
}
const std::map<String, size_t> & map;
mutable std::mutex mutex;
using Iterator = std::map<String, size_t>::const_iterator;
Iterator it;
};
private:
void load();

5
src/Common/NamePrompter.h
View File

@ -106,6 +106,11 @@ public:
return prompter.getHints(name, getAllRegisteredNames());
}
std::vector<String> getHints(const String & name, const std::vector<String> & prompting_strings) const
{
return prompter.getHints(name, prompting_strings);
}
void appendHintsMessage(String & error_message, const String & name) const
{
auto hints = getHints(name);

2
src/Common/SpaceSaving.h
View File

@ -1,6 +1,5 @@
#pragma once
#include <iostream>
#include <vector>
#include <boost/range/adaptor/reversed.hpp>
@ -19,6 +18,7 @@
#include <IO/ReadHelpers.h>
#include <IO/VarInt.h>
/*
* Implementation of the Filtered Space-Saving for TopK streaming analysis.
* http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf

1
src/Common/StudentTTest.cpp
View File

@ -1,7 +1,6 @@
#include "StudentTTest.h"
#include <cmath>
#include <iostream>
#include <iomanip>
#include <sstream>
#include <stdexcept>

1
src/Common/ThreadPool.cpp
View File

@ -6,7 +6,6 @@
#include <Common/noexcept_scope.h>
#include <cassert>
#include <iostream>
#include <type_traits>
#include <Poco/Util/Application.h>

1
src/Common/UnicodeBar.cpp
View File

@ -7,7 +7,6 @@
#include <Common/UnicodeBar.h>
#include <Common/NaNUtils.h>
#include <iostream>
namespace DB
{

7
src/Common/VersionNumber.h
View File

@ -2,9 +2,9 @@
#include <string>
#include <vector>
#include <iostream>
#include <base/types.h>
namespace DB
{
@ -27,11 +27,6 @@ struct VersionNumber
std::string toString() const;
friend std::ostream & operator<<(std::ostream & os, const VersionNumber & v)
{
return os << v.toString();
}
private:
using Components = std::vector<Int64>;
Components components;

1
src/Common/benchmarks/integer_hash_tables_and_hashes.cpp
View File

@ -1,7 +1,6 @@
#include <benchmark/benchmark.h>
#include <iomanip>
#include <iostream>
#include <random>
#include <vector>

3
src/Common/config_version.cpp.in Normal file
View File

@ -0,0 +1,3 @@
/// This file was autogenerated by CMake
const char * VERSION_GITHASH = "@VERSION_GITHASH@";

22
src/Common/config_version.h.in
View File

@ -6,7 +6,6 @@
// only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
#cmakedefine VERSION_REVISION @VERSION_REVISION@
#cmakedefine VERSION_NAME "@VERSION_NAME@"
#define DBMS_NAME VERSION_NAME
#cmakedefine VERSION_MAJOR @VERSION_MAJOR@
#cmakedefine VERSION_MINOR @VERSION_MINOR@
#cmakedefine VERSION_PATCH @VERSION_PATCH@
@ -15,27 +14,10 @@
#cmakedefine VERSION_OFFICIAL "@VERSION_OFFICIAL@"
#cmakedefine VERSION_FULL "@VERSION_FULL@"
#cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@"
#cmakedefine VERSION_GITHASH "@VERSION_GITHASH@"
#cmakedefine VERSION_INTEGER @VERSION_INTEGER@
#cmakedefine VERSION_DATE @VERSION_DATE@
#if defined(VERSION_MAJOR)
#define DBMS_VERSION_MAJOR VERSION_MAJOR
#else
#define DBMS_VERSION_MAJOR 0
#endif
#if defined(VERSION_MINOR)
#define DBMS_VERSION_MINOR VERSION_MINOR
#else
#define DBMS_VERSION_MINOR 0
#endif
#if defined(VERSION_PATCH)
#define DBMS_VERSION_PATCH VERSION_PATCH
#else
#define DBMS_VERSION_PATCH 0
#endif
/// These fields are frequently changing and we don't want to have them in the header file to allow caching.
extern const char * VERSION_GITHASH;
#if !defined(VERSION_OFFICIAL)
# define VERSION_OFFICIAL ""

1
src/Common/examples/average.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
#include <string>
#include <bit>

1
src/Common/examples/chaos_sanitizer.cpp
View File

@ -6,7 +6,6 @@
#include <IO/ReadHelpers.h>
#include <Common/Exception.h>
#include <Common/ThreadFuzzer.h>

2
src/Common/examples/compact_array.cpp
View File

@ -4,11 +4,11 @@
#include <filesystem>
#include <string>
#include <iostream>
#include <fstream>
#include <stdexcept>
#include <cstdlib>
#include <unistd.h>
namespace fs = std::filesystem;
static std::string createTmpPath(const std::string & filename)

2
src/Common/tests/gtest_hash_table.cpp
View File

@ -1,5 +1,5 @@
#include <iomanip>
#include <iostream>
#include <numeric>
#include <Interpreters/AggregationCommon.h>

1
src/Common/tests/gtest_lru_cache.cpp
View File

@ -1,5 +1,4 @@
#include <iomanip>
#include <iostream>
#include <gtest/gtest.h>
#include <Common/CacheBase.h>

1
src/Common/tests/gtest_lru_hash_map.cpp
View File

@ -1,5 +1,4 @@
#include <iomanip>
#include <iostream>
#include <Common/HashTable/LRUHashMap.h>

1
src/Common/tests/gtest_lru_resource_cache.cpp
View File

@ -1,5 +1,4 @@
#include <iomanip>
#include <iostream>
#include <gtest/gtest.h>
#include <Common/LRUResourceCache.h>

1
src/Common/tests/gtest_slru_cache.cpp
View File

@ -1,5 +1,4 @@
#include <iomanip>
#include <iostream>
#include <gtest/gtest.h>
#include <Common/CacheBase.h>

1
src/Common/tests/gtest_thread_pool_limit.cpp
View File

@ -1,5 +1,4 @@
#include <atomic>
#include <iostream>
#include <Common/ThreadPool.h>
#include <Common/CurrentMetrics.h>

1
src/Common/tests/gtest_thread_pool_loop.cpp
View File

@ -1,5 +1,4 @@
#include <atomic>
#include <iostream>
#include <Common/ThreadPool.h>
#include <Common/CurrentMetrics.h>

1
src/Common/tests/gtest_thread_pool_schedule_exception.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
#include <stdexcept>
#include <Common/ThreadPool.h>
#include <Common/CurrentMetrics.h>

4
src/Compression/CompressionCodecDeflateQpl.cpp
View File

@ -1,15 +1,17 @@
#ifdef ENABLE_QPL_COMPRESSION
#include <cstdio>
#include <thread>
#include <Compression/CompressionCodecDeflateQpl.h>
#include <Compression/CompressionFactory.h>
#include <Compression/CompressionInfo.h>
#include <Parsers/ASTIdentifier.h>
#include <Poco/Logger.h>
#include <Common/logger_useful.h>
#include "libaccel_config.h"
#include <Common/MemorySanitizer.h>
#include <base/scope_guard.h>
#include <immintrin.h>
namespace DB
{

1
src/Compression/fuzzers/compressed_buffer_fuzzer.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
#include <IO/ReadBufferFromMemory.h>
#include <Compression/CompressedReadBuffer.h>
#include <Common/Exception.h>

1
src/Compression/fuzzers/delta_decompress_fuzzer.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
#include <string>
#include <Compression/ICompressionCodec.h>

1
src/Compression/fuzzers/double_delta_decompress_fuzzer.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
#include <string>
#include <Compression/ICompressionCodec.h>

1
src/Compression/fuzzers/encrypted_decompress_fuzzer.cpp
View File

@ -1,6 +1,5 @@
#include <cstddef>
#include <cstdint>
#include <iostream>
#include <memory>
#include <string>

1
src/Compression/fuzzers/lz4_decompress_fuzzer.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
#include <string>
#include <Compression/ICompressionCodec.h>

3
src/Coordination/FourLetterCommand.cpp
View File

@ -466,7 +466,7 @@ String EnviCommand::run()
StringBuffer buf;
buf << "Environment:\n";
buf << "clickhouse.keeper.version=" << (String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH) << '\n';
buf << "clickhouse.keeper.version=" << VERSION_DESCRIBE << '-' << VERSION_GITHASH << '\n';
buf << "host.name=" << Environment::nodeName() << '\n';
buf << "os.name=" << Environment::osDisplayName() << '\n';
@ -577,7 +577,6 @@ String FeatureFlagsCommand::run()
}
return ret.str();
}
}

4
src/Coordination/FourLetterCommand.h
View File

@ -9,8 +9,10 @@
#include "config_version.h"
namespace DB
{
struct IFourLetterCommand;
using FourLetterCommandPtr = std::shared_ptr<DB::IFourLetterCommand>;
@ -43,7 +45,7 @@ public:
using Commands = std::unordered_map<int32_t, FourLetterCommandPtr>;
using AllowList = std::vector<int32_t>;
///represent '*' which is used in allow list
/// Represents '*' which is used in allow list.
static constexpr int32_t ALLOW_LIST_ALL = 0;
bool isKnown(int32_t code);

2
src/Coordination/SnapshotableHashTable.h
View File

@ -6,7 +6,7 @@
#include <unordered_map>
#include <list>
#include <atomic>
#include <iostream>
namespace DB
{

1
src/Coordination/SummingStateMachine.cpp
View File

@ -1,5 +1,4 @@
#include <Coordination/SummingStateMachine.h>
#include <iostream>
#include <cstring>
namespace DB

Some files were not shown because too many files have changed in this diff Show More