thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-30 05:30:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into improvement/merge-tree-part-opt-parse

Browse Source
This commit is contained in:
Mike Kot 2021-08-31 14:35:14 +02:00
commit 4194ed2ec7
424 changed files with 8408 additions and 2015 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -12,7 +12,3 @@ ClickHouse® is an open-source column-oriented database management system that a
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
* [Code Browser](https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation.
* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
* You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
## Upcoming Events
* [SF Bay Area ClickHouse August Community Meetup (online)](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/279109379/) on 25 August 2021.

49
base/glibc-compatibility/musl/getauxval.c
View File

@ -1,4 +1,5 @@
#include <sys/auxv.h>
#include "atomic.h"
#include <unistd.h> // __environ
#include <errno.h>
@ -17,18 +18,7 @@ static size_t __find_auxv(unsigned long type)
return (size_t) -1;
}
__attribute__((constructor)) static void __auxv_init()
{
size_t i;
for (i = 0; __environ[i]; i++);
__auxv = (unsigned long *) (__environ + i + 1);
size_t secure_idx = __find_auxv(AT_SECURE);
if (secure_idx != ((size_t) -1))
__auxv_secure = __auxv[secure_idx];
}
unsigned long getauxval(unsigned long type)
unsigned long __getauxval(unsigned long type)
{
if (type == AT_SECURE)
return __auxv_secure;
@ -43,3 +33,38 @@ unsigned long getauxval(unsigned long type)
errno = ENOENT;
return 0;
}
static void * volatile getauxval_func;
static unsigned long __auxv_init(unsigned long type)
{
if (!__environ)
{
// __environ is not initialized yet so we can't initialize __auxv right now.
// That's normally occurred only when getauxval() is called from some sanitizer's internal code.
errno = ENOENT;
return 0;
}
// Initialize __auxv and __auxv_secure.
size_t i;
for (i = 0; __environ[i]; i++);
__auxv = (unsigned long *) (__environ + i + 1);
size_t secure_idx = __find_auxv(AT_SECURE);
if (secure_idx != ((size_t) -1))
__auxv_secure = __auxv[secure_idx];
// Now we've initialized __auxv, next time getauxval() will only call __get_auxval().
a_cas_p(&getauxval_func, (void *)__auxv_init, (void *)__getauxval);
return __getauxval(type);
}
// First time getauxval() will call __auxv_init().
static void * volatile getauxval_func = (void *)__auxv_init;
unsigned long getauxval(unsigned long type)
{
return ((unsigned long (*)(unsigned long))getauxval_func)(type);
}

3
contrib/libpqxx-cmake/CMakeLists.txt
View File

@ -22,6 +22,7 @@ set (SRCS
"${LIBRARY_DIR}/src/transaction.cxx"
"${LIBRARY_DIR}/src/transaction_base.cxx"
"${LIBRARY_DIR}/src/row.cxx"
"${LIBRARY_DIR}/src/params.cxx"
"${LIBRARY_DIR}/src/util.cxx"
"${LIBRARY_DIR}/src/version.cxx"
)
@ -31,6 +32,7 @@ set (SRCS
# conflicts with all includes of <array>.
set (HDRS
"${LIBRARY_DIR}/include/pqxx/array.hxx"
"${LIBRARY_DIR}/include/pqxx/params.hxx"
"${LIBRARY_DIR}/include/pqxx/binarystring.hxx"
"${LIBRARY_DIR}/include/pqxx/composite.hxx"
"${LIBRARY_DIR}/include/pqxx/connection.hxx"
@ -75,4 +77,3 @@ set(CM_CONFIG_PQ "${LIBRARY_DIR}/include/pqxx/config-internal-libpq.h")
configure_file("${CM_CONFIG_H_IN}" "${CM_CONFIG_INT}" @ONLY)
configure_file("${CM_CONFIG_H_IN}" "${CM_CONFIG_PUB}" @ONLY)
configure_file("${CM_CONFIG_H_IN}" "${CM_CONFIG_PQ}" @ONLY)

3
docker/test/fasttest/run.sh
View File

@ -396,6 +396,9 @@ function run_tests
# needs s3
01944_insert_partition_by
# depends on Go
02013_zlib_read_after_eof
)
time clickhouse-test --hung-check -j 8 --order=random --use-skip-list \

4
docker/test/fuzzer/Dockerfile
View File

@ -16,6 +16,8 @@ RUN apt-get update \
p7zip-full \
parallel \
psmisc \
python3 \
python3-pip \
rsync \
tree \
tzdata \
@ -25,6 +27,8 @@ RUN apt-get update \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN pip3 install Jinja2
COPY * /
SHELL ["/bin/bash", "-c"]

62
docker/test/fuzzer/generate-test-j2.py Executable file
View File

@ -0,0 +1,62 @@
#!/usr/bin/env python3
from argparse import ArgumentParser
import os
import jinja2
def removesuffix(text, suffix):
"""
Added in python 3.9
https://www.python.org/dev/peps/pep-0616/
"""
if suffix and text.endswith(suffix):
return text[:-len(suffix)]
else:
return text[:]
def render_test_template(j2env, suite_dir, test_name):
"""
Render template for test and reference file if needed
"""
test_base_name = removesuffix(test_name, ".sql.j2")
reference_file_name = test_base_name + ".reference.j2"
reference_file_path = os.path.join(suite_dir, reference_file_name)
if os.path.isfile(reference_file_path):
tpl = j2env.get_template(reference_file_name)
tpl.stream().dump(os.path.join(suite_dir, test_base_name) + ".gen.reference")
if test_name.endswith(".sql.j2"):
tpl = j2env.get_template(test_name)
generated_test_name = test_base_name + ".gen.sql"
tpl.stream().dump(os.path.join(suite_dir, generated_test_name))
return generated_test_name
return test_name
def main(args):
suite_dir = args.path
print(f"Scanning {suite_dir} directory...")
j2env = jinja2.Environment(
loader=jinja2.FileSystemLoader(suite_dir),
keep_trailing_newline=True,
)
test_names = os.listdir(suite_dir)
for test_name in test_names:
if not test_name.endswith(".sql.j2"):
continue
new_name = render_test_template(j2env, suite_dir, test_name)
print(f"File {new_name} generated")
if __name__ == "__main__":
parser = ArgumentParser(description="Jinja2 test generator")
parser.add_argument("-p", "--path", help="Path to test dir", required=True)
main(parser.parse_args())

15
docker/test/fuzzer/run-fuzzer.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
# shellcheck disable=SC2086
# shellcheck disable=SC2086,SC2001
set -eux
set -o pipefail
@ -71,12 +71,15 @@ function watchdog
kill -9 -- $fuzzer_pid ||:
}
function filter_exists
function filter_exists_and_template
{
local path
for path in "$@"; do
if [ -e "$path" ]; then
echo "$path"
# SC2001 shellcheck suggests:
# echo ${path//.sql.j2/.gen.sql}
# but it doesn't allow to use regex
echo "$path" | sed 's/\.sql\.j2$/.gen.sql/'
else
echo "'$path' does not exists" >&2
fi
@ -85,11 +88,13 @@ function filter_exists
function fuzz
{
/generate-test-j2.py --path ch/tests/queries/0_stateless
# Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests.
# Don't overwrite the NEW_TESTS_OPT so that it can be set from the environment.
NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\)$!ch/\1!p' ci-changed-files.txt | sort -R)"
NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\(\.j2\)\?\)$!ch/\1!p' ci-changed-files.txt | sort -R)"
# ci-changed-files.txt contains also files that has been deleted/renamed, filter them out.
NEW_TESTS="$(filter_exists $NEW_TESTS)"
NEW_TESTS="$(filter_exists_and_template $NEW_TESTS)"
if [[ -n "$NEW_TESTS" ]]
then
NEW_TESTS_OPT="${NEW_TESTS_OPT:---interleave-queries-file ${NEW_TESTS}}"

2
docker/test/integration/runner/Dockerfile
View File

@ -98,6 +98,8 @@ RUN set -x \
&& echo 'dockremap:165536:65536' >> /etc/subuid \
&& echo 'dockremap:165536:65536' >> /etc/subgid
RUN echo '127.0.0.1 localhost test.com' >> /etc/hosts
EXPOSE 2375
ENTRYPOINT ["dockerd-entrypoint.sh"]
CMD ["sh", "-c", "pytest $PYTEST_OPTS"]

11
docker/test/integration/runner/compose/docker_compose_nginx.yml Normal file
View File

@ -0,0 +1,11 @@
version: '2.3'
services:
# nginx server to host static files.
# Accepts only PUT data by test.com/path and GET already existing data on test.com/path.
# Files will be put into /usr/share/nginx/files.
nginx:
image: kssenii/nginx-test:1.1
restart: always
ports:
- 80:80

3
docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml
View File

@ -18,9 +18,6 @@
<!-- One NUMA node w/o hyperthreading -->
<max_threads>12</max_threads>
<!-- mmap shows some improvements in perf tests -->
<min_bytes_to_use_mmap_io>64Mi</min_bytes_to_use_mmap_io>
<!-- disable jit for perf tests -->
<compile_expressions>0</compile_expressions>
<compile_aggregate_expressions>0</compile_aggregate_expressions>

2
docker/test/stateless/Dockerfile
View File

@ -24,6 +24,8 @@ RUN apt-get update -y \
python3-pip \
qemu-user-static \
sudo \
# golang version 1.13 on Ubuntu 20 is enough for tests
golang \
telnet \
tree \
unixodbc \

2
docker/test/stress/run.sh
View File

@ -58,7 +58,7 @@ function start()
echo "Cannot start clickhouse-server"
cat /var/log/clickhouse-server/stdout.log
tail -n1000 /var/log/clickhouse-server/stderr.log
tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | grep -F -v '<Warning> RaftInstance:' -e '<Information> RaftInstance' | tail -n1000
tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | grep -F -v -e '<Warning> RaftInstance:' -e '<Information> RaftInstance' | tail -n1000
break
fi
# use root to match with current uid

19
docs/en/engines/table-engines/integrations/hdfs.md
View File

@ -184,9 +184,10 @@ Similar to GraphiteMergeTree, the HDFS engine supports extended configuration us
|hadoop\_kerberos\_keytab | "" |
|hadoop\_kerberos\_principal | "" |
|hadoop\_kerberos\_kinit\_command | kinit |
|libhdfs3\_conf | "" |
### Limitations {#limitations}
* hadoop\_security\_kerberos\_ticket\_cache\_path can be global only, not user specific
* hadoop\_security\_kerberos\_ticket\_cache\_path and libhdfs3\_conf can be global only, not user specific
## Kerberos support {#kerberos-support}
@ -198,6 +199,22 @@ security approach). Use tests/integration/test\_storage\_kerberized\_hdfs/hdfs_c
If hadoop\_kerberos\_keytab, hadoop\_kerberos\_principal or hadoop\_kerberos\_kinit\_command is specified, kinit will be invoked. hadoop\_kerberos\_keytab and hadoop\_kerberos\_principal are mandatory in this case. kinit tool and krb5 configuration files are required.
## HDFS Namenode HA support{#namenode-ha}
libhdfs3 support HDFS namenode HA.
- Copy `hdfs-site.xml` from an HDFS node to `/etc/clickhouse-server/`.
- Add following piece to ClickHouse config file:
``` xml
<hdfs>
<libhdfs3_conf>/etc/clickhouse-server/hdfs-site.xml</libhdfs3_conf>
</hdfs>
```
- Then use `dfs.nameservices` tag value of `hdfs-site.xml` as the namenode address in the HDFS URI. For example, replace `hdfs://appadmin@192.168.101.11:8020/abc/` with `hdfs://appadmin@my_nameservice/abc/`.
## Virtual Columns {#virtual-columns}
- `_path` — Path to the file.

1
docs/en/engines/table-engines/integrations/postgresql.md
View File

@ -34,6 +34,7 @@ The table structure can differ from the original PostgreSQL table structure:
- `user` — PostgreSQL user.
- `password` — User password.
- `schema` — Non-default table schema. Optional.
- `on conflict ...` — example: `ON CONFLICT DO NOTHING`. Optional. Note: adding this option will make insertion less efficient.
## Implementation Details {#implementation-details}

25
docs/en/engines/table-engines/mergetree-family/mergetree.md
View File

@ -390,20 +390,27 @@ Functions with a constant argument that is less than ngram size can’t be used
- `s != 1`
- `NOT startsWith(s, 'test')`
### Projections {#projections}
Projections are like materialized views but defined in part-level. It provides consistency guarantees along with automatic usage in queries.
## Projections {#projections}
Projections are like [materialized views](../../../sql-reference/statements/create/view.md#materialized) but defined in part-level. It provides consistency guarantees along with automatic usage in queries.
#### Query {#projection-query}
A projection query is what defines a projection. It has the following grammar:
Projections are an experimental feature. To enable them you must set the [allow_experimental_projection_optimization](../../../operations/settings/settings.md#allow-experimental-projection-optimization) to `1`. See also the [force_optimize_projection](../../../operations/settings/settings.md#force-optimize-projection) setting.
`SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]`
Projections are not supported in the `SELECT` statements with the [FINAL](../../../sql-reference/statements/select/from.md#select-from-final) modifier.
It implicitly selects data from the parent table.
### Projection Query {#projection-query}
A projection query is what defines a projection. It implicitly selects data from the parent table.
**Syntax**
#### Storage {#projection-storage}
Projections are stored inside the part directory. It's similar to an index but contains a subdirectory that stores an anonymous MergeTree table's part. The table is induced by the definition query of the projection. If there is a GROUP BY clause, the underlying storage engine becomes AggregatedMergeTree, and all aggregate functions are converted to AggregateFunction. If there is an ORDER BY clause, the MergeTree table will use it as its primary key expression. During the merge process, the projection part will be merged via its storage's merge routine. The checksum of the parent table's part will combine the projection's part. Other maintenance jobs are similar to skip indices.
```sql
SELECT <column list expr> [GROUP BY] <group keys expr> [ORDER BY] <expr>
```
#### Query Analysis {#projection-query-analysis}
Projections can be modified or dropped with the [ALTER](../../../sql-reference/statements/alter/projection.md) statement.
### Projection Storage {#projection-storage}
Projections are stored inside the part directory. It's similar to an index but contains a subdirectory that stores an anonymous `MergeTree` table's part. The table is induced by the definition query of the projection. If there is a `GROUP BY` clause, the underlying storage engine becomes [AggregatingMergeTree](aggregatingmergetree.md), and all aggregate functions are converted to `AggregateFunction`. If there is an `ORDER BY` clause, the `MergeTree` table uses it as its primary key expression. During the merge process the projection part is merged via its storage's merge routine. The checksum of the parent table's part is combined with the projection's part. Other maintenance jobs are similar to skip indices.
### Query Analysis {#projection-query-analysis}
1. Check if the projection can be used to answer the given query, that is, it generates the same answer as querying the base table.
2. Select the best feasible match, which contains the least granules to read.
3. The query pipeline which uses projections will be different from the one that uses the original parts. If the projection is absent in some parts, we can add the pipeline to "project" it on the fly.

28
docs/en/interfaces/cli.md
View File

@ -149,4 +149,30 @@ Example of a config file:
</config>
```
[Original article](https://clickhouse.tech/docs/en/interfaces/cli/) <!--hide-->
### Query ID Format {#query-id-format}
In interactive mode `clickhouse-client` shows query ID for every query. By default, the ID is formatted like this:
```sql
Query id: 927f137d-00f1-4175-8914-0dd066365e96
```
A custom format may be specified in a configuration file inside a `query_id_formats` tag. `{query_id}` placeholder in the format string is replaced with the ID of a query. Several format strings are allowed inside the tag.
This feature can be used to generate URLs to facilitate profiling of queries.
**Example**
```xml
<config>
<query_id_formats>
<speedscope>http://speedscope-host/#profileURL=qp%3Fid%3D{query_id}</speedscope>
</query_id_formats>
</config>
```
If the configuration above is applied, the ID of a query is shown in the following format:
``` text
speedscope:http://speedscope-host/#profileURL=qp%3Fid%3Dc8ecc783-e753-4b38-97f1-42cddfb98b7d
```

6
docs/en/interfaces/mysql.md
View File

@ -44,4 +44,10 @@ Restrictions:
- some data types are sent as strings
To cancel a long query use `KILL QUERY connection_id` statement (it is replaced with `KILL QUERY WHERE query_id = connection_id` while proceeding). For example:
``` bash
$ mysql --protocol tcp -h mysql_server -P 9004 default -u default --password=123 -e "KILL QUERY 123456;"
```
[Original article](https://clickhouse.tech/docs/en/interfaces/mysql/) <!--hide-->

16
docs/en/interfaces/third-party/gui.md vendored
View File

@ -190,4 +190,20 @@ SeekTable is [free](https://www.seektable.com/help/cloud-pricing) for personal/i
[Chadmin](https://github.com/bun4uk/chadmin) is a simple UI where you can visualize your currently running queries on your ClickHouse cluster and info about them and kill them if you want.
### DBM {#dbm}
[DBM](https://dbm.incubator.edurt.io/) DBM is a visual management tool for ClickHouse!
Features:
- Support query history (pagination, clear all, etc.)
- Support selected sql clauses query
- Support terminating query
- Support table management (metadata, delete, preview)
- Support database management (delete, create)
- Support custom query
- Support multiple data sources management(connection test, monitoring)
- Support monitor (processor, connection, query)
- Support migrate data
[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/gui/) <!--hide-->

22
docs/en/operations/settings/settings.md
View File

@ -3435,3 +3435,25 @@ Possible values:
- 1 — The table is automatically updated in the background, when schema changes are detected.
Default value: `0`.
## allow_experimental_projection_optimization {#allow-experimental-projection-optimization}
Enables or disables [projection](../../engines/table-engines/mergetree-family/mergetree.md#projections) optimization when processing `SELECT` queries.
Possible values:
- 0 — Projection optimization disabled.
- 1 — Projection optimization enabled.
Default value: `0`.
## force_optimize_projection {#force-optimize-projection}
Enables or disables the obligatory use of [projections](../../engines/table-engines/mergetree-family/mergetree.md#projections) in `SELECT` queries, when projection optimization is enabled (see [allow_experimental_projection_optimization](#allow-experimental-projection-optimization) setting).
Possible values:
- 0 — Projection optimization is not obligatory.
- 1 — Projection optimization is obligatory.
Default value: `0`.

2
docs/en/sql-reference/aggregate-functions/parametric-functions.md
View File

@ -154,7 +154,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM
└──────────────────────────────────────────────────────────────────────────────────────────┘
```
In this case, the function couldn’t find the event chain matching the pattern, because the event for number 3 occured between 1 and 2. If in the same case we checked the condition for number 4, the sequence would match the pattern.
In this case, the function couldn’t find the event chain matching the pattern, because the event for number 3 occurred between 1 and 2. If in the same case we checked the condition for number 4, the sequence would match the pattern.
``` sql
SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM t

23
docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
View File

@ -58,6 +58,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings
- [direct](#direct)
- [range_hashed](#range-hashed)
- [complex_key_hashed](#complex-key-hashed)
- [complex_key_range_hashed](#complex-key-range-hashed)
- [complex_key_cache](#complex-key-cache)
- [ssd_cache](#ssd-cache)
- [ssd_complex_key_cache](#complex-key-ssd-cache)
@ -269,6 +270,28 @@ PRIMARY KEY Abcdef
RANGE(MIN StartTimeStamp MAX EndTimeStamp)
```
### complex_key_range_hashed {#complex-key-range-hashed}
The dictionary is stored in memory in the form of a hash table with an ordered array of ranges and their corresponding values (see [range_hashed](#range-hashed)). This type of storage is for use with composite [keys](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md).
Configuration example:
``` sql
CREATE DICTIONARY range_dictionary
(
CountryID UInt64,
CountryKey String,
StartDate Date,
EndDate Date,
Tax Float64 DEFAULT 0.2
)
PRIMARY KEY CountryID, CountryKey
SOURCE(CLICKHOUSE(TABLE 'date_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(COMPLEX_KEY_RANGE_HASHED())
RANGE(MIN StartDate MAX EndDate);
```
### cache {#cache}
The dictionary is stored in a cache that has a fixed number of cells. These cells contain frequently used elements.

118
docs/en/sql-reference/functions/other-functions.md
View File

@ -2236,3 +2236,121 @@ defaultRoles()
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
## getServerPort {#getserverport}
Returns the number of the server port. When the port is not used by the server, throws an exception.
**Syntax**
``` sql
getServerPort(port_name)
```
**Arguments**
- `port_name` — The name of the server port. [String](../../sql-reference/data-types/string.md#string). Possible values:
- 'tcp_port'
- 'tcp_port_secure'
- 'http_port'
- 'https_port'
- 'interserver_http_port'
- 'interserver_https_port'
- 'mysql_port'
- 'postgresql_port'
- 'grpc_port'
- 'prometheus.port'
**Returned value**
- The number of the server port.
Type: [UInt16](../../sql-reference/data-types/int-uint.md).
**Example**
Query:
``` sql
SELECT getServerPort('tcp_port');
```
Result:
``` text
┌─getServerPort('tcp_port')─┐
│ 9000 │
└───────────────────────────┘
```
## queryID {#query-id}
Returns the ID of the current query. Other parameters of a query can be extracted from the [system.query_log](../../operations/system-tables/query_log.md) table via `query_id`.
In contrast to [initialQueryID](#initial-query-id) function, `queryID` can return different results on different shards (see example).
**Syntax**
``` sql
queryID()
```
**Returned value**
- The ID of the current query.
Type: [String](../../sql-reference/data-types/string.md)
**Example**
Query:
``` sql
CREATE TABLE tmp (str String) ENGINE = Log;
INSERT INTO tmp (*) VALUES ('a');
SELECT count(DISTINCT t) FROM (SELECT queryID() AS t FROM remote('127.0.0.{1..3}', currentDatabase(), 'tmp') GROUP BY queryID());
```
Result:
``` text
┌─count()─┐
│ 3 │
└─────────┘
```
## initialQueryID {#initial-query-id}
Returns the ID of the initial current query. Other parameters of a query can be extracted from the [system.query_log](../../operations/system-tables/query_log.md) table via `initial_query_id`.
In contrast to [queryID](#query-id) function, `initialQueryID` returns the same results on different shards (see example).
**Syntax**
``` sql
initialQueryID()
```
**Returned value**
- The ID of the initial current query.
Type: [String](../../sql-reference/data-types/string.md)
**Example**
Query:
``` sql
CREATE TABLE tmp (str String) ENGINE = Log;
INSERT INTO tmp (*) VALUES ('a');
SELECT count(DISTINCT t) FROM (SELECT initialQueryID() AS t FROM remote('127.0.0.{1..3}', currentDatabase(), 'tmp') GROUP BY queryID());
```
Result:
``` text
┌─count()─┐
│ 1 │
└─────────┘
```

17
docs/en/sql-reference/functions/uuid-functions.md
View File

@ -245,7 +245,22 @@ SELECT
└──────────────────┴──────────────────────────────────────┘
```
## serverUUID() {#server-uuid}
Returns the random and unique UUID, which is generated when the server is first started and stored forever. The result writes to the file `uuid` created in the ClickHouse server directory `/var/lib/clickhouse/`.
**Syntax**
```sql
serverUUID()
```
**Returned value**
- The UUID of the server.
Type: [UUID](../data-types/uuid.md).
## See Also {#see-also}
- [dictGetUUID](../../sql-reference/functions/ext-dict-functions.md#ext_dict_functions-other)

4
docs/en/sql-reference/statements/alter/projection.md
View File

@ -5,7 +5,7 @@ toc_title: PROJECTION
# Manipulating Projections {#manipulations-with-projections}
The following operations are available:
The following operations with [projections](../../../engines/table-engines/mergetree-family/mergetree.md#projections) are available:
- `ALTER TABLE [db].name ADD PROJECTION name AS SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]` - Adds projection description to tables metadata.
@ -15,7 +15,7 @@ The following operations are available:
- `ALTER TABLE [db.]table CLEAR PROJECTION name IN PARTITION partition_name` - Deletes projection files from disk without removing description.
The commands ADD, DROP and CLEAR are lightweight in a sense that they only change metadata or remove files.
The commands `ADD`, `DROP` and `CLEAR` are lightweight in a sense that they only change metadata or remove files.
Also, they are replicated, syncing projections metadata via ZooKeeper.

3
docs/en/sql-reference/statements/create/view.md
View File

@ -49,6 +49,9 @@ When creating a materialized view with `TO [db].[table]`, you must not use `POPU
A materialized view is implemented as follows: when inserting data to the table specified in `SELECT`, part of the inserted data is converted by this `SELECT` query, and the result is inserted in the view.
!!! important "Important"
Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in `SELECT`'s result ClickHouse will use a default value, even if column is not `Nullable`. A safe practice would be to add aliases for every column when using Materialized views.
!!! important "Important"
Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view.

3
docs/en/sql-reference/statements/select/join.md
View File

@ -36,6 +36,9 @@ Additional join types available in ClickHouse:
- `LEFT ANY JOIN`, `RIGHT ANY JOIN` and `INNER ANY JOIN`, partially (for opposite side of `LEFT` and `RIGHT`) or completely (for `INNER` and `FULL`) disables the cartesian product for standard `JOIN` types.
- `ASOF JOIN` and `LEFT ASOF JOIN`, joining sequences with a non-exact match. `ASOF JOIN` usage is described below.
!!! note "Note"
When [join_algorithm](../../../operations/settings/settings.md#settings-join_algorithm) is set to `partial_merge`, `RIGHT JOIN` and `FULL JOIN` are supported only with `ALL` strictness (`SEMI`, `ANTI`, `ANY`, and `ASOF` are not supported).
## Settings {#join-settings}
The default join type can be overridden using [join_default_strictness](../../../operations/settings/settings.md#settings-join_default_strictness) setting.

2
docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md
View File

@ -171,4 +171,4 @@ default
!!! warning "Внимание"
Прореживание данных производится во время слияний. Обычно для старых партций слияния не запускаются, поэтому для прореживания надо иницировать незапланированное слияние используя [optimize](../../../sql-reference/statements/optimize.md). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer).
Прореживание данных производится во время слияний. Обычно для старых партиций слияния не запускаются, поэтому для прореживания надо инициировать незапланированное слияние используя [optimize](../../../sql-reference/statements/optimize.md). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer).

34
docs/ru/engines/table-engines/mergetree-family/mergetree.md
View File

@ -377,23 +377,33 @@ INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARIT
- `s != 1`
- `NOT startsWith(s, 'test')`
### Проекции {#projections}
Проекции похожи на материализованные представления, но определяются на уровне партов. Это обеспечивает гарантии согласованности наряду с автоматическим использованием в запросах.
## Проекции {#projections}
Проекции похожи на [материализованные представления](../../../sql-reference/statements/create/view.md#materialized), но определяются на уровне кусков данных. Это обеспечивает гарантии согласованности данных наряду с автоматическим использованием в запросах.
#### Запрос {#projection-query}
Запрос проекции — это то, что определяет проекцию. Он имеет следующую грамматику:
Проекции — это экспериментальная возможность. Чтобы включить поддержку проекций, установите настройку [allow_experimental_projection_optimization](../../../operations/settings/settings.md#allow-experimental-projection-optimization) в значение `1`. См. также настройку [force_optimize_projection ](../../../operations/settings/settings.md#force-optimize-projection).
`SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]`
Проекции не поддерживаются для запросов `SELECT` с модификатором [FINAL](../../../sql-reference/statements/select/from.md#select-from-final).
Он неявно выбирает данные из родительской таблицы.
### Запрос проекции {#projection-query}
Запрос проекции — это то, что определяет проекцию. Такой запрос неявно выбирает данные из родительской таблицы.
**Синтаксис**
#### Хранение {#projection-storage}
Проекции хранятся в каталоге парта. Это похоже на хранение индексов, но используется подкаталог, в котором хранится анонимный парт таблицы MergeTree. Таблица создается запросом определения проекции. Если есть конструкция GROUP BY, то базовый механизм хранения становится AggregatedMergeTree, а все агрегатные функции преобразуются в AggregateFunction. Если есть конструкция ORDER BY, таблица MergeTree будет использовать его в качестве выражения первичного ключа. Во время процесса слияния парт проекции будет слит с помощью процедуры слияния ее хранилища. Контрольная сумма парта родительской таблицы будет включать парт проекции. Другие процедуры аналогичны индексам пропуска данных.
```sql
SELECT <column list expr> [GROUP BY] <group keys expr> [ORDER BY] <expr>
```
#### Анализ запросов {#projection-query-analysis}
1. Проверить, можно ли использовать проекцию в данном запросе, то есть, что с ней выходит тот же результат, что и с запросом к базовой таблице.
2. Выбрать наиболее подходящее совпадение, содержащее наименьшее количество гранул для чтения.
3. План запроса, который использует проекции, будет отличаться от того, который использует исходные парты. При отсутствии проекции в некоторых партах можно расширить план, чтобы «проецировать» на лету.
Проекции можно изменить или удалить с помощью запроса [ALTER](../../../sql-reference/statements/alter/projection.md).
### Хранение проекции {#projection-storage}
Проекции хранятся в каталоге куска данных. Это похоже на хранение индексов, но используется подкаталог, в котором хранится анонимный кусок таблицы `MergeTree`. Таблица создается запросом определения проекции.
Если присутствует секция `GROUP BY`, то используется движок [AggregatingMergeTree](aggregatingmergetree.md), а все агрегатные функции преобразуются в `AggregateFunction`.
Если присутствует секция `ORDER BY`, таблица `MergeTree` использует ее в качестве выражения для первичного ключа.
Во время процесса слияния кусок данных проекции объединяется с помощью процедуры слияния хранилища. Контрольная сумма куска данных родительской таблицы включает кусок данных проекции. Другие процедуры аналогичны индексам пропуска данных.
### Анализ запросов {#projection-query-analysis}
1. Проверьте, можно ли использовать проекцию в данном запросе, то есть, что с ней получается тот же результат, что и с запросом к базовой таблице.
2. Выберите наиболее подходящее совпадение, содержащее наименьшее количество гранул для чтения.
3. План запроса, который использует проекции, отличается от того, который использует исходные куски данных. Если в некоторых кусках проекции отсутствуют, можно расширить план, чтобы «проецировать» на лету.
## Конкурентный доступ к данным {#concurrent-data-access}

54
docs/ru/interfaces/cli.md
View File

@ -26,7 +26,7 @@ Connected to ClickHouse server version 20.13.1 revision 54442.
Клиент может быть использован в интерактивном и не интерактивном (batch) режиме.
Чтобы использовать batch режим, укажите параметр query, или отправьте данные в stdin (проверяется, что stdin - не терминал), или и то, и другое.
Аналогично HTTP интерфейсу, при использовании одновременно параметра query и отправке данных в stdin, запрос составляется из конкатенации параметра query, перевода строки, и данных в stdin. Это удобно для больших INSERT запросов.
Аналогично HTTP интерфейсу, при использовании одновременно параметра query и отправке данных в stdin, запрос составляется из конкатенации параметра query, перевода строки и данных в stdin. Это удобно для больших `INSERT` запросов.
Примеры использования клиента для вставки данных:
@ -41,17 +41,17 @@ _EOF
$ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
```
В batch режиме в качестве формата данных по умолчанию используется формат TabSeparated. Формат может быть указан в секции FORMAT запроса.
В batch режиме в качестве формата данных по умолчанию используется формат `TabSeparated`. Формат может быть указан в запросе в секции `FORMAT`.
По умолчанию, в batch режиме вы можете выполнить только один запрос. Чтобы выполнить несколько запросов из «скрипта», используйте параметр –multiquery. Это работает для всех запросов кроме INSERT. Результаты запросов выводятся подряд без дополнительных разделителей.
Также, при необходимости выполнить много запросов, вы можете запускать clickhouse-client на каждый запрос. Заметим, что запуск программы clickhouse-client может занимать десятки миллисекунд.
По умолчанию в batch режиме вы можете выполнить только один запрос. Чтобы выполнить несколько запросов из «скрипта», используйте параметр `–-multiquery`. Это работает для всех запросов кроме `INSERT`. Результаты запросов выводятся подряд без дополнительных разделителей.
Если нужно выполнить много запросов, вы можете запускать clickhouse-client отдельно на каждый запрос. Заметим, что запуск программы clickhouse-client может занимать десятки миллисекунд.
В интерактивном режиме, вы получите командную строку, в которую можно вводить запросы.
В интерактивном режиме вы получаете командную строку, в которую можно вводить запросы.
Если не указано multiline (по умолчанию):
Чтобы выполнить запрос, нажмите Enter. Точка с запятой на конце запроса не обязательна. Чтобы ввести запрос, состоящий из нескольких строк, перед переводом строки, введите символ обратного слеша: `\` - тогда после нажатия Enter, вам предложат ввести следующую строку запроса.
Чтобы выполнить запрос, нажмите Enter. Точка с запятой на конце запроса необязательна. Чтобы ввести запрос, состоящий из нескольких строк, в конце строки поставьте символ обратного слеша `\`, тогда после нажатия Enter вы сможете ввести следующую строку запроса.
Если указано multiline (многострочный режим):
Если указан параметр `--multiline` (многострочный режим):
Чтобы выполнить запрос, завершите его точкой с запятой и нажмите Enter. Если в конце введённой строки не было точки с запятой, то вам предложат ввести следующую строчку запроса.
Исполняется только один запрос, поэтому всё, что введено после точки с запятой, игнорируется.
@ -61,20 +61,20 @@ $ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FOR
Командная строка сделана на основе readline (и history) (или libedit, или без какой-либо библиотеки, в зависимости от сборки) - то есть, в ней работают привычные сочетания клавиш, а также присутствует история.
История пишется в `~/.clickhouse-client-history`.
По умолчанию, в качестве формата, используется формат PrettyCompact (красивые таблички). Вы можете изменить формат с помощью секции FORMAT запроса, или с помощью указания `\G` на конце запроса, с помощью аргумента командной строки `--format` или `--vertical`, или с помощью конфигурационного файла клиента.
По умолчанию используется формат вывода `PrettyCompact` (он поддерживает красивый вывод таблиц). Вы можете изменить формат вывода результатов запроса следующими способами: с помощью секции `FORMAT` в запросе, указав символ `\G` в конце запроса, используя аргументы командной строки `--format` или `--vertical` или с помощью конфигурационного файла клиента.
Чтобы выйти из клиента, нажмите Ctrl+D, или наберите вместо запроса одно из: «exit», «quit», «logout», «учше», «йгше», «дщпщге», «exit;», «quit;», «logout;», «учшеж», «йгшеж», «дщпщгеж», «q», «й», «q», «Q», «:q», «й», «Й», «Жй»
Чтобы выйти из клиента, нажмите Ctrl+D или наберите вместо запроса одно из: «exit», «quit», «logout», «учше», «йгше», «дщпщге», «exit;», «quit;», «logout;», «учшеж», «йгшеж», «дщпщгеж», «q», «й», «q», «Q», «:q», «й», «Й», «Жй».
При выполнении запроса, клиент показывает:
При выполнении запроса клиент показывает:
1. Прогресс выполнение запроса, который обновляется не чаще, чем 10 раз в секунду (по умолчанию). При быстрых запросах, прогресс может не успеть отобразиться.
1. Прогресс выполнение запроса, который обновляется не чаще, чем 10 раз в секунду (по умолчанию). При быстрых запросах прогресс может не успеть отобразиться.
2. Отформатированный запрос после его парсинга - для отладки.
3. Результат в заданном формате.
4. Количество строк результата, прошедшее время, а также среднюю скорость выполнения запроса.
Вы можете прервать длинный запрос, нажав Ctrl+C. При этом вам всё равно придётся чуть-чуть подождать, пока сервер остановит запрос. На некоторых стадиях выполнения, запрос невозможно прервать. Если вы не дождётесь и нажмёте Ctrl+C второй раз, то клиент будет завершён.
Вы можете прервать длинный запрос, нажав Ctrl+C. При этом вам всё равно придётся чуть-чуть подождать, пока сервер остановит запрос. На некоторых стадиях выполнения запрос невозможно прервать. Если вы не дождётесь и нажмёте Ctrl+C второй раз, то клиент будет завершён.
Клиент командной строки позволяет передать внешние данные (внешние временные таблицы) для использования запроса. Подробнее смотрите раздел «Внешние данные для обработки запроса»
Клиент командной строки позволяет передать внешние данные (внешние временные таблицы) для выполнения запроса. Подробнее смотрите раздел «Внешние данные для обработки запроса».
### Запросы с параметрами {#cli-queries-with-parameters}
@ -84,7 +84,7 @@ $ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FOR
clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}"
```
#### Cинтаксис запроса {#cli-queries-with-parameters-syntax}
#### Синтаксис запроса {#cli-queries-with-parameters-syntax}
Отформатируйте запрос обычным способом. Представьте значения, которые вы хотите передать из параметров приложения в запрос в следующем формате:
@ -155,3 +155,29 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe
</config>
```
### Формат ID запроса {#query-id-format}
В интерактивном режиме `clickhouse-client` показывает ID для каждого запроса. По умолчанию ID выводится в таком виде:
```sql
Query id: 927f137d-00f1-4175-8914-0dd066365e96
```
Произвольный формат ID можно задать в конфигурационном файле внутри тега `query_id_formats`. ID подставляется вместо `{query_id}` в строке формата. В теге может быть перечислено несколько строк формата.
Эта возможность может быть полезна для генерации URL, с помощью которых выполняется профилирование запросов.
**Пример**
```xml
<config>
<query_id_formats>
<speedscope>http://speedscope-host/#profileURL=qp%3Fid%3D{query_id}</speedscope>
</query_id_formats>
</config>
```
Если применить приведённую выше конфигурацию, то ID запроса будет выводиться в следующем виде:
``` text
speedscope:http://speedscope-host/#profileURL=qp%3Fid%3Dc8ecc783-e753-4b38-97f1-42cddfb98b7d
```

6
docs/ru/interfaces/mysql.md
View File

@ -43,3 +43,9 @@ mysql>
- не поддерживаются подготовленные запросы
- некоторые типы данных отправляются как строки
Чтобы прервать долго выполняемый запрос, используйте запрос `KILL QUERY connection_id` (во время выполнения он будет заменен на `KILL QUERY WHERE query_id = connection_id`). Например:
``` bash
$ mysql --protocol tcp -h mysql_server -P 9004 default -u default --password=123 -e "KILL QUERY 123456;"
```

22
docs/ru/operations/settings/settings.md
View File

@ -3252,3 +3252,25 @@ SETTINGS index_granularity = 8192 │
- 1 — таблица обновляется автоматически в фоновом режиме при обнаружении изменений схемы.
Значение по умолчанию: `0`.
## allow_experimental_projection_optimization {#allow-experimental-projection-optimization}
Включает или отключает поддержку [проекций](../../engines/table-engines/mergetree-family/mergetree.md#projections) при обработке запросов `SELECT`.
Возможные значения:
- 0 — Проекции не поддерживаются.
- 1 — Проекции поддерживаются.
Значение по умолчанию: `0`.
## force_optimize_projection {#force-optimize-projection}
Включает или отключает обязательное использование [проекций](../../engines/table-engines/mergetree-family/mergetree.md#projections) в запросах `SELECT`, если поддержка проекций включена (см. настройку [allow_experimental_projection_optimization](#allow-experimental-projection-optimization)).
Возможные значения:
- 0 — Проекции используются опционально.
- 1 — Проекции обязательно используются.
Значение по умолчанию: `0`.

23
docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
View File

@ -59,6 +59,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings
- [direct](#direct)
- [range_hashed](#range-hashed)
- [complex_key_hashed](#complex-key-hashed)
- [complex_key_range_hashed](#complex-key-range-hashed)
- [complex_key_cache](#complex-key-cache)
- [complex_key_direct](#complex-key-direct)
- [ip_trie](#ip-trie)
@ -268,6 +269,28 @@ PRIMARY KEY Abcdef
RANGE(MIN StartTimeStamp MAX EndTimeStamp)
```
### complex_key_range_hashed {#complex-key-range-hashed}
Словарь хранится в оперативной памяти в виде хэш-таблицы с упорядоченным массивом диапазонов и соответствующих им значений (см. [range_hashed](#range-hashed)). Данный тип размещения предназначен для использования с составными [ключами](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md).
Пример конфигурации:
``` sql
CREATE DICTIONARY range_dictionary
(
CountryID UInt64,
CountryKey String,
StartDate Date,
EndDate Date,
Tax Float64 DEFAULT 0.2
)
PRIMARY KEY CountryID, CountryKey
SOURCE(CLICKHOUSE(TABLE 'date_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(COMPLEX_KEY_RANGE_HASHED())
RANGE(MIN StartDate MAX EndDate);
```
### cache {#cache}
Словарь хранится в кэше, состоящем из фиксированного количества ячеек. Ячейки содержат часто используемые элементы.

119
docs/ru/sql-reference/functions/other-functions.md
View File

@ -2185,3 +2185,122 @@ defaultRoles()
- Список ролей по умолчанию.
Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
## getServerPort {#getserverport}
Возвращает номер порта сервера. Если порт не используется сервером, генерируется исключение.
**Синтаксис**
``` sql
getServerPort(port_name)
```
**Аргументы**
- `port_name` — имя порта сервера. [String](../../sql-reference/data-types/string.md#string). Возможные значения:
- 'tcp_port'
- 'tcp_port_secure'
- 'http_port'
- 'https_port'
- 'interserver_http_port'
- 'interserver_https_port'
- 'mysql_port'
- 'postgresql_port'
- 'grpc_port'
- 'prometheus.port'
**Возвращаемое значение**
- Номер порта сервера.
Тип: [UInt16](../../sql-reference/data-types/int-uint.md).
**Пример**
Запрос:
``` sql
SELECT getServerPort('tcp_port');
```
Результат:
``` text
┌─getServerPort('tcp_port')─┐
│ 9000 │
└───────────────────────────┘
```
## queryID {#query-id}
Возвращает идентификатор текущего запроса. Другие параметры запроса могут быть извлечены из системной таблицы [system.query_log](../../operations/system-tables/query_log.md) через `query_id`.
В отличие от [initialQueryID](#initial-query-id), функция `queryID` может возвращать различные значения для разных шардов (см. пример).
**Синтаксис**
``` sql
queryID()
```
**Возвращаемое значение**
- Идентификатор текущего запроса.
Тип: [String](../../sql-reference/data-types/string.md)
**Пример**
Запрос:
``` sql
CREATE TABLE tmp (str String) ENGINE = Log;
INSERT INTO tmp (*) VALUES ('a');
SELECT count(DISTINCT t) FROM (SELECT queryID() AS t FROM remote('127.0.0.{1..3}', currentDatabase(), 'tmp') GROUP BY queryID());
```
Результат:
``` text
┌─count()─┐
│ 3 │
└─────────┘
```
## initialQueryID {#initial-query-id}
Возвращает идентификатор родительского запроса. Другие параметры запроса могут быть извлечены из системной таблицы [system.query_log](../../operations/system-tables/query_log.md) через `initial_query_id`.
В отличие от [queryID](#query-id), функция `initialQueryID` возвращает одинаковые значения для разных шардов (см. пример).
**Синтаксис**
``` sql
initialQueryID()
```
**Возвращаемое значение**
- Идентификатор родительского запроса.
Тип: [String](../../sql-reference/data-types/string.md)
**Пример**
Запрос:
``` sql
CREATE TABLE tmp (str String) ENGINE = Log;
INSERT INTO tmp (*) VALUES ('a');
SELECT count(DISTINCT t) FROM (SELECT initialQueryID() AS t FROM remote('127.0.0.{1..3}', currentDatabase(), 'tmp') GROUP BY queryID());
```
Результат:
``` text
┌─count()─┐
│ 1 │
└─────────┘
```

18
docs/ru/sql-reference/functions/uuid-functions.md
View File

@ -243,9 +243,23 @@ SELECT
└──────────────────┴──────────────────────────────────────┘
```
## serverUUID() {#server-uuid}
Возвращает случайный и уникальный UUID, который генерируется при первом запуске сервера и сохраняется навсегда. Результат записывается в файл `uuid`, расположенный в каталоге сервера ClickHouse `/var/lib/clickhouse/`.
**Синтаксис**
```sql
serverUUID()
```
**Возвращаемое значение**
- UUID сервера.
Тип: [UUID](../data-types/uuid.md).
## См. также: {#sm-takzhe}
- [dictGetUUID](ext-dict-functions.md)
- [dictGetUUIDOrDefault](ext-dict-functions.md)
[Original article](https://clickhouse.tech/docs/en/query_language/functions/uuid_function/) <!--hide-->

4
docs/ru/sql-reference/statements/alter/projection.md
View File

@ -5,7 +5,7 @@ toc_title: PROJECTION
# Манипуляции с проекциями {#manipulations-with-projections}
Доступны следующие операции:
Доступны следующие операции с [проекциями](../../../engines/table-engines/mergetree-family/mergetree.md#projections):
- `ALTER TABLE [db].name ADD PROJECTION name AS SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]` — добавляет описание проекции в метаданные.
@ -15,7 +15,7 @@ toc_title: PROJECTION
- `ALTER TABLE [db.]table CLEAR PROJECTION name IN PARTITION partition_name` — удаляет файлы проекции с диска без удаления описания.
Комманды ADD, DROP и CLEAR — легковесны, поскольку они только меняют метаданные или удаляют файлы.
Команды `ADD`, `DROP` и `CLEAR` — легковесны, поскольку они только меняют метаданные или удаляют файлы.
Также команды реплицируются, синхронизируя описания проекций в метаданных с помощью ZooKeeper.

3
docs/ru/sql-reference/statements/select/join.md
View File

@ -36,6 +36,9 @@ FROM <left_table>
- `LEFT ANY JOIN`, `RIGHT ANY JOIN` и `INNER ANY JOIN`, Частично (для противоположных сторон `LEFT` и `RIGHT`) или полностью (для `INNER` и `FULL`) отключает декартово произведение для стандартных видов `JOIN`.
- `ASOF JOIN` и `LEFT ASOF JOIN`, Для соединения последовательностей по нечеткому совпадению. Использование `ASOF JOIN` описано ниже.
!!! note "Примечание"
Если настройка [join_algorithm](../../../operations/settings/settings.md#settings-join_algorithm) установлена в значение `partial_merge`, то для `RIGHT JOIN` и `FULL JOIN` поддерживается только уровень строгости `ALL` (`SEMI`, `ANTI`, `ANY` и `ASOF` не поддерживаются).
## Настройки {#join-settings}
Значение строгости по умолчанию может быть переопределено с помощью настройки [join_default_strictness](../../../operations/settings/settings.md#settings-join_default_strictness).

48
docs/zh/engines/database-engines/atomic.md
View File

@ -3,15 +3,55 @@ toc_priority: 32
toc_title: Atomic
---
# Atomic {#atomic}
它支持非阻塞 DROP 和 RENAME TABLE 查询以及原子 EXCHANGE TABLES t1 AND t2 查询。默认情况下使用Atomic数据库引擎。
它支持非阻塞的[DROP TABLE](#drop-detach-table)和[RENAME TABLE](#rename-table)查询和原子的[EXCHANGE TABLES t1 AND t2](#exchange-tables)查询。默认情况下使用`Atomic`数据库引擎。
## 创建数据库 {#creating-a-database}
``` sql
CREATE DATABASE test ENGINE = Atomic;
CREATE DATABASE test[ ENGINE = Atomic];
```
[原文](https://clickhouse.tech/docs/en/engines/database_engines/atomic/) <!--hide-->
## 使用方式 {#specifics-and-recommendations}
### Table UUID {#table-uuid}
数据库`Atomic`中的所有表都有唯一的[UUID](../../sql-reference/data-types/uuid.md)，并将数据存储在目录`/clickhouse_path/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/`，其中`xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy`是该表的UUID。
通常，UUID是自动生成的，但用户也可以在创建表时以相同的方式显式指定UUID(不建议这样做)。可以使用 [show_table_uuid_in_table_create_query_if_not_nil](../../operations/settings/settings.md#show_table_uuid_in_table_create_query_if_not_nil)设置。显示UUID的使用`SHOW CREATE`查询。例如:
```sql
CREATE TABLE name UUID '28f1c61c-2970-457a-bffe-454156ddcfef' (n UInt64) ENGINE = ...;
```
### RENAME TABLES {#rename-table}
`RENAME`查询是在不更改UUID和移动表数据的情况下执行的。这些查询不会等待使用表的查询完成，而是会立即执行。
### DROP/DETACH TABLES {#drop-detach-table}
在`DROP TABLE`上，不删除任何数据，数据库`Atomic`只是通过将元数据移动到`/clickhouse_path/metadata_dropped/`将表标记为已删除，并通知后台线程。最终表数据删除前的延迟由[database_atomic_delay_before_drop_table_sec](../../operations/server-configuration-parameters/settings.md#database_atomic_delay_before_drop_table_sec)设置指定。
可以使用`SYNC`修饰符指定同步模式。使用[database_atomic_wait_for_drop_and_detach_synchronously](../../operations/settings/settings.md#database_atomic_wait_for_drop_and_detach_synchronously)设置执行此操作。在本例中，`DROP`等待运行 `SELECT`, `INSERT`和其他使用表完成的查询。表在不使用时将被实际删除。
### EXCHANGE TABLES {#exchange-tables}
`EXCHANGE`以原子方式交换表。因此，不是这种非原子操作：
```sql
RENAME TABLE new_table TO tmp, old_table TO new_table, tmp TO old_table;
```
可以使用一个原子查询：
``` sql
EXCHANGE TABLES new_table AND old_table;
```
### ReplicatedMergeTree in Atomic Database {#replicatedmergetree-in-atomic-database}
对于[ReplicatedMergeTree](../table-engines/mergetree-family/replication.md#table_engines-replication)表，建议不要在ZooKeeper和副本名称中指定engine-path的参数。在这种情况下，将使用配置的参数[default_replica_path](../../operations/server-configuration-parameters/settings.md#default_replica_path)和[default_replica_name](../../operations/server-configuration-parameters/settings.md#default_replica_name)。如果要显式指定引擎的参数，建议使用{uuid}宏。这是非常有用的，以便为ZooKeeper中的每个表自动生成唯一的路径。
## 另请参考
- [system.databases](../../operations/system-tables/databases.md) 系统表

28
docs/zh/engines/database-engines/index.md
View File

@ -1,11 +1,29 @@
# 数据库引擎 {#shu-ju-ku-yin-qing}
---
toc_folder_title: 数据库引擎
toc_priority: 27
toc_title: Introduction
---
您使用的所有表都是由数据库引擎所提供的
# 数据库引擎 {#database-engines}
默认情况下，ClickHouse使用自己的数据库引擎，该引擎提供可配置的[表引擎](../../engines/database-engines/index.md)和[所有支持的SQL语法](../../engines/database-engines/index.md).
数据库引擎允许您处理数据表。
除此之外，您还可以选择使用以下的数据库引擎：
默认情况下，ClickHouse使用[Atomic](../../engines/database-engines/atomic.md)数据库引擎。它提供了可配置的[table engines](../../engines/table-engines/index.md)和[SQL dialect](../../sql-reference/syntax.md)。
- [MySQL](mysql.md)
您还可以使用以下数据库引擎：
- [MySQL](../../engines/database-engines/mysql.md)
- [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md)
- [Lazy](../../engines/database-engines/lazy.md)
- [Atomic](../../engines/database-engines/atomic.md)
- [PostgreSQL](../../engines/database-engines/postgresql.md)
- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md)
- [Replicated](../../engines/database-engines/replicated.md)
[来源文章](https://clickhouse.tech/docs/en/database_engines/) <!--hide-->

12
docs/zh/engines/database-engines/lazy.md
View File

@ -1,16 +1,18 @@
---
toc_priority: 31
toc_title: "延时引擎"
toc_title: Lazy
---
# 延时引擎Lazy {#lazy}
# Lazy {#lazy}
在距最近一次访问间隔`expiration_time_in_seconds`时间段内，将表保存在内存中，仅适用于 \*Log引擎表
在最后一次访问之后，只在RAM中保存`expiration_time_in_seconds`秒。只能用于\*Log表。
由于针对这类表的访问间隔较长，对保存大量小的 \*Log引擎表进行了优化，
它是为存储许多小的\*Log表而优化的，对于这些表，访问之间有很长的时间间隔。
## 创建数据库 {#creating-a-database}
``` sql
CREATE DATABASE testlazy ENGINE = Lazy(expiration_time_in_seconds);
```
[原始文章](https://clickhouse.tech/docs/en/database_engines/lazy/) <!--hide-->
[来源文章](https://clickhouse.tech/docs/en/database_engines/lazy/) <!--hide-->

197
docs/zh/engines/database-engines/materialize-mysql.md Normal file
View File

@ -0,0 +1,197 @@
---
toc_priority: 29
toc_title: "[experimental] MaterializedMySQL"
---
# [experimental] MaterializedMySQL {#materialized-mysql}
**这是一个实验性的特性，不应该在生产中使用。**
创建ClickHouse数据库，包含MySQL中所有的表，以及这些表中的所有数据。
ClickHouse服务器作为MySQL副本工作。它读取binlog并执行DDL和DML查询。
这个功能是实验性的。
## 创建数据库 {#creating-a-database}
``` sql
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
```
**引擎参数**
- `host:port` — MySQL服务地址
- `database` — MySQL数据库名称
- `user` — MySQL用户名
- `password` — MySQL用户密码
**引擎配置**
- `max_rows_in_buffer` — 允许数据缓存到内存中的最大行数(对于单个表和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `65505`。
- `max_bytes_in_buffer` — 允许在内存中缓存数据的最大字节数(对于单个表和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `1048576`.
- `max_rows_in_buffers` — 允许数据缓存到内存中的最大行数(对于数据库和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `65505`.
- `max_bytes_in_buffers` — 允许在内存中缓存数据的最大字节数(对于数据库和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `1048576`.
- `max_flush_data_time` — 允许数据在内存中缓存的最大毫秒数(对于数据库和无法查询的缓存数据)。当超过这个时间时，数据将被物化。默认值: `1000`.
- `max_wait_time_when_mysql_unavailable` — 当MySQL不可用时重试间隔(毫秒)。负值禁止重试。默认值: `1000`.
- `allows_query_when_mysql_lost` — 当mysql丢失时，允许查询物化表。默认值: `0` (`false`).
```
CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***')
SETTINGS
allows_query_when_mysql_lost=true,
max_wait_time_when_mysql_unavailable=10000;
```
**MySQL服务器端配置**
为了`MaterializeMySQL`正确的工作，有一些强制性的`MySQL`侧配置设置应该设置:
- `default_authentication_plugin = mysql_native_password`，因为`MaterializeMySQL`只能使用此方法授权。
- `gtid_mode = on`，因为要提供正确的`MaterializeMySQL`复制，基于GTID的日志记录是必须的。注意，在打开这个模式`On`时，你还应该指定`enforce_gtid_consistency = on`。
## 虚拟列 {#virtual-columns}
当使用`MaterializeMySQL`数据库引擎时，[ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md)表与虚拟的`_sign`和`_version`列一起使用。
- `_version` — 同步版本。 类型[UInt64](../../sql-reference/data-types/int-uint.md).
- `_sign` — 删除标记。类型 [Int8](../../sql-reference/data-types/int-uint.md). Possible values:
- `1` — 行不会删除,
- `-1` — 行被删除。
## 支持的数据类型 {#data_types-support}
| MySQL | ClickHouse |
|-------------------------|--------------------------------------------------------------|
| TINY | [Int8](../../sql-reference/data-types/int-uint.md) |
| SHORT | [Int16](../../sql-reference/data-types/int-uint.md) |
| INT24 | [Int32](../../sql-reference/data-types/int-uint.md) |
| LONG | [UInt32](../../sql-reference/data-types/int-uint.md) |
| LONGLONG | [UInt64](../../sql-reference/data-types/int-uint.md) |
| FLOAT | [Float32](../../sql-reference/data-types/float.md) |
| DOUBLE | [Float64](../../sql-reference/data-types/float.md) |
| DECIMAL, NEWDECIMAL | [Decimal](../../sql-reference/data-types/decimal.md) |
| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) |
| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) |
| ENUM | [Enum](../../sql-reference/data-types/enum.md) |
| STRING | [String](../../sql-reference/data-types/string.md) |
| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) |
| BLOB | [String](../../sql-reference/data-types/string.md) |
| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) |
不支持其他类型。如果MySQL表包含此类类型的列，ClickHouse抛出异常"Unhandled data type"并停止复制。
[Nullable](../../sql-reference/data-types/nullable.md)已经支持
## 使用方式 {#specifics-and-recommendations}
### 兼容性限制
除了数据类型的限制外，与`MySQL`数据库相比，还存在一些限制，在实现复制之前应先解决这些限制：
- `MySQL`中的每个表都应该包含`PRIMARY KEY`
- 对于包含`ENUM`字段值超出范围（在`ENUM`签名中指定）的行的表，复制将不起作用。
### DDL查询 {#ddl-queries}
MySQL DDL查询转换为相应的ClickHouse DDL查询([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create/index.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md))。如果ClickHouse无法解析某个DDL查询，则该查询将被忽略。
### Data Replication {#data-replication}
`MaterializeMySQL`不支持直接`INSERT`, `DELETE`和`UPDATE`查询. 但是，它们是在数据复制方面支持的：
- MySQL的`INSERT`查询转换为`INSERT`并携带`_sign=1`.
- MySQL的`DELETE`查询转换为`INSERT`并携带`_sign=-1`.
- MySQL的`UPDATE`查询转换为`INSERT`并携带`_sign=-1`, `INSERT`和`_sign=1`.
### 查询MaterializeMySQL表 {#select}
`SELECT`查询`MaterializeMySQL`表有一些细节:
- 如果`_version`在`SELECT`中没有指定，则使用[FINAL](../../sql-reference/statements/select/from.md#select-from-final)修饰符。所以只有带有`MAX(_version)`的行才会被选中。
- 如果`_sign`在`SELECT`中没有指定，则默认使用`WHERE _sign=1`。因此，删除的行不会包含在结果集中。
- 结果包括列中的列注释，因为它们存在于SQL数据库表中。
### Index Conversion {#index-conversion}
MySQL的`PRIMARY KEY`和`INDEX`子句在ClickHouse表中转换为`ORDER BY`元组。
ClickHouse只有一个物理顺序，由`ORDER BY`子句决定。要创建一个新的物理顺序，使用[materialized views](../../sql-reference/statements/create/view.md#materialized)。
**Notes**
- 带有`_sign=-1`的行不会从表中物理删除。
- `MaterializeMySQL`引擎不支持级联`UPDATE/DELETE`查询。
- 复制很容易被破坏。
- 禁止对数据库和表进行手工操作。
- `MaterializeMySQL`受[optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert)设置的影响。当MySQL服务器中的表发生变化时，数据会合并到`MaterializeMySQL`数据库中相应的表中。
## 使用示例 {#examples-of-use}
MySQL操作:
``` sql
mysql> CREATE DATABASE db;
mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT);
mysql> INSERT INTO db.test VALUES (1, 11), (2, 22);
mysql> DELETE FROM db.test WHERE a=1;
mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16);
mysql> UPDATE db.test SET c='Wow!', b=222;
mysql> SELECT * FROM test;
```
```text
+---+------+------+
| a | b | c |
+---+------+------+
| 2 | 222 | Wow! |
+---+------+------+
```
ClickHouse中的数据库，与MySQL服务器交换数据:
创建的数据库和表:
``` sql
CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***');
SHOW TABLES FROM mysql;
```
``` text
┌─name─┐
│ test │
└──────┘
```
然后插入数据:
``` sql
SELECT * FROM mysql.test;
```
``` text
┌─a─┬──b─┐
│ 1 │ 11 │
│ 2 │ 22 │
└───┴────┘
```
删除数据后，添加列并更新:
``` sql
SELECT * FROM mysql.test;
```
``` text
┌─a─┬───b─┬─c────┐
│ 2 │ 222 │ Wow! │
└───┴─────┴──────┘
```
[来源文章](https://clickhouse.tech/docs/en/engines/database-engines/materialize-mysql/) <!--hide-->

85
docs/zh/engines/database-engines/materialized-postgresql.md Normal file
View File

@ -0,0 +1,85 @@
---
toc_priority: 30
toc_title: MaterializedPostgreSQL
---
# [experimental] MaterializedPostgreSQL {#materialize-postgresql}
使用PostgreSQL数据库表的初始数据转储创建ClickHouse数据库，并启动复制过程，即执行后台作业，以便在远程PostgreSQL数据库中的PostgreSQL数据库表上发生新更改时应用这些更改。
ClickHouse服务器作为PostgreSQL副本工作。它读取WAL并执行DML查询。DDL不是复制的，但可以处理（如下所述）。
## 创建数据库 {#creating-a-database}
``` sql
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
ENGINE = MaterializedPostgreSQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
```
**Engine参数**
- `host:port` — PostgreSQL服务地址
- `database` — PostgreSQL数据库名
- `user` — PostgreSQL用户名
- `password` — 用户密码
## 设置 {#settings}
- [materialized_postgresql_max_block_size](../../operations/settings/settings.md#materialized-postgresql-max-block-size)
- [materialized_postgresql_tables_list](../../operations/settings/settings.md#materialized-postgresql-tables-list)
- [materialized_postgresql_allow_automatic_update](../../operations/settings/settings.md#materialized-postgresql-allow-automatic-update)
``` sql
CREATE DATABASE database1
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password')
SETTINGS materialized_postgresql_max_block_size = 65536,
materialized_postgresql_tables_list = 'table1,table2,table3';
SELECT * FROM database1.table1;
```
## 必备条件 {#requirements}
- 在postgresql配置文件中将[wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html)设置为`logical`，将`max_replication_slots`设置为`2`。
- 每个复制表必须具有以下一个[replica identity](https://www.postgresql.org/docs/10/sql-altertable.html#SQL-CREATETABLE-REPLICA-IDENTITY):
1. **default** (主键)
2. **index**
``` bash
postgres# CREATE TABLE postgres_table (a Integer NOT NULL, b Integer, c Integer NOT NULL, d Integer, e Integer NOT NULL);
postgres# CREATE unique INDEX postgres_table_index on postgres_table(a, c, e);
postgres# ALTER TABLE postgres_table REPLICA IDENTITY USING INDEX postgres_table_index;
```
总是先检查主键。如果不存在，则检查索引(定义为副本标识索引)。
如果使用index作为副本标识，则表中必须只有一个这样的索引。
你可以用下面的命令来检查一个特定的表使用了什么类型:
``` bash
postgres# SELECT CASE relreplident
WHEN 'd' THEN 'default'
WHEN 'n' THEN 'nothing'
WHEN 'f' THEN 'full'
WHEN 'i' THEN 'index'
END AS replica_identity
FROM pg_class
WHERE oid = 'postgres_table'::regclass;
```
## 注意 {#warning}
1. [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.html)不支持值转换。将使用数据类型的默认值。
## 使用示例 {#example-of-use}
``` sql
CREATE DATABASE postgresql_db
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password');
SELECT * FROM postgresql_db.postgres_table;
```

58
docs/zh/engines/database-engines/mysql.md
View File

@ -1,6 +1,11 @@
---
toc_priority: 30
toc_title: MySQL
---
# MySQL {#mysql}
MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并允许您对表进行`INSERT`和`SELECT`查询，以方便您在ClickHouse与MySQL之间进行数据交换。
MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并允许您对表进行`INSERT`和`SELECT`查询，以方便您在ClickHouse与MySQL之间进行数据交换
`MySQL`数据库引擎会将对其的查询转换为MySQL语法并发送到MySQL服务器中，因此您可以执行诸如`SHOW TABLES`或`SHOW CREATE TABLE`之类的操作。
@ -10,24 +15,24 @@ MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并
- `CREATE TABLE`
- `ALTER`
## CREATE DATABASE {#create-database}
## 创建数据库 {#creating-a-database}
``` sql
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
ENGINE = MySQL('host:port', ['database' | database], 'user', 'password')
```
**MySQL数据库引擎参数**
**引擎参数**
- `host:port` — 链接的MySQL地址。
- `database` — 链接的MySQL数据库。
- `user` — 链接的MySQL用户。
- `password` — 链接的MySQL用户密码。
- `host:port` — MySQL服务地址
- `database` — MySQL数据库名称
- `user` — MySQL用户名
- `password` — MySQL用户密码
## 支持的类型对应 {#zhi-chi-de-lei-xing-dui-ying}
## 支持的数据类型 {#data_types-support}
| MySQL | ClickHouse |
|----------------------------------|-------------------------------------------------------------|
|----------------------------------|--------------------------------------------------------------|
| UNSIGNED TINYINT | [UInt8](../../sql-reference/data-types/int-uint.md) |
| TINYINT | [Int8](../../sql-reference/data-types/int-uint.md) |
| UNSIGNED SMALLINT | [UInt16](../../sql-reference/data-types/int-uint.md) |
@ -42,14 +47,32 @@ ENGINE = MySQL('host:port', ['database' | database], 'user', 'password')
| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) |
其他的MySQL数据类型将全部都转换为[String](../../sql-reference/data-types/string.md)。
其他的MySQL数据类型将全部都转换为[String](../../sql-reference/data-types/string.md).
同时以上的所有类型都支持[Nullable](../../sql-reference/data-types/nullable.md)。
[Nullable](../../sql-reference/data-types/nullable.md)已经支持
## 使用示例 {#shi-yong-shi-li}
## 全局变量支持 {#global-variables-support}
在MySQL中创建表:
为了更好地兼容，您可以在SQL样式中设置全局变量，如`@@identifier`.
支持这些变量：
- `version`
- `max_allowed_packet`
!!! warning "警告"
到目前为止，这些变量是存根，并且不对应任何内容。
示例:
``` sql
SELECT @@version;
```
## 使用示例 {#examples-of-use}
MySQL操作:
``` text
mysql> USE test;
Database changed
@ -63,14 +86,15 @@ ENGINE = MySQL('host:port', ['database' | database], 'user', 'password')
Query OK, 1 row affected (0,00 sec)
mysql> select * from mysql_table;
+--------+-------+
+------+-----+
| int_id | value |
+--------+-------+
+------+-----+
| 1 | 2 |
+--------+-------+
+------+-----+
1 row in set (0,00 sec)
```
在ClickHouse中创建MySQL类型的数据库，同时与MySQL服务器交换数据：
ClickHouse中的数据库，与MySQL服务器交换数据:
``` sql
CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password')

138
docs/zh/engines/database-engines/postgresql.md Normal file
View File

@ -0,0 +1,138 @@
---
toc_priority: 35
toc_title: PostgreSQL
---
# PostgreSQL {#postgresql}
允许连接到远程[PostgreSQL](https://www.postgresql.org)服务。支持读写操作(`SELECT`和`INSERT`查询)，以在ClickHouse和PostgreSQL之间交换数据。
在`SHOW TABLES`和`DESCRIBE TABLE`查询的帮助下，从远程PostgreSQL实时访问表列表和表结构。
支持表结构修改(`ALTER TABLE ... ADD|DROP COLUMN`)。如果`use_table_cache`参数(参见下面的引擎参数)设置为`1`，则会缓存表结构，不会检查是否被修改，但可以用`DETACH`和`ATTACH`查询进行更新。
## 创建数据库 {#creating-a-database}
``` sql
CREATE DATABASE test_database
ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cache`]);
```
**引擎参数**
- `host:port` — PostgreSQL服务地址
- `database` — 远程数据库名次
- `user` — PostgreSQL用户名称
- `password` — PostgreSQL用户密码
- `use_table_cache` — 定义数据库表结构是否已缓存或不进行。可选的。默认值： `0`.
## 支持的数据类型 {#data_types-support}
| PostgerSQL | ClickHouse |
|------------------|--------------------------------------------------------------|
| DATE | [Date](../../sql-reference/data-types/date.md) |
| TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
| REAL | [Float32](../../sql-reference/data-types/float.md) |
| DOUBLE | [Float64](../../sql-reference/data-types/float.md) |
| DECIMAL, NUMERIC | [Decimal](../../sql-reference/data-types/decimal.md) |
| SMALLINT | [Int16](../../sql-reference/data-types/int-uint.md) |
| INTEGER | [Int32](../../sql-reference/data-types/int-uint.md) |
| BIGINT | [Int64](../../sql-reference/data-types/int-uint.md) |
| SERIAL | [UInt32](../../sql-reference/data-types/int-uint.md) |
| BIGSERIAL | [UInt64](../../sql-reference/data-types/int-uint.md) |
| TEXT, CHAR | [String](../../sql-reference/data-types/string.md) |
| INTEGER | Nullable([Int32](../../sql-reference/data-types/int-uint.md))|
| ARRAY | [Array](../../sql-reference/data-types/array.md) |
## 使用示例 {#examples-of-use}
ClickHouse中的数据库，与PostgreSQL服务器交换数据:
``` sql
CREATE DATABASE test_database
ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', 1);
```
``` sql
SHOW DATABASES;
```
``` text
┌─name──────────┐
│ default │
│ test_database │
│ system │
└───────────────┘
```
``` sql
SHOW TABLES FROM test_database;
```
``` text
┌─name───────┐
│ test_table │
└────────────┘
```
从PostgreSQL表中读取数据：
``` sql
SELECT * FROM test_database.test_table;
```
``` text
┌─id─┬─value─┐
│ 1 │ 2 │
└────┴───────┘
```
将数据写入PostgreSQL表：
``` sql
INSERT INTO test_database.test_table VALUES (3,4);
SELECT * FROM test_database.test_table;
```
``` text
┌─int_id─┬─value─┐
│ 1 │ 2 │
│ 3 │ 4 │
└────────┴───────┘
```
在PostgreSQL中修改了表结构：
``` sql
postgre> ALTER TABLE test_table ADD COLUMN data Text
```
当创建数据库时，参数`use_table_cache`被设置为`1`，ClickHouse中的表结构被缓存，因此没有被修改:
``` sql
DESCRIBE TABLE test_database.test_table;
```
``` text
┌─name───┬─type──────────────┐
│ id │ Nullable(Integer) │
│ value │ Nullable(Integer) │
└────────┴───────────────────┘
```
分离表并再次附加它之后，结构被更新了:
``` sql
DETACH TABLE test_database.test_table;
ATTACH TABLE test_database.test_table;
DESCRIBE TABLE test_database.test_table;
```
``` text
┌─name───┬─type──────────────┐
│ id │ Nullable(Integer) │
│ value │ Nullable(Integer) │
│ data │ Nullable(String) │
└────────┴───────────────────┘
```
[来源文章](https://clickhouse.tech/docs/en/database-engines/postgresql/) <!--hide-->

116
docs/zh/engines/database-engines/replicated.md Normal file
View File

@ -0,0 +1,116 @@
# [experimental] Replicated {#replicated}
该引擎基于[Atomic](../../engines/database-engines/atomic.md)引擎。它支持通过将DDL日志写入ZooKeeper并在给定数据库的所有副本上执行的元数据复制。
一个ClickHouse服务器可以同时运行和更新多个复制的数据库。但是同一个复制的数据库不能有多个副本。
## 创建数据库 {#creating-a-database}
``` sql
CREATE DATABASE testdb ENGINE = Replicated('zoo_path', 'shard_name', 'replica_name') [SETTINGS ...]
```
**引擎参数**
- `zoo_path` — ZooKeeper地址，同一个ZooKeeper路径对应同一个数据库。
- `shard_name` — 分片的名字。数据库副本按`shard_name`分组到分片中。
- `replica_name` — 副本的名字。同一分片的所有副本的副本名称必须不同。
!!! note "警告"
对于[ReplicatedMergeTree](../table-engines/mergetree-family/replication.md#table_engines-replication)表，如果没有提供参数，则使用默认参数:`/clickhouse/tables/{uuid}/{shard}`和`{replica}`。这些可以在服务器设置[default_replica_path](../../operations/server-configuration-parameters/settings.md#default_replica_path)和[default_replica_name](../../operations/server-configuration-parameters/settings.md#default_replica_name)中更改。宏`{uuid}`被展开到表的uuid， `{shard}`和`{replica}`被展开到服务器配置的值，而不是数据库引擎参数。但是在将来，可以使用Replicated数据库的`shard_name`和`replica_name`。
## 使用方式 {#specifics-and-recommendations}
使用`Replicated`数据库的DDL查询的工作方式类似于[ON CLUSTER](../../sql-reference/distributed-ddl.md)查询，但有细微差异。
首先，DDL请求尝试在启动器(最初从用户接收请求的主机)上执行。如果请求没有完成，那么用户立即收到一个错误，其他主机不会尝试完成它。如果在启动器上成功地完成了请求，那么所有其他主机将自动重试，直到完成请求。启动器将尝试在其他主机上等待查询完成(不超过[distributed_ddl_task_timeout](../../operations/settings/settings.md#distributed_ddl_task_timeout))，并返回一个包含每个主机上查询执行状态的表。
错误情况下的行为是由[distributed_ddl_output_mode](../../operations/settings/settings.md#distributed_ddl_output_mode)设置调节的，对于`Replicated`数据库，最好将其设置为`null_status_on_timeout` - 例如，如果一些主机没有时间执行[distributed_ddl_task_timeout](../../operations/settings/settings.md#distributed_ddl_task_timeout)的请求，那么不要抛出异常，但在表中显示它们的`NULL`状态。
[system.clusters](../../operations/system-tables/clusters.md)系统表包含一个名为复制数据库的集群，它包含数据库的所有副本。当创建/删除副本时，这个集群会自动更新，它可以用于[Distributed](../../engines/table-engines/special/distributed.md#distributed)表。
当创建数据库的新副本时，该副本会自己创建表。如果副本已经不可用很长一段时间，并且已经滞后于复制日志-它用ZooKeeper中的当前元数据检查它的本地元数据，将带有数据的额外表移动到一个单独的非复制数据库(以免意外地删除任何多余的东西)，创建缺失的表，如果表名已经被重命名，则更新表名。数据在`ReplicatedMergeTree`级别被复制，也就是说，如果表没有被复制，数据将不会被复制(数据库只负责元数据)。
## 使用示例 {#usage-example}
创建三台主机的集群:
``` sql
node1 :) CREATE DATABASE r ENGINE=Replicated('some/path/r','shard1','replica1');
node2 :) CREATE DATABASE r ENGINE=Replicated('some/path/r','shard1','other_replica');
node3 :) CREATE DATABASE r ENGINE=Replicated('some/path/r','other_shard','{replica}');
```
运行DDL:
``` sql
CREATE TABLE r.rmt (n UInt64) ENGINE=ReplicatedMergeTree ORDER BY n;
```
``` text
┌─────hosts────────────┬──status─┬─error─┬─num_hosts_remaining─┬─num_hosts_active─┐
│ shard1|replica1 │ 0 │ │ 2 │ 0 │
│ shard1|other_replica │ 0 │ │ 1 │ 0 │
│ other_shard|r1 │ 0 │ │ 0 │ 0 │
└──────────────────────┴─────────┴───────┴─────────────────────┴──────────────────┘
```
显示系统表:
``` sql
SELECT cluster, shard_num, replica_num, host_name, host_address, port, is_local
FROM system.clusters WHERE cluster='r';
```
``` text
┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐
│ r │ 1 │ 1 │ node3 │ 127.0.0.1 │ 9002 │ 0 │
│ r │ 2 │ 1 │ node2 │ 127.0.0.1 │ 9001 │ 0 │
│ r │ 2 │ 2 │ node1 │ 127.0.0.1 │ 9000 │ 1 │
└─────────┴───────────┴─────────────┴───────────┴──────────────┴──────┴──────────┘
```
创建分布式表并插入数据:
``` sql
node2 :) CREATE TABLE r.d (n UInt64) ENGINE=Distributed('r','r','rmt', n % 2);
node3 :) INSERT INTO r SELECT * FROM numbers(10);
node1 :) SELECT materialize(hostName()) AS host, groupArray(n) FROM r.d GROUP BY host;
```
``` text
┌─hosts─┬─groupArray(n)─┐
│ node1 │ [1,3,5,7,9] │
│ node2 │ [0,2,4,6,8] │
└───────┴───────────────┘
```
向一台主机添加副本:
``` sql
node4 :) CREATE DATABASE r ENGINE=Replicated('some/path/r','other_shard','r2');
```
集群配置如下所示:
``` text
┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐
│ r │ 1 │ 1 │ node3 │ 127.0.0.1 │ 9002 │ 0 │
│ r │ 1 │ 2 │ node4 │ 127.0.0.1 │ 9003 │ 0 │
│ r │ 2 │ 1 │ node2 │ 127.0.0.1 │ 9001 │ 0 │
│ r │ 2 │ 2 │ node1 │ 127.0.0.1 │ 9000 │ 1 │
└─────────┴───────────┴─────────────┴───────────┴──────────────┴──────┴──────────┘
```
分布式表也将从新主机获取数据:
```sql
node2 :) SELECT materialize(hostName()) AS host, groupArray(n) FROM r.d GROUP BY host;
```
```text
┌─hosts─┬─groupArray(n)─┐
│ node2 │ [1,3,5,7,9] │
│ node4 │ [0,2,4,6,8] │
└───────┴───────────────┘
```

2
docs/zh/getting-started/tutorial.md
View File

@ -520,7 +520,7 @@ WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartU
ClickHouse集群是一个同质集群。 设置步骤:
1. 在群集的所有机器上安装ClickHouse服务端
2. 在配置文件中设置群集配置
2. 在配置文件中设置集群配置
3. 在每个实例上创建本地表
4. 创建一个[分布式表](../engines/table-engines/special/distributed.md)

16
docs/zh/interfaces/third-party/gui.md vendored
View File

@ -99,4 +99,20 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix).
- 重构。
- 搜索和导航。
### DBM {#dbm}
[DBM](https://dbm.incubator.edurt.io/) DBM是一款ClickHouse可视化管理工具!
特征：
- 支持查询历史（分页、全部清除等）
- 支持选中的sql子句查询(多窗口等)
- 支持终止查询
- 支持表管理
- 支持数据库管理
- 支持自定义查询
- 支持多数据源管理（连接测试、监控）
- 支持监控（处理进程、连接、查询）
- 支持迁移数据
[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/gui/) <!--hide-->

21
docs/zh/sql-reference/distributed-ddl.md Normal file
View File

@ -0,0 +1,21 @@
---
toc_priority: 32
toc_title: Distributed DDL
---
# 分布式DDL查询(ON CLUSTER条件) {#distributed-ddl-queries-on-cluster-clause}
默认情况下，`CREATE`、`DROP`、`ALTER`和`RENAME`查询仅影响执行它们的当前服务器。 在集群设置中，可以使用`ON CLUSTER`子句以分布式方式运行此类查询。
例如，以下查询在`cluster`中的每个主机上创建`all_hits` `Distributed`表：
``` sql
CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE = Distributed(cluster, default, hits)
```
为了正确运行这些查询，每个主机必须具有相同的集群定义（为了简化同步配置，您可以使用ZooKeeper替换）。 他们还必须连接到ZooKeeper服务器。
本地版本的查询最终会在集群中的每台主机上执行，即使某些主机当前不可用。
!!! warning "警告"
在单个主机内执行查询的顺序是有保证的。

2
docs/zh/sql-reference/operators/in.md
View File

@ -5,7 +5,7 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3
# IN 操作符 {#select-in-operators}
该 `IN`, `NOT IN`, `GLOBAL IN`，和 `GLOBAL NOT IN` 运算符是单独复盖的，因为它们的功能相当丰富。
该 `IN`, `NOT IN`, `GLOBAL IN`，和 `GLOBAL NOT IN` 运算符是单独考虑的，因为它们的功能相当丰富。
运算符的左侧是单列或元组。

23
docs/zh/sql-reference/statements/alter/index.md Normal file
View File

@ -0,0 +1,23 @@
---
toc_hidden_folder: true
toc_priority: 42
toc_title: INDEX
---
# 操作数据跳过索引 {#manipulations-with-data-skipping-indices}
可以使用以下操作：
- `ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value [FIRST|AFTER name]` - 向表元数据添加索引描述。
- `ALTER TABLE [db].name DROP INDEX name` - 从表元数据中删除索引描述并从磁盘中删除索引文件。
- `ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name` - 查询在分区`partition_name`中重建二级索引`name`。 操作为[mutation](../../../sql-reference/statements/alter/index.md#mutations).
前两个命令是轻量级的，它们只更改元数据或删除文件。
Also, they are replicated, syncing indices metadata via ZooKeeper.
此外，它们会被复制，会通过ZooKeeper同步索引元数据。
!!! note "注意"
索引操作仅支持具有以下特征的表 [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md)引擎 (包括[replicated](../../../engines/table-engines/mergetree-family/replication.md)).

29
docs/zh/sql-reference/statements/create/database.md Normal file
View File

@ -0,0 +1,29 @@
---
toc_priority: 35
toc_title: DATABASE
---
# CREATE DATABASE {#query-language-create-database}
创建数据库.
``` sql
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(...)]
```
## 条件 {#clauses}
### IF NOT EXISTS {#if-not-exists}
如果`db_name`数据库已经存在，则ClickHouse不会创建新数据库并且：
- 如果指定了子句，则不会引发异常。
- 如果未指定子句，则抛出异常。
### ON CLUSTER {#on-cluster}
ClickHouse在指定集群的所有服务器上创建`db_name`数据库。 更多细节在 [Distributed DDL](../../../sql-reference/distributed-ddl.md) article.
### ENGINE {#engine}
[MySQL](../../../engines/database-engines/mysql.md) 允许您从远程MySQL服务器检索数据. 默认情况下，ClickHouse使用自己的[database engine](../../../engines/database-engines/index.md). 还有一个[lazy](../../../engines/database-engines/lazy.md)引擎.

13
docs/zh/sql-reference/statements/create/index.md Normal file
View File

@ -0,0 +1,13 @@
---
toc_folder_title: CREATE
toc_priority: 34
toc_title: Overview
---
# CREATE语法 {#create-queries}
CREATE语法包含以下子集:
- [DATABASE](../../../sql-reference/statements/create/database.md)
[原始文章](https://clickhouse.tech/docs/zh/sql-reference/statements/create/) <!--hide-->

243
docs/zh/sql-reference/statements/create/view.md Normal file
View File

@ -0,0 +1,243 @@
---
toc_priority: 37
toc_title: VIEW
---
# CREATE VIEW {#create-view}
创建一个新视图。 有两种类型的视图：普通视图和物化视图。
## Normal {#normal}
语法:
``` sql
CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] AS SELECT ...
```
普通视图不存储任何数据。 他们只是在每次访问时从另一个表执行读取。换句话说，普通视图只不过是一个保存的查询。 从视图中读取时，此保存的查询用作[FROM](../../../sql-reference/statements/select/from.md)子句中的子查询.
例如，假设您已经创建了一个视图：
``` sql
CREATE VIEW view AS SELECT ...
```
并写了一个查询：
``` sql
SELECT a, b, c FROM view
```
这个查询完全等同于使用子查询：
``` sql
SELECT a, b, c FROM (SELECT ...)
```
## Materialized {#materialized}
``` sql
CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]name] [ENGINE = engine] [POPULATE] AS SELECT ...
```
物化视图存储由相应的[SELECT](../../../sql-reference/statements/select/index.md)管理.
创建不带`TO [db].[table]`的物化视图时，必须指定`ENGINE` – 用于存储数据的表引擎。
使用`TO [db].[table]` 创建物化视图时，不得使用`POPULATE`。
一个物化视图的实现是这样的：当向SELECT中指定的表插入数据时，插入数据的一部分被这个SELECT查询转换，结果插入到视图中。
!!! important "重要"
ClickHouse 中的物化视图更像是插入触发器。 如果视图查询中有一些聚合，则它仅应用于一批新插入的数据。 对源表现有数据的任何更改（如更新、删除、删除分区等）都不会更改物化视图。
如果指定`POPULATE`，则在创建视图时将现有表数据插入到视图中，就像创建一个`CREATE TABLE ... AS SELECT ...`一样。 否则，查询仅包含创建视图后插入表中的数据。 我们**不建议**使用POPULATE，因为在创建视图期间插入表中的数据不会插入其中。
`SELECT` 查询可以包含`DISTINCT`、`GROUP BY`、`ORDER BY`、`LIMIT`……请注意，相应的转换是在每个插入数据块上独立执行的。 例如，如果设置了`GROUP BY`，则在插入期间聚合数据，但仅在插入数据的单个数据包内。 数据不会被进一步聚合。 例外情况是使用独立执行数据聚合的`ENGINE`，例如`SummingMergeTree`。
在物化视图上执行[ALTER](../../../sql-reference/statements/alter/index.md)查询有局限性，因此可能不方便。 如果物化视图使用构造`TO [db.]name`，你可以`DETACH`视图，为目标表运行`ALTER`，然后`ATTACH`先前分离的（`DETACH`）视图。
请注意，物化视图受[optimize_on_insert](../../../operations/settings/settings.md#optimize-on-insert)设置的影响。 在插入视图之前合并数据。
视图看起来与普通表相同。 例如，它们列在1SHOW TABLES1查询的结果中。
删除视图,使用[DROP VIEW](../../../sql-reference/statements/drop.md#drop-view). `DROP TABLE`也适用于视图。
## Live View (实验性) {#live-view}
!!! important "重要"
这是一项实验性功能，可能会在未来版本中以向后不兼容的方式进行更改。
使用[allow_experimental_live_view](../../../operations/settings/settings.md#allow-experimental-live-view)设置启用实时视图和`WATCH`查询的使用。 输入命令`set allow_experimental_live_view = 1`。
```sql
CREATE LIVE VIEW [IF NOT EXISTS] [db.]table_name [WITH [TIMEOUT [value_in_sec] [AND]] [REFRESH [value_in_sec]]] AS SELECT ...
```
实时视图存储相应[SELECT](../../../sql-reference/statements/select/index.md)查询的结果，并在查询结果更改时随时更新。 查询结果以及与新数据结合所需的部分结果存储在内存中，为重复查询提供更高的性能。当使用[WATCH](../../../sql-reference/statements/watch.md)查询更改查询结果时，实时视图可以提供推送通知。
实时视图是通过插入到查询中指定的最里面的表来触发的。
实时视图的工作方式类似于分布式表中查询的工作方式。 但不是组合来自不同服务器的部分结果，而是将当前数据的部分结果与新数据的部分结果组合在一起。当实时视图查询包含子查询时，缓存的部分结果仅存储在最里面的子查询中。
!!! info "限制"
- [Table function](../../../sql-reference/table-functions/index.md)不支持作为最里面的表.
- 没有插入的表，例如[dictionary](../../../sql-reference/dictionaries/index.md), [system table](../../../operations/system-tables/index.md), [normal view](#normal), [materialized view](#materialized)不会触发实时视图。
- 只有可以将旧数据的部分结果与新数据的部分结果结合起来的查询才有效。 实时视图不适用于需要完整数据集来计算最终结果或必须保留聚合状态的聚合的查询。
- 不适用于在不同节点上执行插入的复制或分布式表。
- 不能被多个表触发。
[WITH REFRESH](#live-view-with-refresh)强制定期更新实时视图，在某些情况下可以用作解决方法。
### Monitoring Changes {#live-view-monitoring}
您可以使用[WATCH](../../../sql-reference/statements/watch.md)命令监视`LIVE VIEW`查询结果的变化
```sql
WATCH [db.]live_view
```
**示例:**
```sql
CREATE TABLE mt (x Int8) Engine = MergeTree ORDER BY x;
CREATE LIVE VIEW lv AS SELECT sum(x) FROM mt;
```
在对源表进行并行插入时观看实时视图。
```sql
WATCH lv;
```
```bash
┌─sum(x)─┬─_version─┐
│ 1 │ 1 │
└────────┴──────────┘
┌─sum(x)─┬─_version─┐
│ 2 │ 2 │
└────────┴──────────┘
┌─sum(x)─┬─_version─┐
│ 6 │ 3 │
└────────┴──────────┘
...
```
```sql
INSERT INTO mt VALUES (1);
INSERT INTO mt VALUES (2);
INSERT INTO mt VALUES (3);
```
添加[EVENTS](../../../sql-reference/statements/watch.md#events-clause)子句只获取更改事件。
```sql
WATCH [db.]live_view EVENTS;
```
**示例:**
```sql
WATCH lv EVENTS;
```
```bash
┌─version─┐
│ 1 │
└─────────┘
┌─version─┐
│ 2 │
└─────────┘
┌─version─┐
│ 3 │
└─────────┘
...
```
你可以执行[SELECT](../../../sql-reference/statements/select/index.md)与任何常规视图或表格相同的方式查询实时视图。如果查询结果被缓存，它将立即返回结果而不在基础表上运行存储的查询。
```sql
SELECT * FROM [db.]live_view WHERE ...
```
### Force Refresh {#live-view-alter-refresh}
您可以使用`ALTER LIVE VIEW [db.]table_name REFRESH`语法.
### WITH TIMEOUT条件 {#live-view-with-timeout}
当使用`WITH TIMEOUT`子句创建实时视图时，[WATCH](../../../sql-reference/statements/watch.md)观察实时视图的查询。
```sql
CREATE LIVE VIEW [db.]table_name WITH TIMEOUT [value_in_sec] AS SELECT ...
```
如果未指定超时值，则由指定的值[temporary_live_view_timeout](../../../operations/settings/settings.md#temporary-live-view-timeout)决定.
**示例:**
```sql
CREATE TABLE mt (x Int8) Engine = MergeTree ORDER BY x;
CREATE LIVE VIEW lv WITH TIMEOUT 15 AS SELECT sum(x) FROM mt;
```
### WITH REFRESH条件 {#live-view-with-refresh}
当使用`WITH REFRESH`子句创建实时视图时，它将在自上次刷新或触发后经过指定的秒数后自动刷新。
```sql
CREATE LIVE VIEW [db.]table_name WITH REFRESH [value_in_sec] AS SELECT ...
```
如果未指定刷新值，则由指定的值[periodic_live_view_refresh](../../../operations/settings/settings.md#periodic-live-view-refresh)决定.
**示例:**
```sql
CREATE LIVE VIEW lv WITH REFRESH 5 AS SELECT now();
WATCH lv
```
```bash
┌───────────────now()─┬─_version─┐
│ 2021-02-21 08:47:05 │ 1 │
└─────────────────────┴──────────┘
┌───────────────now()─┬─_version─┐
│ 2021-02-21 08:47:10 │ 2 │
└─────────────────────┴──────────┘
┌───────────────now()─┬─_version─┐
│ 2021-02-21 08:47:15 │ 3 │
└─────────────────────┴──────────┘
```
您可以使用`AND`子句组合`WITH TIMEOUT`和`WITH REFRESH`子句。
```sql
CREATE LIVE VIEW [db.]table_name WITH TIMEOUT [value_in_sec] AND REFRESH [value_in_sec] AS SELECT ...
```
**示例:**
```sql
CREATE LIVE VIEW lv WITH TIMEOUT 15 AND REFRESH 5 AS SELECT now();
```
15 秒后，如果没有活动的`WATCH`查询，实时视图将自动删除。
```sql
WATCH lv
```
```
Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.lv does not exist..
```
### Usage {#live-view-usage}
实时视图表的最常见用途包括：
- 为查询结果更改提供推送通知以避免轮询。
- 缓存最频繁查询的结果以提供即时查询结果。
- 监视表更改并触发后续选择查询。
- 使用定期刷新从系统表中查看指标。
[原始文章](https://clickhouse.tech/docs/en/sql-reference/statements/create/view/) <!--hide-->

100
docs/zh/sql-reference/statements/drop.md Normal file
View File

@ -0,0 +1,100 @@
---
toc_priority: 44
toc_title: DROP
---
# DROP语法 {#drop}
删除现有实体。 如果指定了`IF EXISTS`子句，如果实体不存在，这些查询不会返回错误。
## DROP DATABASE {#drop-database}
删除`db`数据库中的所有表，然后删除`db`数据库本身。
语法:
``` sql
DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster]
```
## DROP TABLE {#drop-table}
删除数据表
语法:
``` sql
DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]
```
## DROP DICTIONARY {#drop-dictionary}
删除字典。
语法:
``` sql
DROP DICTIONARY [IF EXISTS] [db.]name
```
## DROP USER {#drop-user-statement}
删除用户.
语法:
``` sql
DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
```
## DROP ROLE {#drop-role-statement}
删除角色。删除的角色将从分配给它的所有实体中撤消。
语法:
``` sql
DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
```
## DROP ROW POLICY {#drop-row-policy-statement}
删除行策略。 删除的行策略从分配到它的所有实体中撤消。
语法:
``` sql
DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name]
```
## DROP QUOTA {#drop-quota-statement}
Deletes a quota. The deleted quota is revoked from all the entities where it was assigned.
语法:
``` sql
DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
```
## DROP SETTINGS PROFILE {#drop-settings-profile-statement}
删除配置文件。 已删除的设置配置文件将从分配给它的所有实体中撤销。
语法:
``` sql
DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
```
## DROP VIEW {#drop-view}
删除视图。 视图也可以通过`DROP TABLE`命令删除，但`DROP VIEW`会检查`[db.]name`是否是一个视图。
语法:
``` sql
DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster]
```
[原始文章](https://clickhouse.tech/docs/zh/sql-reference/statements/drop/) <!--hide-->

42
docs/zh/sql-reference/statements/exchange.md Normal file
View File

@ -0,0 +1,42 @@
---
toc_priority: 49
toc_title: EXCHANGE
---
# EXCHANGE语法 {#exchange}
以原子方式交换两个表或字典的名称。
此任务也可以通过使用[RENAME](./rename.md)来完成，但在这种情况下操作不是原子的。
!!! note "注意"
`EXCHANGE`仅支持[Atomic](../../engines/database-engines/atomic.md)数据库引擎.
**语法**
```sql
EXCHANGE TABLES|DICTIONARIES [db0.]name_A AND [db1.]name_B
```
## EXCHANGE TABLES {#exchange_tables}
交换两个表的名称。
**语法**
```sql
EXCHANGE TABLES [db0.]table_A AND [db1.]table_B
```
## EXCHANGE DICTIONARIES {#exchange_dictionaries}
交换两个字典的名称。
**语法**
```sql
EXCHANGE DICTIONARIES [db0.]dict_A AND [db1.]dict_B
```
**参考**
- [Dictionaries](../../sql-reference/dictionaries/index.md)

61
docs/zh/sql-reference/statements/rename.md Normal file
View File

@ -0,0 +1,61 @@
---
toc_priority: 48
toc_title: RENAME
---
# RENAME语法 {#misc_operations-rename}
重命名数据库、表或字典。 可以在单个查询中重命名多个实体。
请注意，具有多个实体的`RENAME`查询是非原子操作。 要以原子方式交换实体名称，请使用[EXCHANGE](./exchange.md)语法.
!!! note "注意"
`RENAME`仅支持[Atomic](../../engines/database-engines/atomic.md)数据库引擎.
**语法**
```sql
RENAME DATABASE|TABLE|DICTIONARY name TO new_name [,...] [ON CLUSTER cluster]
```
## RENAME DATABASE {#misc_operations-rename_database}
重命名数据库.
**语法**
```sql
RENAME DATABASE atomic_database1 TO atomic_database2 [,...] [ON CLUSTER cluster]
```
## RENAME TABLE {#misc_operations-rename_table}
重命名一个或多个表
重命名表是一个轻量级的操作。 如果在`TO`之后传递一个不同的数据库，该表将被移动到这个数据库。 但是，包含数据库的目录必须位于同一文件系统中。 否则，返回错误。
如果在一个查询中重命名多个表，则该操作不是原子操作。 可能会部分执行，其他会话中可能会得到`Table ... does not exist ...`错误。
**语法**
``` sql
RENAME TABLE [db1.]name1 TO [db2.]name2 [,...] [ON CLUSTER cluster]
```
**示例**
```sql
RENAME TABLE table_A TO table_A_bak, table_B TO table_B_bak;
```
## RENAME DICTIONARY {#rename_dictionary}
重命名一个或多个词典。 此查询可用于在数据库之间移动字典。
**语法**
```sql
RENAME DICTIONARY [db0.]dict_A TO [db1.]dict_B [,...] [ON CLUSTER cluster]
```
**参考**
- [Dictionaries](../../sql-reference/dictionaries/index.md)

4
docs/zh/sql-reference/statements/watch.md Normal file
View File

@ -0,0 +1,4 @@
---
toc_priority: 53
toc_title: WATCH
---

20
programs/CMakeLists.txt
View File

@ -45,9 +45,9 @@ option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Li
${ENABLE_CLICKHOUSE_ALL})
# https://presentations.clickhouse.tech/matemarketing_2020/
option (ENABLE_CLICKHOUSE_GIT_IMPORT "A tool to analyze Git repositories"
${ENABLE_CLICKHOUSE_ALL})
option (ENABLE_CLICKHOUSE_GIT_IMPORT "A tool to analyze Git repositories" ${ENABLE_CLICKHOUSE_ALL})
option (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER "A tool to export table data files to be later put to a static files web server" ${ENABLE_CLICKHOUSE_ALL})
option (ENABLE_CLICKHOUSE_KEEPER "ClickHouse alternative to ZooKeeper" ${ENABLE_CLICKHOUSE_ALL})
@ -227,6 +227,7 @@ add_subdirectory (obfuscator)
add_subdirectory (install)
add_subdirectory (git-import)
add_subdirectory (bash-completion)
add_subdirectory (static-files-disk-uploader)
if (ENABLE_CLICKHOUSE_KEEPER)
add_subdirectory (keeper)
@ -258,7 +259,8 @@ if (CLICKHOUSE_ONE_SHARED)
${CLICKHOUSE_GIT_IMPORT_SOURCES}
${CLICKHOUSE_ODBC_BRIDGE_SOURCES}
${CLICKHOUSE_KEEPER_SOURCES}
${CLICKHOUSE_KEEPER_CONVERTER_SOURCES})
${CLICKHOUSE_KEEPER_CONVERTER_SOURCES}
${CLICKHOUSE_STATIC_FILES_DISK_UPLOADER_SOURCES})
target_link_libraries(clickhouse-lib
${CLICKHOUSE_SERVER_LINK}
@ -273,7 +275,8 @@ if (CLICKHOUSE_ONE_SHARED)
${CLICKHOUSE_GIT_IMPORT_LINK}
${CLICKHOUSE_ODBC_BRIDGE_LINK}
${CLICKHOUSE_KEEPER_LINK}
${CLICKHOUSE_KEEPER_CONVERTER_LINK})
${CLICKHOUSE_KEEPER_CONVERTER_LINK}
${CLICKHOUSE_STATIC_FILES_DISK_UPLOADER_LINK})
target_include_directories(clickhouse-lib
${CLICKHOUSE_SERVER_INCLUDE}
@ -306,6 +309,7 @@ if (CLICKHOUSE_SPLIT_BINARY)
clickhouse-obfuscator
clickhouse-git-import
clickhouse-copier
clickhouse-static-files-disk-uploader
)
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
@ -371,6 +375,9 @@ else ()
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
clickhouse_target_link_split_lib(clickhouse git-import)
endif ()
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
clickhouse_target_link_split_lib(clickhouse static-files-disk-uploader)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER)
clickhouse_target_link_split_lib(clickhouse keeper)
endif()
@ -432,6 +439,11 @@ else ()
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import)
endif ()
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
add_custom_target (clickhouse-static-files-disk-uploader ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-static-files-disk-uploader DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-static-files-disk-uploader" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-static-files-disk-uploader)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER)
add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)

5
programs/client/Client.cpp
View File

@ -2285,7 +2285,10 @@ private:
if (!pager.empty())
{
signal(SIGPIPE, SIG_IGN);
pager_cmd = ShellCommand::execute(pager, true);
ShellCommand::Config config(pager);
config.pipe_stdin_only = true;
pager_cmd = ShellCommand::execute(config);
out_buf = &pager_cmd->in;
}
else

1
programs/config_tools.h.in
View File

@ -18,3 +18,4 @@
#cmakedefine01 ENABLE_CLICKHOUSE_LIBRARY_BRIDGE
#cmakedefine01 ENABLE_CLICKHOUSE_KEEPER
#cmakedefine01 ENABLE_CLICKHOUSE_KEEPER_CONVERTER
#cmakedefine01 ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER

8
programs/keeper/Keeper.cpp
View File

@ -300,9 +300,9 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
if (config().has("keeper_server.storage_path"))
path = config().getString("keeper_server.storage_path");
else if (config().has("keeper_server.log_storage_path"))
path = config().getString("keeper_server.log_storage_path");
path = std::filesystem::path(config().getString("keeper_server.log_storage_path")).parent_path();
else if (config().has("keeper_server.snapshot_storage_path"))
path = config().getString("keeper_server.snapshot_storage_path");
path = std::filesystem::path(config().getString("keeper_server.snapshot_storage_path")).parent_path();
else
path = std::filesystem::path{KEEPER_DEFAULT_PATH};
@ -359,7 +359,7 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
auto servers = std::make_shared<std::vector<ProtocolServerAdapter>>();
/// Initialize test keeper RAFT. Do nothing if no nu_keeper_server in config.
global_context->initializeKeeperStorageDispatcher();
global_context->initializeKeeperDispatcher();
for (const auto & listen_host : listen_hosts)
{
/// TCP Keeper
@ -428,7 +428,7 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
else
LOG_INFO(log, "Closed connections to Keeper.");
global_context->shutdownKeeperStorageDispatcher();
global_context->shutdownKeeperDispatcher();
/// Wait server pool to avoid use-after-free of destroyed context in the handlers
server_pool.joinAll();

3
programs/local/LocalServer.cpp
View File

@ -271,6 +271,9 @@ try
/// Load global settings from default_profile and system_profile.
global_context->setDefaultProfiles(config());
/// We load temporary database first, because projections need it.
DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
/** Init dummy default DB
* NOTE: We force using isolated default database to avoid conflicts with default database from server environment
* Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory;

6
programs/main.cpp
View File

@ -62,6 +62,9 @@ int mainEntryClickHouseKeeper(int argc, char ** argv);
#if ENABLE_CLICKHOUSE_KEEPER
int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
#endif
#if ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER
int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv);
#endif
#if ENABLE_CLICKHOUSE_INSTALL
int mainEntryClickHouseInstall(int argc, char ** argv);
int mainEntryClickHouseStart(int argc, char ** argv);
@ -131,6 +134,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
{"stop", mainEntryClickHouseStop},
{"status", mainEntryClickHouseStatus},
{"restart", mainEntryClickHouseRestart},
#endif
#if ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER
{"static-files-disk-uploader", mainEntryClickHouseStaticFilesDiskUploader},
#endif
{"hash-binary", mainEntryClickHouseHashBinary},
};

26
programs/odbc-bridge/ODBCBlockOutputStream.cpp
View File

@ -5,40 +5,16 @@
#include <Core/Field.h>
#include <common/LocalDate.h>
#include <common/LocalDateTime.h>
#include <Parsers/ASTInsertQuery.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTIdentifier.h>
#include "getIdentifierQuote.h"
#include <IO/WriteHelpers.h>
#include <IO/Operators.h>
#include <Formats/FormatFactory.h>
#include <Parsers/getInsertQuery.h>
namespace DB
{
namespace
{
using ValueType = ExternalResultDescription::ValueType;
std::string getInsertQuery(const std::string & db_name, const std::string & table_name, const ColumnsWithTypeAndName & columns, IdentifierQuotingStyle quoting)
{
ASTInsertQuery query;
query.table_id.database_name = db_name;
query.table_id.table_name = table_name;
query.columns = std::make_shared<ASTExpressionList>(',');
query.children.push_back(query.columns);
for (const auto & column : columns)
query.columns->children.emplace_back(std::make_shared<ASTIdentifier>(column.name));
WriteBufferFromOwnString buf;
IAST::FormatSettings settings(buf, true);
settings.always_quote_identifiers = true;
settings.identifier_quoting_style = quoting;
query.IAST::format(settings);
return buf.str();
}
}
ODBCBlockOutputStream::ODBCBlockOutputStream(nanodbc::ConnectionHolderPtr connection_holder_,
const std::string & remote_database_name_,

1
programs/odbc-bridge/ODBCBlockOutputStream.h
View File

@ -13,6 +13,7 @@ namespace DB
class ODBCBlockOutputStream : public IBlockOutputStream
{
using ValueType = ExternalResultDescription::ValueType;
public:
ODBCBlockOutputStream(

3
programs/odbc-bridge/ODBCConnectionFactory.h
View File

@ -6,6 +6,7 @@
#include <common/BorrowedObjectPool.h>
#include <unordered_map>
namespace DB
{
namespace ErrorCodes
@ -57,7 +58,7 @@ public:
private:
PoolPtr pool;
ConnectionPtr connection;
const String & connection_string;
String connection_string;
};
using ConnectionHolderPtr = std::shared_ptr<ConnectionHolder>;

16
programs/server/Server.cpp
View File

@ -766,6 +766,12 @@ if (ThreadFuzzer::instance().isEffective())
fs::create_directories(dictionaries_lib_path);
}
{
std::string user_scripts_path = config().getString("user_scripts_path", path / "user_scripts/");
global_context->setUserScriptsPath(user_scripts_path);
fs::create_directories(user_scripts_path);
}
/// top_level_domains_lists
{
const std::string & top_level_domains_path = config().getString("top_level_domains_path", path / "top_level_domains/");
@ -996,7 +1002,7 @@ if (ThreadFuzzer::instance().isEffective())
{
#if USE_NURAFT
/// Initialize test keeper RAFT. Do nothing if no nu_keeper_server in config.
global_context->initializeKeeperStorageDispatcher();
global_context->initializeKeeperDispatcher();
for (const auto & listen_host : listen_hosts)
{
/// TCP Keeper
@ -1079,7 +1085,7 @@ if (ThreadFuzzer::instance().isEffective())
else
LOG_INFO(log, "Closed connections to servers for tables.");
global_context->shutdownKeeperStorageDispatcher();
global_context->shutdownKeeperDispatcher();
}
/// Wait server pool to avoid use-after-free of destroyed context in the handlers
@ -1116,15 +1122,15 @@ if (ThreadFuzzer::instance().isEffective())
try
{
auto & database_catalog = DatabaseCatalog::instance();
/// We load temporary database first, because projections need it.
database_catalog.initializeAndLoadTemporaryDatabase();
loadMetadataSystem(global_context);
/// After attaching system databases we can initialize system log.
global_context->initializeSystemLogs();
global_context->setSystemZooKeeperLogAfterInitializationIfNeeded();
auto & database_catalog = DatabaseCatalog::instance();
/// After the system database is created, attach virtual system tables (in addition to query_log and part_log)
attachSystemTablesServer(*database_catalog.getSystemDatabase(), has_zookeeper);
/// We load temporary database first, because projections need it.
database_catalog.initializeAndLoadTemporaryDatabase();
/// Then, load remaining databases
loadMetadata(global_context, default_database);
database_catalog.loadDatabases();

9
programs/static-files-disk-uploader/CMakeLists.txt Normal file
View File

@ -0,0 +1,9 @@
set (CLICKHOUSE_STATIC_FILES_DISK_UPLOADER_SOURCES static-files-disk-uploader.cpp)
set (CLICKHOUSE_STATIC_FILES_DISK_UPLOADER_LINK
PRIVATE
boost::program_options
dbms
)
clickhouse_program_add(static-files-disk-uploader)

2
programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp Normal file
View File

@ -0,0 +1,2 @@
int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseStaticFilesDiskUploader(argc_, argv_); }

162
programs/static-files-disk-uploader/static-files-disk-uploader.cpp Normal file
View File

@ -0,0 +1,162 @@
#include <Common/Exception.h>
#include <Common/TerminalSize.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteHelpers.h>
#include <IO/WriteBufferFromHTTP.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/copyData.h>
#include <IO/createReadBufferFromFileBase.h>
#include <boost/program_options.hpp>
#include <re2/re2.h>
#include <filesystem>
namespace fs = std::filesystem;
#define UUID_PATTERN "[\\w]{8}-[\\w]{4}-[\\w]{4}-[\\w]{4}-[\\w]{12}"
#define EXTRACT_UUID_PATTERN fmt::format(".*/({})/.*", UUID_PATTERN)
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
/*
* A tool to collect table data files on local fs as is (into current directory or into path from --output-dir option).
* If test-mode option is added, files will be put by given url via PUT request.
*/
void processTableFiles(const fs::path & path, const String & files_prefix, String uuid,
WriteBuffer & metadata_buf, std::function<std::shared_ptr<WriteBuffer>(const String &)> create_dst_buf)
{
fs::directory_iterator dir_end;
auto process_file = [&](const String & file_name, const String & file_path)
{
auto remote_file_name = files_prefix + "-" + uuid + "-" + file_name;
writeText(remote_file_name, metadata_buf);
writeChar('\t', metadata_buf);
writeIntText(fs::file_size(file_path), metadata_buf);
writeChar('\n', metadata_buf);
auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
auto dst_buf = create_dst_buf(remote_file_name);
copyData(*src_buf, *dst_buf);
dst_buf->next();
dst_buf->finalize();
};
for (fs::directory_iterator dir_it(path); dir_it != dir_end; ++dir_it)
{
if (dir_it->is_directory())
{
fs::directory_iterator files_end;
for (fs::directory_iterator file_it(dir_it->path()); file_it != files_end; ++file_it)
process_file(dir_it->path().filename().string() + "-" + file_it->path().filename().string(), file_it->path());
}
else
{
process_file(dir_it->path().filename(), dir_it->path());
}
}
}
}
int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv)
try
{
using namespace DB;
namespace po = boost::program_options;
po::options_description description("Allowed options", getTerminalWidth());
description.add_options()
("help,h", "produce help message")
("metadata-path", po::value<std::string>(), "Metadata path (select data_paths from system.tables where name='table_name'")
("test-mode", "Use test mode, which will put data on given url via PUT")
("url", po::value<std::string>(), "Web server url for test mode")
("output-dir", po::value<std::string>(), "Directory to put files in non-test mode")
("files-prefix", po::value<std::string>(), "Prefix for stored files");
po::parsed_options parsed = po::command_line_parser(argc, argv).options(description).run();
po::variables_map options;
po::store(parsed, options);
po::notify(options);
if (options.empty() || options.count("help"))
{
std::cout << description << std::endl;
exit(0);
}
String url, metadata_path, files_prefix;
if (options.count("metadata-path"))
metadata_path = options["metadata-path"].as<std::string>();
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "No metadata-path option passed");
if (options.count("files-prefix"))
files_prefix = options["files-prefix"].as<std::string>();
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "No files-prefix option passed");
fs::path fs_path = fs::weakly_canonical(metadata_path);
if (!fs::exists(fs_path))
{
std::cerr << fmt::format("Data path ({}) does not exist", fs_path.string());
return 1;
}
String uuid;
if (!RE2::Extract(metadata_path, EXTRACT_UUID_PATTERN, "\\1", &uuid))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot extract uuid for: {}", metadata_path);
std::shared_ptr<WriteBuffer> metadata_buf;
std::function<std::shared_ptr<WriteBuffer>(const String &)> create_dst_buf;
String root_path;
if (options.count("test-mode"))
{
if (options.count("url"))
url = options["url"].as<std::string>();
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "No url option passed for test mode");
metadata_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(fs::path(url) / (".index-" + uuid)), Poco::Net::HTTPRequest::HTTP_PUT);
create_dst_buf = [&](const String & remote_file_name)
{
return std::make_shared<WriteBufferFromHTTP>(Poco::URI(fs::path(url) / remote_file_name), Poco::Net::HTTPRequest::HTTP_PUT);
};
}
else
{
if (options.count("output-dir"))
root_path = options["output-dir"].as<std::string>();
else
root_path = fs::current_path();
metadata_buf = std::make_shared<WriteBufferFromFile>(fs::path(root_path) / (".index-" + uuid));
create_dst_buf = [&](const String & remote_file_name)
{
return std::make_shared<WriteBufferFromFile>(fs::path(root_path) / remote_file_name);
};
}
processTableFiles(fs_path, files_prefix, uuid, *metadata_buf, create_dst_buf);
metadata_buf->next();
metadata_buf->finalize();
return 0;
}
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(false);
return 1;
}

1
programs/ya.make
View File

@ -23,6 +23,7 @@ SRCS(
client/QueryFuzzer.cpp
client/ConnectionParameters.cpp
client/Suggest.cpp
client/TestHint.cpp
extract-from-config/ExtractFromConfig.cpp
server/Server.cpp
server/MetricsTransmitter.cpp

2
src/Access/ya.make.in
View File

@ -8,7 +8,7 @@ PEERDIR(
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | sed 's/^\.\// /' | sort ?>
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

56
src/AggregateFunctions/AggregateFunctionIf.cpp
View File

@ -2,7 +2,6 @@
#include <AggregateFunctions/AggregateFunctionIf.h>
#include "AggregateFunctionNull.h"
namespace DB
{
@ -46,6 +45,29 @@ public:
}
};
/** Given an array of flags, checks if it's all zeros
* When the buffer is all zeros, this is slightly faster than doing a memcmp since doesn't require allocating memory
* When the buffer has values, this is much faster since it avoids visiting all memory (and the allocation and function calls)
*/
static bool ALWAYS_INLINE inline is_all_zeros(const UInt8 * flags, size_t size)
{
size_t unroll_size = size - size % 8;
size_t i = 0;
while (i < unroll_size)
{
UInt64 v = *reinterpret_cast<const UInt64 *>(&flags[i]);
if (v)
return false;
i += 8;
}
for (; i < size; i++)
if (flags[i])
return false;
return true;
}
/** There are two cases: for single argument and variadic.
* Code for single argument is much more efficient.
*/
@ -112,6 +134,38 @@ public:
}
}
void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t) const override
{
const ColumnNullable * column = assert_cast<const ColumnNullable *>(columns[0]);
const UInt8 * null_map = column->getNullMapData().data();
const IColumn * columns_param[] = {&column->getNestedColumn()};
const IColumn * filter_column = columns[num_arguments - 1];
if (const ColumnNullable * nullable_column = typeid_cast<const ColumnNullable *>(filter_column))
filter_column = nullable_column->getNestedColumnPtr().get();
if constexpr (result_is_nullable)
{
/// We need to check if there is work to do as otherwise setting the flag would be a mistake,
/// it would mean that the return value would be the default value of the nested type instead of NULL
if (is_all_zeros(assert_cast<const ColumnUInt8 *>(filter_column)->getData().data(), batch_size))
return;
}
/// Combine the 2 flag arrays so we can call a simplified version (one check vs 2)
/// Note that now the null map will contain 0 if not null and not filtered, or 1 for null or filtered (or both)
const auto * filter_flags = assert_cast<const ColumnUInt8 *>(filter_column)->getData().data();
auto final_nulls = std::make_unique<UInt8[]>(batch_size);
for (size_t i = 0; i < batch_size; ++i)
final_nulls[i] = (!!null_map[i]) | (!filter_flags[i]);
this->nested_function->addBatchSinglePlaceNotNull(
batch_size, this->nestedPlace(place), columns_param, final_nulls.get(), arena, -1);
if constexpr (result_is_nullable)
if (!memoryIsByte(null_map, batch_size, 1))
this->setFlag(place);
}
#if USE_EMBEDDED_COMPILER
void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const DataTypes & arguments_types, const std::vector<llvm::Value *> & argument_values) const override

112
src/AggregateFunctions/AggregateFunctionSequenceMatch.h
View File

@ -48,6 +48,8 @@ struct AggregateFunctionSequenceMatchData final
bool sorted = true;
PODArrayWithStackMemory<TimestampEvents, 64> events_list;
/// sequenceMatch conditions met at least once in events_list
std::bitset<max_events> conditions_met;
void add(const Timestamp timestamp, const Events & events)
{
@ -56,6 +58,7 @@ struct AggregateFunctionSequenceMatchData final
{
events_list.emplace_back(timestamp, events);
sorted = false;
conditions_met |= events;
}
}
@ -64,29 +67,9 @@ struct AggregateFunctionSequenceMatchData final
if (other.events_list.empty())
return;
const auto size = events_list.size();
events_list.insert(std::begin(other.events_list), std::end(other.events_list));
/// either sort whole container or do so partially merging ranges afterwards
if (!sorted && !other.sorted)
std::sort(std::begin(events_list), std::end(events_list), Comparator{});
else
{
const auto begin = std::begin(events_list);
const auto middle = std::next(begin, size);
const auto end = std::end(events_list);
if (!sorted)
std::sort(begin, middle, Comparator{});
if (!other.sorted)
std::sort(middle, end, Comparator{});
std::inplace_merge(begin, middle, end, Comparator{});
}
sorted = true;
sorted = false;
conditions_met |= other.conditions_met;
}
void sort()
@ -290,6 +273,7 @@ private:
dfa_states.back().transition = DFATransition::SpecificEvent;
dfa_states.back().event = event_number - 1;
dfa_states.emplace_back();
conditions_in_pattern.set(event_number - 1);
}
if (!match(")"))
@ -518,6 +502,64 @@ protected:
return action_it == action_end;
}
/// Splits the pattern into deterministic parts separated by non-deterministic fragments
/// (time constraints and Kleene stars), and tries to match the deterministic parts in their specified order,
/// ignoring the non-deterministic fragments.
/// This function can quickly check that a full match is not possible if some deterministic fragment is missing.
template <typename EventEntry>
bool couldMatchDeterministicParts(const EventEntry events_begin, const EventEntry events_end, bool limit_iterations = true) const
{
size_t events_processed = 0;
auto events_it = events_begin;
const auto actions_end = std::end(actions);
auto actions_it = std::begin(actions);
auto det_part_begin = actions_it;
auto match_deterministic_part = [&events_it, events_end, &events_processed, det_part_begin, actions_it, limit_iterations]()
{
auto events_it_init = events_it;
auto det_part_it = det_part_begin;
while (det_part_it != actions_it && events_it != events_end)
{
/// matching any event
if (det_part_it->type == PatternActionType::AnyEvent)
++events_it, ++det_part_it;
/// matching specific event
else
{
if (events_it->second.test(det_part_it->extra))
++events_it, ++det_part_it;
/// abandon current matching, try to match the deterministic fragment further in the list
else
{
events_it = ++events_it_init;
det_part_it = det_part_begin;
}
}
if (limit_iterations && ++events_processed > sequence_match_max_iterations)
throw Exception{"Pattern application proves too difficult, exceeding max iterations (" + toString(sequence_match_max_iterations) + ")",
ErrorCodes::TOO_SLOW};
}
return det_part_it == actions_it;
};
for (; actions_it != actions_end; ++actions_it)
if (actions_it->type != PatternActionType::SpecificEvent && actions_it->type != PatternActionType::AnyEvent)
{
if (!match_deterministic_part())
return false;
det_part_begin = std::next(actions_it);
}
return match_deterministic_part();
}
private:
enum class DFATransition : char
{
@ -558,6 +600,8 @@ private:
protected:
/// `True` if the parsed pattern contains time assertions (?t...), `false` otherwise.
bool pattern_has_time;
/// sequenceMatch conditions met at least once in the pattern
std::bitset<max_events> conditions_in_pattern;
private:
std::string pattern;
@ -584,6 +628,12 @@ public:
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
{
auto & output = assert_cast<ColumnUInt8 &>(to).getData();
if ((this->conditions_in_pattern & this->data(place).conditions_met) != this->conditions_in_pattern)
{
output.push_back(false);
return;
}
this->data(place).sort();
const auto & data_ref = this->data(place);
@ -592,8 +642,10 @@ public:
const auto events_end = std::end(data_ref.events_list);
auto events_it = events_begin;
bool match = this->pattern_has_time ? this->backtrackingMatch(events_it, events_end) : this->dfaMatch(events_it, events_end);
assert_cast<ColumnUInt8 &>(to).getData().push_back(match);
bool match = (this->pattern_has_time ?
(this->couldMatchDeterministicParts(events_begin, events_end) && this->backtrackingMatch(events_it, events_end)) :
this->dfaMatch(events_it, events_end));
output.push_back(match);
}
};
@ -614,8 +666,14 @@ public:
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
{
auto & output = assert_cast<ColumnUInt64 &>(to).getData();
if ((this->conditions_in_pattern & this->data(place).conditions_met) != this->conditions_in_pattern)
{
output.push_back(0);
return;
}
this->data(place).sort();
assert_cast<ColumnUInt64 &>(to).getData().push_back(count(place));
output.push_back(count(place));
}
private:
@ -628,8 +686,12 @@ private:
auto events_it = events_begin;
size_t count = 0;
// check if there is a chance of matching the sequence at least once
if (this->couldMatchDeterministicParts(events_begin, events_end))
{
while (events_it != events_end && this->backtrackingMatch(events_it, events_end))
++count;
}
return count;
}

78
src/AggregateFunctions/AggregateFunctionSum.h
View File

@ -1,5 +1,6 @@
#pragma once
#include <memory>
#include <experimental/type_traits>
#include <type_traits>
@ -96,8 +97,9 @@ struct AggregateFunctionSumData
Impl::add(sum, local_sum);
}
template <typename Value>
void NO_SANITIZE_UNDEFINED NO_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count)
template <typename Value, bool add_if_zero>
void NO_SANITIZE_UNDEFINED NO_INLINE
addManyConditional_internal(const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t count)
{
const auto * end = ptr + count;
@ -110,10 +112,10 @@ struct AggregateFunctionSumData
T local_sum{};
while (ptr < end)
{
T multiplier = !*null_map;
T multiplier = !*condition_map == add_if_zero;
Impl::add(local_sum, *ptr * multiplier);
++ptr;
++null_map;
++condition_map;
}
Impl::add(sum, local_sum);
return;
@ -130,13 +132,13 @@ struct AggregateFunctionSumData
{
for (size_t i = 0; i < unroll_count; ++i)
{
if (!null_map[i])
if (!condition_map[i] == add_if_zero)
{
Impl::add(partial_sums[i], ptr[i]);
}
}
ptr += unroll_count;
null_map += unroll_count;
condition_map += unroll_count;
}
for (size_t i = 0; i < unroll_count; ++i)
@ -146,14 +148,26 @@ struct AggregateFunctionSumData
T local_sum{};
while (ptr < end)
{
if (!*null_map)
if (!*condition_map == add_if_zero)
Impl::add(local_sum, *ptr);
++ptr;
++null_map;
++condition_map;
}
Impl::add(sum, local_sum);
}
template <typename Value>
void ALWAYS_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count)
{
return addManyConditional_internal<Value, true>(ptr, null_map, count);
}
template <typename Value>
void ALWAYS_INLINE addManyConditional(const Value * __restrict ptr, const UInt8 * __restrict cond_map, size_t count)
{
return addManyConditional_internal<Value, false>(ptr, cond_map, count);
}
void NO_SANITIZE_UNDEFINED merge(const AggregateFunctionSumData & rhs)
{
Impl::add(sum, rhs.sum);
@ -229,8 +243,8 @@ struct AggregateFunctionSumKahanData
}
}
template <typename Value>
void NO_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count)
template <typename Value, bool add_if_zero>
void NO_INLINE addManyConditional_internal(const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t count)
{
constexpr size_t unroll_count = 4;
T partial_sums[unroll_count]{};
@ -242,10 +256,10 @@ struct AggregateFunctionSumKahanData
while (ptr < unrolled_end)
{
for (size_t i = 0; i < unroll_count; ++i)
if (!null_map[i])
if ((!condition_map[i]) == add_if_zero)
addImpl(ptr[i], partial_sums[i], partial_compensations[i]);
ptr += unroll_count;
null_map += unroll_count;
condition_map += unroll_count;
}
for (size_t i = 0; i < unroll_count; ++i)
@ -253,13 +267,25 @@ struct AggregateFunctionSumKahanData
while (ptr < end)
{
if (!*null_map)
if ((!*condition_map) == add_if_zero)
addImpl(*ptr, sum, compensation);
++ptr;
++null_map;
++condition_map;
}
}
template <typename Value>
void ALWAYS_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count)
{
return addManyConditional_internal<Value, true>(ptr, null_map, count);
}
template <typename Value>
void ALWAYS_INLINE addManyConditional(const Value * __restrict ptr, const UInt8 * __restrict cond_map, size_t count)
{
return addManyConditional_internal<Value, false>(ptr, cond_map, count);
}
void ALWAYS_INLINE mergeImpl(T & to_sum, T & to_compensation, T from_sum, T from_compensation)
{
auto raw_sum = to_sum + from_sum;
@ -352,40 +378,38 @@ public:
this->data(place).add(column.getData()[row_num]);
}
/// Vectorized version when there is no GROUP BY keys.
void addBatchSinglePlace(
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos) const override
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, ssize_t if_argument_pos) const override
{
const auto & column = assert_cast<const ColVecType &>(*columns[0]);
if (if_argument_pos >= 0)
{
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
for (size_t i = 0; i < batch_size; ++i)
{
if (flags[i])
add(place, columns, i, arena);
}
this->data(place).addManyConditional(column.getData().data(), flags.data(), batch_size);
}
else
{
const auto & column = assert_cast<const ColVecType &>(*columns[0]);
this->data(place).addMany(column.getData().data(), batch_size);
}
}
void addBatchSinglePlaceNotNull(
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena, ssize_t if_argument_pos)
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena *, ssize_t if_argument_pos)
const override
{
const auto & column = assert_cast<const ColVecType &>(*columns[0]);
if (if_argument_pos >= 0)
{
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
/// Merge the 2 sets of flags (null and if) into a single one. This allows us to use parallelizable sums when available
const auto * if_flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData().data();
auto final_flags = std::make_unique<UInt8[]>(batch_size);
for (size_t i = 0; i < batch_size; ++i)
if (!null_map[i] && flags[i])
add(place, columns, i, arena);
final_flags[i] = (!null_map[i]) & if_flags[i];
this->data(place).addManyConditional(column.getData().data(), final_flags.get(), batch_size);
}
else
{
const auto & column = assert_cast<const ColVecType &>(*columns[0]);
this->data(place).addManyNotNull(column.getData().data(), null_map, batch_size);
}
}

1
src/AggregateFunctions/IAggregateFunction.h
View File

@ -190,6 +190,7 @@ public:
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const = 0;
/** The same for single place when need to aggregate only filtered data.
* Instead of using an if-column, the condition is combined inside the null_map
*/
virtual void addBatchSinglePlaceNotNull(
size_t batch_size,

1
src/AggregateFunctions/ya.make
View File

@ -48,6 +48,7 @@ SRCS(
AggregateFunctionSequenceNextNode.cpp
AggregateFunctionSimpleLinearRegression.cpp
AggregateFunctionSimpleState.cpp
AggregateFunctionSingleValueOrNull.cpp
AggregateFunctionState.cpp
AggregateFunctionStatistics.cpp
AggregateFunctionStatisticsSimple.cpp

2
src/AggregateFunctions/ya.make.in
View File

@ -8,7 +8,7 @@ PEERDIR(
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F GroupBitmap | sed 's/^\.\// /' | sort ?>
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | grep -v -F GroupBitmap | sed 's/^\.\// /' | sort ?>
)
END()

2
src/Backups/BackupEntryFromImmutableFile.cpp
View File

@ -41,7 +41,7 @@ std::unique_ptr<ReadBuffer> BackupEntryFromImmutableFile::getReadBuffer() const
if (disk)
return disk->readFile(file_path);
else
return createReadBufferFromFileBase(file_path, 0, 0, 0, nullptr);
return createReadBufferFromFileBase(file_path, {}, 0);
}
}

2
src/Backups/BackupEntryFromSmallFile.cpp
View File

@ -10,7 +10,7 @@ namespace
{
String readFile(const String & file_path)
{
auto buf = createReadBufferFromFileBase(file_path, 0, 0, 0, nullptr);
auto buf = createReadBufferFromFileBase(file_path, {}, 0);
String s;
readStringUntilEOF(s, *buf);
return s;

27
src/Backups/ya.make Normal file
View File

@ -0,0 +1,27 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
BackupEntryConcat.cpp
BackupEntryFromAppendOnlyFile.cpp
BackupEntryFromImmutableFile.cpp
BackupEntryFromMemory.cpp
BackupEntryFromSmallFile.cpp
BackupFactory.cpp
BackupInDirectory.cpp
BackupRenamingConfig.cpp
BackupSettings.cpp
BackupUtils.cpp
hasCompatibleDataToRestoreTable.cpp
renameInCreateQuery.cpp
)
END()

14
src/Backups/ya.make.in Normal file
View File

@ -0,0 +1,14 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

6
src/Bridge/IBridgeHelper.cpp
View File

@ -113,7 +113,11 @@ std::unique_ptr<ShellCommand> IBridgeHelper::startBridgeCommand()
LOG_TRACE(getLog(), "Starting {}", serviceAlias());
return ShellCommand::executeDirect(path.string(), cmd_args, ShellCommandDestructorStrategy(true));
ShellCommand::Config command_config(path.string());
command_config.arguments = cmd_args;
command_config.terminate_in_destructor_strategy = ShellCommand::DestructorStrategy(true);
return ShellCommand::executeDirect(command_config);
}
}

2
src/Bridge/ya.make.in
View File

@ -8,7 +8,7 @@ PEERDIR(
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | sed 's/^\.\// /' | sort ?>
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

2
src/Client/ya.make.in
View File

@ -9,7 +9,7 @@ PEERDIR(
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | sed 's/^\.\// /' | sort ?>
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

2
src/Columns/ColumnUnique.h
View File

@ -301,7 +301,7 @@ size_t ColumnUnique<ColumnType>::getNullValueIndex() const
template <typename ColumnType>
size_t ColumnUnique<ColumnType>::uniqueInsert(const Field & x)
{
if (x.getType() == Field::Types::Null)
if (x.isNull())
return getNullValueIndex();
if (valuesHaveFixedSize())

2
src/Columns/ColumnsCommon.cpp
View File

@ -85,7 +85,7 @@ size_t countBytesInFilterWithNull(const IColumn::Filter & filt, const UInt8 * nu
/// TODO Add duff device for tail?
#endif
for (; pos < end; ++pos)
for (; pos < end; ++pos, ++pos2)
count += (*pos & ~*pos2) != 0;
return count;

2
src/Columns/ya.make.in
View File

@ -17,7 +17,7 @@ PEERDIR(
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | sed 's/^\.\// /' | sort ?>
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

120
src/Common/ConcurrentBoundedQueue.h
View File

@ -2,11 +2,21 @@
#include <queue>
#include <type_traits>
#include <atomic>
#include <Poco/Mutex.h>
#include <Poco/Semaphore.h>
#include <common/MoveOrCopyIfThrow.h>
#include <Common/Exception.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
}
/** A very simple thread-safe queue of limited size.
* If you try to pop an item from an empty queue, the thread is blocked until the queue becomes nonempty.
@ -17,9 +27,41 @@ class ConcurrentBoundedQueue
{
private:
std::queue<T> queue;
Poco::FastMutex mutex;
mutable Poco::FastMutex mutex;
Poco::Semaphore fill_count;
Poco::Semaphore empty_count;
std::atomic_bool closed = false;
template <typename... Args>
bool tryEmplaceImpl(Args &&... args)
{
bool emplaced = true;
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
if (closed)
emplaced = false;
else
queue.emplace(std::forward<Args>(args)...);
}
if (emplaced)
fill_count.set();
else
empty_count.set();
return emplaced;
}
void popImpl(T & x)
{
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
detail::moveOrCopyIfThrow(std::move(queue.front()), x);
queue.pop();
}
empty_count.set();
}
public:
explicit ConcurrentBoundedQueue(size_t max_fill)
@ -30,91 +72,75 @@ public:
void push(const T & x)
{
empty_count.wait();
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
queue.push(x);
}
fill_count.set();
if (!tryEmplaceImpl(x))
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "tryPush/tryEmplace must be used with close()");
}
template <typename... Args>
void emplace(Args &&... args)
{
empty_count.wait();
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
queue.emplace(std::forward<Args>(args)...);
}
fill_count.set();
if (!tryEmplaceImpl(std::forward<Args>(args)...))
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "tryPush/tryEmplace must be used with close()");
}
void pop(T & x)
{
fill_count.wait();
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
detail::moveOrCopyIfThrow(std::move(queue.front()), x);
queue.pop();
}
empty_count.set();
popImpl(x);
}
bool tryPush(const T & x, UInt64 milliseconds = 0)
{
if (empty_count.tryWait(milliseconds))
{
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
queue.push(x);
}
fill_count.set();
return true;
}
if (!empty_count.tryWait(milliseconds))
return false;
return tryEmplaceImpl(x);
}
template <typename... Args>
bool tryEmplace(UInt64 milliseconds, Args &&... args)
{
if (empty_count.tryWait(milliseconds))
{
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
queue.emplace(std::forward<Args>(args)...);
}
fill_count.set();
return true;
}
if (!empty_count.tryWait(milliseconds))
return false;
return tryEmplaceImpl(std::forward<Args>(args)...);
}
bool tryPop(T & x, UInt64 milliseconds = 0)
{
if (fill_count.tryWait(milliseconds))
{
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
detail::moveOrCopyIfThrow(std::move(queue.front()), x);
queue.pop();
}
empty_count.set();
if (!fill_count.tryWait(milliseconds))
return false;
popImpl(x);
return true;
}
return false;
}
size_t size()
size_t size() const
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
return queue.size();
}
size_t empty()
size_t empty() const
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
return queue.empty();
}
/// Forbids to push new elements to queue.
/// Returns false if queue was not closed before call, returns true if queue was already closed.
bool close()
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
return closed.exchange(true);
}
bool isClosed() const
{
return closed.load();
}
void clear()
{
while (fill_count.tryWait(0))

1
src/Common/CurrentMetrics.cpp
View File

@ -76,6 +76,7 @@
M(ActiveAsyncDrainedConnections, "Number of active connections drained asynchronously.") \
M(SyncDrainedConnections, "Number of connections drained synchronously.") \
M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \
M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \
namespace CurrentMetrics
{

15
src/Common/ErrorCodes.cpp
View File

@ -577,14 +577,15 @@
M(606, BACKUP_IS_EMPTY) \
M(607, BACKUP_ELEMENT_DUPLICATE) \
M(608, CANNOT_RESTORE_TABLE) \
M(609, FUNCTION_ALREADY_EXISTS) \
M(610, CANNOT_DROP_SYSTEM_FUNCTION) \
M(611, CANNOT_CREATE_RECURSIVE_FUNCTION) \
M(612, OBJECT_ALREADY_STORED_ON_DISK) \
M(613, OBJECT_WAS_NOT_STORED_ON_DISK) \
M(614, POSTGRESQL_CONNECTION_FAILURE) \
M(615, CANNOT_ADVISE) \
M(616, UNKNOWN_READ_METHOD) \
\
M(598, FUNCTION_ALREADY_EXISTS) \
M(599, CANNOT_DROP_SYSTEM_FUNCTION) \
M(600, CANNOT_CREATE_RECURSIVE_FUNCTION) \
M(601, OBJECT_ALREADY_STORED_ON_DISK) \
M(602, OBJECT_WAS_NOT_STORED_ON_DISK) \
\
M(998, POSTGRESQL_CONNECTION_FAILURE) \
M(999, KEEPER_EXCEPTION) \
M(1000, POCO_EXCEPTION) \
M(1001, STD_EXCEPTION) \

9
src/Common/ProfileEvents.cpp
View File

@ -251,6 +251,15 @@
\
M(SleepFunctionCalls, "Number of times a sleep function (sleep, sleepEachRow) has been called.") \
M(SleepFunctionMicroseconds, "Time spent sleeping due to a sleep function call.") \
\
M(ThreadPoolReaderPageCacheHit, "Number of times the read inside ThreadPoolReader was done from page cache.") \
M(ThreadPoolReaderPageCacheHitBytes, "Number of bytes read inside ThreadPoolReader when it was done from page cache.") \
M(ThreadPoolReaderPageCacheHitElapsedMicroseconds, "Time spent reading data from page cache in ThreadPoolReader.") \
M(ThreadPoolReaderPageCacheMiss, "Number of times the read inside ThreadPoolReader was not done from page cache and was hand off to thread pool.") \
M(ThreadPoolReaderPageCacheMissBytes, "Number of bytes read inside ThreadPoolReader when read was not done from page cache and was hand off to thread pool.") \
M(ThreadPoolReaderPageCacheMissElapsedMicroseconds, "Time spent reading data inside the asynchronous job in ThreadPoolReader - when read was not done from page cache.") \
\
M(AsynchronousReadWaitMicroseconds, "Time spent in waiting for asynchronous reads.") \
namespace ProfileEvents

6
src/Common/QueryProfiler.cpp
View File

@ -173,7 +173,7 @@ void QueryProfilerBase<ProfilerImpl>::tryCleanup()
}
template class QueryProfilerBase<QueryProfilerReal>;
template class QueryProfilerBase<QueryProfilerCpu>;
template class QueryProfilerBase<QueryProfilerCPU>;
QueryProfilerReal::QueryProfilerReal(const UInt64 thread_id, const UInt32 period)
: QueryProfilerBase(thread_id, CLOCK_MONOTONIC, period, SIGUSR1)
@ -185,11 +185,11 @@ void QueryProfilerReal::signalHandler(int sig, siginfo_t * info, void * context)
writeTraceInfo(TraceType::Real, sig, info, context);
}
QueryProfilerCpu::QueryProfilerCpu(const UInt64 thread_id, const UInt32 period)
QueryProfilerCPU::QueryProfilerCPU(const UInt64 thread_id, const UInt32 period)
: QueryProfilerBase(thread_id, CLOCK_THREAD_CPUTIME_ID, period, SIGUSR2)
{}
void QueryProfilerCpu::signalHandler(int sig, siginfo_t * info, void * context)
void QueryProfilerCPU::signalHandler(int sig, siginfo_t * info, void * context)
{
DENY_ALLOCATIONS_IN_SCOPE;
writeTraceInfo(TraceType::CPU, sig, info, context);

6
src/Common/QueryProfiler.h
View File

@ -26,7 +26,7 @@ namespace DB
* 3. write collected stack trace to trace_pipe for TraceCollector
*
* Destructor tries to unset timer and restore previous signal handler.
* Note that signal handler implementation is defined by template parameter. See QueryProfilerReal and QueryProfilerCpu.
* Note that signal handler implementation is defined by template parameter. See QueryProfilerReal and QueryProfilerCPU.
*/
template <typename ProfilerImpl>
class QueryProfilerBase
@ -62,10 +62,10 @@ public:
};
/// Query profiler with timer based on CPU clock
class QueryProfilerCpu : public QueryProfilerBase<QueryProfilerCpu>
class QueryProfilerCPU : public QueryProfilerBase<QueryProfilerCPU>
{
public:
QueryProfilerCpu(const UInt64 thread_id, const UInt32 period);
QueryProfilerCPU(const UInt64 thread_id, const UInt32 period);
static void signalHandler(int sig, siginfo_t * info, void * context);
};

100
src/Common/ShellCommand.cpp
View File

@ -24,6 +24,8 @@ namespace
CANNOT_DUP_STDOUT = 0x55555556,
CANNOT_DUP_STDERR = 0x55555557,
CANNOT_EXEC = 0x55555558,
CANNOT_DUP_READ_DESCRIPTOR = 0x55555559,
CANNOT_DUP_WRITE_DESCRIPTOR = 0x55555560,
};
}
@ -39,12 +41,12 @@ namespace ErrorCodes
extern const int CANNOT_CREATE_CHILD_PROCESS;
}
ShellCommand::ShellCommand(pid_t pid_, int & in_fd_, int & out_fd_, int & err_fd_, ShellCommandDestructorStrategy destructor_strategy_)
: pid(pid_)
, destructor_strategy(destructor_strategy_)
, in(in_fd_)
ShellCommand::ShellCommand(pid_t pid_, int & in_fd_, int & out_fd_, int & err_fd_, const ShellCommand::Config & config_)
: in(in_fd_)
, out(out_fd_)
, err(err_fd_)
, pid(pid_)
, config(config_)
{
}
@ -58,9 +60,9 @@ ShellCommand::~ShellCommand()
if (wait_called)
return;
if (destructor_strategy.terminate_in_destructor)
if (config.terminate_in_destructor_strategy.terminate_in_destructor)
{
size_t try_wait_timeout = destructor_strategy.wait_for_normal_exit_before_termination_seconds;
size_t try_wait_timeout = config.terminate_in_destructor_strategy.wait_for_normal_exit_before_termination_seconds;
bool process_terminated_normally = tryWaitProcessWithTimeout(try_wait_timeout);
if (!process_terminated_normally)
@ -149,8 +151,7 @@ void ShellCommand::logCommand(const char * filename, char * const argv[])
std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
const char * filename,
char * const argv[],
bool pipe_stdin_only,
ShellCommandDestructorStrategy terminate_in_destructor_strategy)
const Config & config)
{
logCommand(filename, argv);
@ -168,9 +169,18 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
PipeFDs pipe_stdout;
PipeFDs pipe_stderr;
std::vector<std::unique_ptr<PipeFDs>> read_pipe_fds;
std::vector<std::unique_ptr<PipeFDs>> write_pipe_fds;
for (size_t i = 0; i < config.read_fds.size(); ++i)
read_pipe_fds.emplace_back(std::make_unique<PipeFDs>());
for (size_t i = 0; i < config.write_fds.size(); ++i)
write_pipe_fds.emplace_back(std::make_unique<PipeFDs>());
pid_t pid = reinterpret_cast<pid_t(*)()>(real_vfork)();
if (-1 == pid)
if (pid == -1)
throwFromErrno("Cannot vfork", ErrorCodes::CANNOT_FORK);
if (0 == pid)
@ -184,7 +194,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
if (STDIN_FILENO != dup2(pipe_stdin.fds_rw[0], STDIN_FILENO))
_exit(int(ReturnCodes::CANNOT_DUP_STDIN));
if (!pipe_stdin_only)
if (!config.pipe_stdin_only)
{
if (STDOUT_FILENO != dup2(pipe_stdout.fds_rw[1], STDOUT_FILENO))
_exit(int(ReturnCodes::CANNOT_DUP_STDOUT));
@ -193,6 +203,24 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
_exit(int(ReturnCodes::CANNOT_DUP_STDERR));
}
for (size_t i = 0; i < config.read_fds.size(); ++i)
{
auto & fds = *read_pipe_fds[i];
auto fd = config.read_fds[i];
if (fd != dup2(fds.fds_rw[1], fd))
_exit(int(ReturnCodes::CANNOT_DUP_READ_DESCRIPTOR));
}
for (size_t i = 0; i < config.write_fds.size(); ++i)
{
auto & fds = *write_pipe_fds[i];
auto fd = config.write_fds[i];
if (fd != dup2(fds.fds_rw[0], fd))
_exit(int(ReturnCodes::CANNOT_DUP_WRITE_DESCRIPTOR));
}
// Reset the signal mask: it may be non-empty and will be inherited
// by the child process, which might not expect this.
sigset_t mask;
@ -207,35 +235,49 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
}
std::unique_ptr<ShellCommand> res(new ShellCommand(
pid, pipe_stdin.fds_rw[1], pipe_stdout.fds_rw[0], pipe_stderr.fds_rw[0], terminate_in_destructor_strategy));
pid,
pipe_stdin.fds_rw[1],
pipe_stdout.fds_rw[0],
pipe_stderr.fds_rw[0],
config));
for (size_t i = 0; i < config.read_fds.size(); ++i)
{
auto & fds = *read_pipe_fds[i];
auto fd = config.read_fds[i];
res->read_fds.emplace(fd, fds.fds_rw[0]);
}
for (size_t i = 0; i < config.write_fds.size(); ++i)
{
auto & fds = *write_pipe_fds[i];
auto fd = config.write_fds[i];
res->write_fds.emplace(fd, fds.fds_rw[1]);
}
LOG_TRACE(getLogger(), "Started shell command '{}' with pid {}", filename, pid);
return res;
}
std::unique_ptr<ShellCommand> ShellCommand::execute(
const std::string & command,
bool pipe_stdin_only,
ShellCommandDestructorStrategy terminate_in_destructor_strategy)
std::unique_ptr<ShellCommand> ShellCommand::execute(const ShellCommand::Config & config)
{
/// Arguments in non-constant chunks of memory (as required for `execv`).
/// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
std::vector<char> argv0("sh", &("sh"[3]));
std::vector<char> argv1("-c", &("-c"[3]));
std::vector<char> argv2(command.data(), command.data() + command.size() + 1);
auto config_copy = config;
config_copy.command = "/bin/sh";
config_copy.arguments = {"-c", config.command};
char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr };
for (const auto & argument : config.arguments)
config_copy.arguments.emplace_back(argument);
return executeImpl("/bin/sh", argv, pipe_stdin_only, terminate_in_destructor_strategy);
return executeDirect(config_copy);
}
std::unique_ptr<ShellCommand> ShellCommand::executeDirect(
const std::string & path,
const std::vector<std::string> & arguments,
ShellCommandDestructorStrategy terminate_in_destructor_strategy)
std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const ShellCommand::Config & config)
{
const auto & path = config.command;
const auto & arguments = config.arguments;
size_t argv_sum_size = path.size() + 1;
for (const auto & arg : arguments)
argv_sum_size += arg.size() + 1;
@ -255,7 +297,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeDirect(
argv[arguments.size() + 1] = nullptr;
return executeImpl(path.data(), argv.data(), false, terminate_in_destructor_strategy);
return executeImpl(path.data(), argv.data(), config);
}
@ -307,6 +349,10 @@ void ShellCommand::wait()
throw Exception("Cannot dup2 stderr of child process", ErrorCodes::CANNOT_CREATE_CHILD_PROCESS);
case int(ReturnCodes::CANNOT_EXEC):
throw Exception("Cannot execv in child process", ErrorCodes::CANNOT_CREATE_CHILD_PROCESS);
case int(ReturnCodes::CANNOT_DUP_READ_DESCRIPTOR):
throw Exception("Cannot dup2 read descriptor of child process", ErrorCodes::CANNOT_CREATE_CHILD_PROCESS);
case int(ReturnCodes::CANNOT_DUP_WRITE_DESCRIPTOR):
throw Exception("Cannot dup2 write descriptor of child process", ErrorCodes::CANNOT_CREATE_CHILD_PROCESS);
default:
throw Exception("Child process was exited with return code " + toString(retcode), ErrorCodes::CHILD_WAS_NOT_EXITED_NORMALLY);
}

88
src/Common/ShellCommand.h
View File

@ -23,10 +23,15 @@ namespace DB
* The second difference - allows to work simultaneously with stdin, and with stdout, and with stderr of running process,
* and also to obtain the return code and completion status.
*/
struct ShellCommandDestructorStrategy final
class ShellCommand final
{
explicit ShellCommandDestructorStrategy(bool terminate_in_destructor_, size_t wait_for_normal_exit_before_termination_seconds_ = 0)
public:
~ShellCommand();
struct DestructorStrategy final
{
explicit DestructorStrategy(bool terminate_in_destructor_, size_t wait_for_normal_exit_before_termination_seconds_ = 0)
: terminate_in_destructor(terminate_in_destructor_)
, wait_for_normal_exit_before_termination_seconds(wait_for_normal_exit_before_termination_seconds_)
{
@ -38,14 +43,56 @@ struct ShellCommandDestructorStrategy final
size_t wait_for_normal_exit_before_termination_seconds = 0;
};
class ShellCommand final
struct Config
{
private:
pid_t pid;
bool wait_called = false;
ShellCommandDestructorStrategy destructor_strategy;
Config(const std::string & command_)
: command(command_)
{}
ShellCommand(pid_t pid_, int & in_fd_, int & out_fd_, int & err_fd_, ShellCommandDestructorStrategy destructor_strategy_);
Config(const char * command_)
: command(command_)
{}
std::string command;
std::vector<std::string> arguments;
std::vector<int> read_fds;
std::vector<int> write_fds;
bool pipe_stdin_only = false;
DestructorStrategy terminate_in_destructor_strategy = DestructorStrategy(false);
};
/// Run the command using /bin/sh -c.
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
static std::unique_ptr<ShellCommand> execute(const Config & config);
/// Run the executable with the specified arguments. `arguments` - without argv[0].
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
static std::unique_ptr<ShellCommand> executeDirect(const Config & config);
/// Wait for the process to end, throw an exception if the code is not 0 or if the process was not completed by itself.
void wait();
/// Wait for the process to finish, see the return code. To throw an exception if the process was not completed independently.
int tryWait();
WriteBufferFromFile in; /// If the command reads from stdin, do not forget to call in.close() after writing all the data there.
ReadBufferFromFile out;
ReadBufferFromFile err;
std::unordered_map<int, ReadBufferFromFile> read_fds;
std::unordered_map<int, WriteBufferFromFile> write_fds;
private:
pid_t pid;
Config config;
bool wait_called = false;
ShellCommand(pid_t pid_, int & in_fd_, int & out_fd_, int & err_fd_, const Config & config);
bool tryWaitProcessWithTimeout(size_t timeout_in_seconds);
@ -54,28 +101,7 @@ private:
/// Print command name and the list of arguments to log. NOTE: No escaping of arguments is performed.
static void logCommand(const char * filename, char * const argv[]);
static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, ShellCommandDestructorStrategy terminate_in_destructor_strategy);
public:
WriteBufferFromFile in; /// If the command reads from stdin, do not forget to call in.close() after writing all the data there.
ReadBufferFromFile out;
ReadBufferFromFile err;
~ShellCommand();
/// Run the command using /bin/sh -c.
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false, ShellCommandDestructorStrategy terminate_in_destructor_strategy = ShellCommandDestructorStrategy(false));
/// Run the executable with the specified arguments. `arguments` - without argv[0].
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments, ShellCommandDestructorStrategy terminate_in_destructor_strategy = ShellCommandDestructorStrategy(false));
/// Wait for the process to end, throw an exception if the code is not 0 or if the process was not completed by itself.
void wait();
/// Wait for the process to finish, see the return code. To throw an exception if the process was not completed independently.
int tryWait();
static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], const Config & config);
};

1
src/Common/TLDListsHolder.cpp
View File

@ -64,6 +64,7 @@ size_t TLDListsHolder::parseAndAddTldList(const std::string & name, const std::s
while (!in.eof())
{
readEscapedStringUntilEOL(line, in);
if (!in.eof())
++in.position();
/// Skip comments
if (line.size() > 2 && line[0] == '/' && line[1] == '/')

4
src/Common/ThreadStatus.h
View File

@ -29,7 +29,7 @@ namespace DB
class QueryStatus;
class ThreadStatus;
class QueryProfilerReal;
class QueryProfilerCpu;
class QueryProfilerCPU;
class QueryThreadLog;
struct OpenTelemetrySpanHolder;
class TasksStatsCounters;
@ -140,7 +140,7 @@ protected:
// CPU and Real time query profilers
std::unique_ptr<QueryProfilerReal> query_profiler_real;
std::unique_ptr<QueryProfilerCpu> query_profiler_cpu;
std::unique_ptr<QueryProfilerCPU> query_profiler_cpu;
Poco::Logger * log = nullptr;

13
src/Common/TimerDescriptor.cpp
View File

@ -74,17 +74,24 @@ void TimerDescriptor::drain() const
}
}
void TimerDescriptor::setRelative(Poco::Timespan timespan) const
void TimerDescriptor::setRelative(uint64_t usec) const
{
static constexpr uint32_t TIMER_PRECISION = 1e6;
itimerspec spec;
spec.it_interval.tv_nsec = 0;
spec.it_interval.tv_sec = 0;
spec.it_value.tv_sec = timespan.totalSeconds();
spec.it_value.tv_nsec = timespan.useconds() * 1000;
spec.it_value.tv_sec = usec / TIMER_PRECISION;
spec.it_value.tv_nsec = (usec % TIMER_PRECISION) * 1'000;
if (-1 == timerfd_settime(timer_fd, 0 /*relative timer */, &spec, nullptr))
throwFromErrno("Cannot set time for timer_fd", ErrorCodes::CANNOT_SET_TIMER_PERIOD);
}
void TimerDescriptor::setRelative(Poco::Timespan timespan) const
{
setRelative(timespan.totalMicroseconds());
}
}
#endif

1
src/Common/TimerDescriptor.h
View File

@ -24,6 +24,7 @@ public:
void reset() const;
void drain() const;
void setRelative(uint64_t usec) const;
void setRelative(Poco::Timespan timespan) const;
};

31
src/Common/ZooKeeper/ZooKeeperImpl.cpp
View File

@ -387,6 +387,7 @@ void ZooKeeper::connect(
}
socket.connect(node.address, connection_timeout);
socket_address = socket.peerAddress();
socket.setReceiveTimeout(operation_timeout);
socket.setSendTimeout(operation_timeout);
@ -539,7 +540,7 @@ void ZooKeeper::sendThread()
try
{
while (!expired)
while (!requests_queue.isClosed())
{
auto prev_bytes_sent = out->count();
@ -571,7 +572,7 @@ void ZooKeeper::sendThread()
info.request->has_watch = true;
}
if (expired)
if (requests_queue.isClosed())
{
break;
}
@ -616,7 +617,7 @@ void ZooKeeper::receiveThread()
try
{
Int64 waited = 0;
while (!expired)
while (!requests_queue.isClosed())
{
auto prev_bytes_received = in->count();
@ -639,7 +640,7 @@ void ZooKeeper::receiveThread()
if (in->poll(max_wait))
{
if (expired)
if (requests_queue.isClosed())
break;
receiveEvent();
@ -839,12 +840,10 @@ void ZooKeeper::finalize(bool error_send, bool error_receive)
auto expire_session_if_not_expired = [&]
{
std::lock_guard lock(push_request_mutex);
if (!expired)
{
expired = true;
/// No new requests will appear in queue after close()
bool was_already_closed = requests_queue.close();
if (!was_already_closed)
active_session_metric_increment.destroy();
}
};
try
@ -1017,18 +1016,16 @@ void ZooKeeper::pushRequest(RequestInfo && info)
}
}
/// We must serialize 'pushRequest' and 'finalize' (from sendThread, receiveThread) calls
/// to avoid forgotten operations in the queue when session is expired.
/// Invariant: when expired, no new operations will be pushed to the queue in 'pushRequest'
/// and the queue will be drained in 'finalize'.
std::lock_guard lock(push_request_mutex);
if (expired)
if (requests_queue.isClosed())
throw Exception("Session expired", Error::ZSESSIONEXPIRED);
if (!requests_queue.tryPush(std::move(info), operation_timeout.totalMilliseconds()))
{
if (requests_queue.isClosed())
throw Exception("Session expired", Error::ZSESSIONEXPIRED);
throw Exception("Cannot push request to queue within operation timeout", Error::ZOPERATIONTIMEOUT);
}
}
catch (...)
{
finalize(false, false);
@ -1255,7 +1252,7 @@ void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const
{
elem.type = log_type;
elem.event_time = event_time;
elem.address = socket.peerAddress();
elem.address = socket_address;
elem.session_id = session_id;
maybe_zk_log->add(elem);
}

Some files were not shown because too many files have changed in this diff Show More