mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 01:22:04 +00:00
Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into taiyang-li-disk-reload
This commit is contained in:
kssenii
commit
5f5bbc1fca
6
.github/PULL_REQUEST_TEMPLATE.md
vendored
6
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -19,9 +19,9 @@ Detailed description / Documentation draft:
|
||||
...
|
||||
|
||||
|
||||
By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
|
||||
> By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
|
||||
|
||||
If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.
|
||||
> If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.
|
||||
|
||||
|
||||
Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/
|
||||
> Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/
|
||||
|
||||
@ -259,10 +259,25 @@ private:
|
||||
Poco::Logger * log;
|
||||
BaseDaemon & daemon;
|
||||
|
||||
void onTerminate(const std::string & message, UInt32 thread_num) const
|
||||
void onTerminate(std::string_view message, UInt32 thread_num) const
|
||||
{
|
||||
size_t pos = message.find('\n');
|
||||
|
||||
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) {}",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, thread_num, message);
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, thread_num, message.substr(0, pos));
|
||||
|
||||
/// Print trace from std::terminate exception line-by-line to make it easy for grep.
|
||||
while (pos != std::string_view::npos)
|
||||
{
|
||||
++pos;
|
||||
size_t next_pos = message.find('\n', pos);
|
||||
size_t size = next_pos;
|
||||
if (next_pos != std::string_view::npos)
|
||||
size = next_pos - pos;
|
||||
|
||||
LOG_FATAL(log, "{}", message.substr(pos, size));
|
||||
pos = next_pos;
|
||||
}
|
||||
}
|
||||
|
||||
void onFault(
|
||||
|
||||
@ -8,6 +8,7 @@ if [ -x ./clickhouse ]
|
||||
then
|
||||
CLICKHOUSE_CLIENT="./clickhouse client"
|
||||
elif command -v clickhouse-client >/dev/null 2>&1
|
||||
then
|
||||
CLICKHOUSE_CLIENT="clickhouse-client"
|
||||
else
|
||||
echo "clickhouse-client is not found"
|
||||
|
||||
2
contrib/NuRaft
vendored
2
contrib/NuRaft
vendored
@ -1 +1 @@
|
||||
Subproject commit 976874b7aa7f422bf4ea595bb7d1166c617b1c26
|
||||
Subproject commit 0ce9490093021c63564cca159571a8b27772ad48
|
||||
@ -22,6 +22,7 @@ set(SRCS
|
||||
"${LIBRARY_DIR}/src/launcher.cxx"
|
||||
"${LIBRARY_DIR}/src/srv_config.cxx"
|
||||
"${LIBRARY_DIR}/src/snapshot_sync_req.cxx"
|
||||
"${LIBRARY_DIR}/src/snapshot_sync_ctx.cxx"
|
||||
"${LIBRARY_DIR}/src/handle_timeout.cxx"
|
||||
"${LIBRARY_DIR}/src/handle_append_entries.cxx"
|
||||
"${LIBRARY_DIR}/src/cluster_config.cxx"
|
||||
|
||||
2
contrib/protobuf
vendored
2
contrib/protobuf
vendored
@ -1 +1 @@
|
||||
Subproject commit 73b12814204ad9068ba352914d0dc244648b48ee
|
||||
Subproject commit 75601841d172c73ae6bf4ce8121f42b875cdbabd
|
||||
@ -299,6 +299,7 @@ function run_tests
|
||||
01318_decrypt # Depends on OpenSSL
|
||||
01663_aes_msan # Depends on OpenSSL
|
||||
01667_aes_args_check # Depends on OpenSSL
|
||||
01683_codec_encrypted # Depends on OpenSSL
|
||||
01776_decrypt_aead_size_check # Depends on OpenSSL
|
||||
01811_filter_by_null # Depends on OpenSSL
|
||||
01281_unsucceeded_insert_select_queries_counter
|
||||
|
||||
@ -20,6 +20,7 @@ def get_skip_list_cmd(path):
|
||||
|
||||
def get_options(i):
|
||||
options = []
|
||||
client_options = []
|
||||
if 0 < i:
|
||||
options.append("--order=random")
|
||||
|
||||
@ -27,25 +28,29 @@ def get_options(i):
|
||||
options.append("--db-engine=Ordinary")
|
||||
|
||||
if i % 3 == 2:
|
||||
options.append('''--client-option='allow_experimental_database_replicated=1' --db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i))
|
||||
options.append('''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i))
|
||||
client_options.append('allow_experimental_database_replicated=1')
|
||||
|
||||
# If database name is not specified, new database is created for each functional test.
|
||||
# Run some threads with one database for all tests.
|
||||
if i % 2 == 1:
|
||||
options.append(" --database=test_{}".format(i))
|
||||
|
||||
if i % 7 == 0:
|
||||
options.append(" --client-option='join_use_nulls=1'")
|
||||
if i % 5 == 1:
|
||||
client_options.append("join_use_nulls=1")
|
||||
|
||||
if i % 14 == 0:
|
||||
options.append(' --client-option="join_algorithm=\'partial_merge\'"')
|
||||
if i % 15 == 6:
|
||||
client_options.append("join_algorithm='partial_merge'")
|
||||
|
||||
if i % 21 == 0:
|
||||
options.append(' --client-option="join_algorithm=\'auto\'"')
|
||||
options.append(' --client-option="max_rows_in_join=1000"')
|
||||
if i % 15 == 11:
|
||||
client_options.append("join_algorithm='auto'")
|
||||
client_options.append('max_rows_in_join=1000')
|
||||
|
||||
if i == 13:
|
||||
options.append(" --client-option='memory_tracker_fault_probability=0.00001'")
|
||||
client_options.append('memory_tracker_fault_probability=0.001')
|
||||
|
||||
if client_options:
|
||||
options.append(" --client-option " + ' '.join(client_options))
|
||||
|
||||
return ' '.join(options)
|
||||
|
||||
|
||||
@ -35,7 +35,7 @@ RUN apt-get update \
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
RUN pip3 install urllib3 testflows==1.6.90 docker-compose==1.29.1 docker==5.0.0 dicttoxml kazoo tzlocal python-dateutil numpy
|
||||
RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0 dicttoxml kazoo tzlocal python-dateutil numpy
|
||||
|
||||
ENV DOCKER_CHANNEL stable
|
||||
ENV DOCKER_VERSION 20.10.6
|
||||
|
||||
@ -43,7 +43,7 @@ toc_title: Integrations
|
||||
- Monitoring
|
||||
- [Graphite](https://graphiteapp.org)
|
||||
- [graphouse](https://github.com/yandex/graphouse)
|
||||
- [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) +
|
||||
- [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse)
|
||||
- [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse)
|
||||
- [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - optimizes staled partitions in [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) if rules from [rollup configuration](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) could be applied
|
||||
- [Grafana](https://grafana.com/)
|
||||
|
||||
@ -69,6 +69,28 @@ If no conditions met for a data part, ClickHouse uses the `lz4` compression.
|
||||
</compression>
|
||||
```
|
||||
|
||||
## encryption {#server-settings-encryption}
|
||||
|
||||
Configures a command to obtain a key to be used by [encryption codecs](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). The command, or a shell script, is expected to write a Base64-encoded key of any length to the stdout.
|
||||
|
||||
**Example**
|
||||
|
||||
For Linux with systemd:
|
||||
|
||||
```xml
|
||||
<encryption>
|
||||
<key_command>/usr/bin/systemd-ask-password --id="clickhouse-server" --timeout=0 "Enter the ClickHouse encryption passphrase:" | base64</key_command>
|
||||
</encryption>
|
||||
```
|
||||
|
||||
For other systems:
|
||||
|
||||
```xml
|
||||
<encryption>
|
||||
<key_command><![CDATA[IFS=; echo -n >/dev/tty "Enter the ClickHouse encryption passphrase: "; stty=`stty -F /dev/tty -g`; stty -F /dev/tty -echo; read k </dev/tty; stty -F /dev/tty "$stty"; echo -n $k | base64]]></key_command>
|
||||
</encryption>
|
||||
```
|
||||
|
||||
## custom_settings_prefixes {#custom_settings_prefixes}
|
||||
|
||||
List of prefixes for [custom settings](../../operations/settings/index.md#custom_settings). The prefixes must be separated with commas.
|
||||
|
||||
@ -20,6 +20,29 @@ Possible values:
|
||||
- `global` — Replaces the `IN`/`JOIN` query with `GLOBAL IN`/`GLOBAL JOIN.`
|
||||
- `allow` — Allows the use of these types of subqueries.
|
||||
|
||||
## prefer_global_in_and_join {#prefer-global-in-and-join}
|
||||
|
||||
Enables the replacement of `IN`/`JOIN` operators with `GLOBAL IN`/`GLOBAL JOIN`.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — Disabled. `IN`/`JOIN` operators are not replaced with `GLOBAL IN`/`GLOBAL JOIN`.
|
||||
- 1 — Enabled. `IN`/`JOIN` operators are replaced with `GLOBAL IN`/`GLOBAL JOIN`.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
**Usage**
|
||||
|
||||
Although `SET distributed_product_mode=global` can change the queries behavior for the distributed tables, it's not suitable for local tables or tables from external resources. Here is when the `prefer_global_in_and_join` setting comes into play.
|
||||
|
||||
For example, we have query serving nodes that contain local tables, which are not suitable for distribution. We need to scatter their data on the fly during distributed processing with the `GLOBAL` keyword — `GLOBAL IN`/`GLOBAL JOIN`.
|
||||
|
||||
Another use case of `prefer_global_in_and_join` is accessing tables created by external engines. This setting helps to reduce the number of calls to external sources while joining such tables: only one call per query.
|
||||
|
||||
**See also:**
|
||||
|
||||
- [Distributed subqueries](../../sql-reference/operators/in.md#select-distributed-subqueries) for more information on how to use `GLOBAL IN`/`GLOBAL JOIN`
|
||||
|
||||
## enable_optimize_predicate_expression {#enable-optimize-predicate-expression}
|
||||
|
||||
Turns on predicate pushdown in `SELECT` queries.
|
||||
|
||||
@ -62,4 +62,3 @@ exception_code: ZOK
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/distributed_ddl_queuedistributed_ddl_queue.md) <!--hide-->
|
||||
|
||||
@ -51,6 +51,7 @@ Columns:
|
||||
- `databases` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the databases present in the query.
|
||||
- `tables` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the tables present in the query.
|
||||
- `columns` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the columns present in the query.
|
||||
- `projections` ([String](../../sql-reference/data-types/string.md)) — Names of the projections used during the query execution.
|
||||
- `exception_code` ([Int32](../../sql-reference/data-types/int-uint.md)) — Code of an exception.
|
||||
- `exception` ([String](../../sql-reference/data-types/string.md)) — Exception message.
|
||||
- `stack_trace` ([String](../../sql-reference/data-types/string.md)) — [Stack trace](https://en.wikipedia.org/wiki/Stack_trace). An empty string, if the query was completed successfully.
|
||||
@ -65,6 +66,8 @@ Columns:
|
||||
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
|
||||
- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
|
||||
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the parent query.
|
||||
- `initial_query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Initial query starting time (for distributed query execution).
|
||||
- `initial_query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Initial query starting time with microseconds precision (for distributed query execution).
|
||||
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Interface that the query was initiated from. Possible values:
|
||||
- 1 — TCP.
|
||||
- 2 — HTTP.
|
||||
@ -101,55 +104,77 @@ Columns:
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT * FROM system.query_log WHERE type = 'QueryFinish' AND (query LIKE '%toDate(\'2000-12-05\')%') ORDER BY query_start_time DESC LIMIT 1 FORMAT Vertical;
|
||||
SELECT * FROM system.query_log WHERE type = 'QueryFinish' ORDER BY query_start_time DESC LIMIT 1 FORMAT Vertical;
|
||||
```
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
type: QueryStart
|
||||
event_date: 2020-09-11
|
||||
event_time: 2020-09-11 10:08:17
|
||||
event_time_microseconds: 2020-09-11 10:08:17.063321
|
||||
query_start_time: 2020-09-11 10:08:17
|
||||
query_start_time_microseconds: 2020-09-11 10:08:17.063321
|
||||
query_duration_ms: 0
|
||||
read_rows: 0
|
||||
read_bytes: 0
|
||||
type: QueryFinish
|
||||
event_date: 2021-07-28
|
||||
event_time: 2021-07-28 13:46:56
|
||||
event_time_microseconds: 2021-07-28 13:46:56.719791
|
||||
query_start_time: 2021-07-28 13:46:56
|
||||
query_start_time_microseconds: 2021-07-28 13:46:56.704542
|
||||
query_duration_ms: 14
|
||||
read_rows: 8393
|
||||
read_bytes: 374325
|
||||
written_rows: 0
|
||||
written_bytes: 0
|
||||
result_rows: 0
|
||||
result_bytes: 0
|
||||
memory_usage: 0
|
||||
result_rows: 4201
|
||||
result_bytes: 153024
|
||||
memory_usage: 4714038
|
||||
current_database: default
|
||||
query: INSERT INTO test1 VALUES
|
||||
query: SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)
|
||||
normalized_query_hash: 6666026786019643712
|
||||
query_kind: Select
|
||||
databases: ['system']
|
||||
tables: ['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']
|
||||
columns: ['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']
|
||||
projections: []
|
||||
exception_code: 0
|
||||
exception:
|
||||
stack_trace:
|
||||
is_initial_query: 1
|
||||
user: default
|
||||
query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef
|
||||
query_id: a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf
|
||||
address: ::ffff:127.0.0.1
|
||||
port: 33452
|
||||
port: 51006
|
||||
initial_user: default
|
||||
initial_query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef
|
||||
initial_query_id: a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf
|
||||
initial_address: ::ffff:127.0.0.1
|
||||
initial_port: 33452
|
||||
initial_port: 51006
|
||||
initial_query_start_time: 2021-07-28 13:46:56
|
||||
initial_query_start_time_microseconds: 2021-07-28 13:46:56.704542
|
||||
interface: 1
|
||||
os_user: bharatnc
|
||||
client_hostname: tower
|
||||
client_name: ClickHouse
|
||||
client_revision: 54437
|
||||
client_version_major: 20
|
||||
client_version_minor: 7
|
||||
client_version_patch: 2
|
||||
os_user:
|
||||
client_hostname:
|
||||
client_name: ClickHouse client
|
||||
client_revision: 54449
|
||||
client_version_major: 21
|
||||
client_version_minor: 8
|
||||
client_version_patch: 0
|
||||
http_method: 0
|
||||
http_user_agent:
|
||||
http_referer:
|
||||
forwarded_for:
|
||||
quota_key:
|
||||
revision: 54440
|
||||
thread_ids: []
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
Settings: {'background_pool_size':'32','load_balancing':'random','allow_suspicious_low_cardinality_types':'1','distributed_aggregation_memory_efficient':'1','skip_unavailable_shards':'1','log_queries':'1','max_bytes_before_external_group_by':'20000000000','max_bytes_before_external_sort':'20000000000','allow_introspection_functions':'1'}
|
||||
revision: 54453
|
||||
log_comment:
|
||||
thread_ids: [5058,22097,22110,22094]
|
||||
ProfileEvents.Names: ['Query','SelectQuery','ArenaAllocChunks','ArenaAllocBytes','FunctionExecute','NetworkSendElapsedMicroseconds','SelectedRows','SelectedBytes','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SystemTimeMicroseconds','SoftPageFaults','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSWriteChars']
|
||||
ProfileEvents.Values: [1,1,39,352256,64,360,8393,374325,412,440,34480,13108,4723,671,19,17828,8192,10240]
|
||||
Settings.Names: ['load_balancing','max_memory_usage']
|
||||
Settings.Values: ['random','10000000000']
|
||||
used_aggregate_functions: []
|
||||
used_aggregate_function_combinators: []
|
||||
used_database_engines: []
|
||||
used_data_type_families: ['UInt64','UInt8','Nullable','String','date']
|
||||
used_dictionaries: []
|
||||
used_formats: []
|
||||
used_functions: ['concat','notEmpty','extractAll']
|
||||
used_storages: []
|
||||
used_table_functions: []
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
@ -85,7 +85,7 @@ hex(arg)
|
||||
|
||||
The function is using uppercase letters `A-F` and not using any prefixes (like `0x`) or suffixes (like `h`).
|
||||
|
||||
For integer arguments, it prints hex digits (“nibbles”) from the most significant to least significant (big endian or “human readable” order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints both digits of every byte even if leading digit is zero.
|
||||
For integer arguments, it prints hex digits (“nibbles”) from the most significant to least significant (big-endian or “human-readable” order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints both digits of every byte even if the leading digit is zero.
|
||||
|
||||
**Example**
|
||||
|
||||
@ -105,7 +105,7 @@ Values of type `Date` and `DateTime` are formatted as corresponding integers (th
|
||||
|
||||
For `String` and `FixedString`, all bytes are simply encoded as two hexadecimal numbers. Zero bytes are not omitted.
|
||||
|
||||
Values of floating point and Decimal types are encoded as their representation in memory. As we support little endian architecture, they are encoded in little endian. Zero leading/trailing bytes are not omitted.
|
||||
Values of floating point and Decimal types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted.
|
||||
|
||||
**Arguments**
|
||||
|
||||
@ -206,6 +206,141 @@ Result:
|
||||
└──────┘
|
||||
```
|
||||
|
||||
## bin {#bin}
|
||||
|
||||
Returns a string containing the argument’s binary representation.
|
||||
|
||||
Alias: `BIN`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
bin(arg)
|
||||
```
|
||||
|
||||
For integer arguments, it prints bin digits from the most significant to least significant (big-endian or “human-readable” order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints eight digits of every byte if the leading digit is zero.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT bin(1);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
00000001
|
||||
```
|
||||
|
||||
Values of type `Date` and `DateTime` are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime).
|
||||
|
||||
For `String` and `FixedString`, all bytes are simply encoded as eight binary numbers. Zero bytes are not omitted.
|
||||
|
||||
Values of floating-point and Decimal types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `arg` — A value to convert to binary. Types: [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A string with the binary representation of the argument.
|
||||
|
||||
Type: `String`.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT bin(toFloat32(number)) as bin_presentation FROM numbers(15, 2);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─bin_presentation─────────────────┐
|
||||
│ 00000000000000000111000001000001 │
|
||||
│ 00000000000000001000000001000001 │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT bin(toFloat64(number)) as bin_presentation FROM numbers(15, 2);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─bin_presentation─────────────────────────────────────────────────┐
|
||||
│ 0000000000000000000000000000000000000000000000000010111001000000 │
|
||||
│ 0000000000000000000000000000000000000000000000000011000001000000 │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## unbin {#unbinstr}
|
||||
|
||||
Performs the opposite operation of [bin](#bin). It interprets each pair of binary digits (in the argument) as a number and converts it to the byte represented by the number. The return value is a binary string (BLOB).
|
||||
|
||||
If you want to convert the result to a number, you can use the [reverse](../../sql-reference/functions/string-functions.md#reverse) and [reinterpretAs<Type>](../../sql-reference/functions/type-conversion-functions.md#type-conversion-functions) functions.
|
||||
|
||||
!!! note "Note"
|
||||
If `unbin` is invoked from within the `clickhouse-client`, binary strings display using UTF-8.
|
||||
|
||||
Alias: `UNBIN`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
unbin(arg)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `arg` — A string containing any number of binary digits. Type: [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
Supports binary digits `0-1`. The number of binary digits does not have to be multiples of eight. If the argument string contains anything other than binary digits, some implementation-defined result is returned (an exception isn’t thrown). For a numeric argument the inverse of bin(N) is not performed by unbin().
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A binary string (BLOB).
|
||||
|
||||
Type: [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT UNBIN('001100000011000100110010'), UNBIN('0100110101111001010100110101000101001100');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─unbin('001100000011000100110010')─┬─unbin('0100110101111001010100110101000101001100')─┐
|
||||
│ 012 │ MySQL │
|
||||
└───────────────────────────────────┴───────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT reinterpretAsUInt64(reverse(unbin('1010'))) AS num;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─num─┐
|
||||
│ 10 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## UUIDStringToNum(str) {#uuidstringtonumstr}
|
||||
|
||||
Accepts a string containing 36 characters in the format `123e4567-e89b-12d3-a456-426655440000`, and returns it as a set of bytes in a FixedString(16).
|
||||
|
||||
@ -20,12 +20,11 @@ The following actions are supported:
|
||||
|
||||
- [ADD COLUMN](#alter_add-column) — Adds a new column to the table.
|
||||
- [DROP COLUMN](#alter_drop-column) — Deletes the column.
|
||||
- [RENAME COLUMN](#alter_rename-column) — Renames the column.
|
||||
- [RENAME COLUMN](#alter_rename-column) — Renames an existing column.
|
||||
- [CLEAR COLUMN](#alter_clear-column) — Resets column values.
|
||||
- [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column.
|
||||
- [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL.
|
||||
- [MODIFY COLUMN REMOVE](#modify-remove) — Removes one of the column properties.
|
||||
- [RENAME COLUMN](#alter_rename-column) — Renames an existing column.
|
||||
|
||||
These actions are described in detail below.
|
||||
|
||||
@ -35,7 +34,7 @@ These actions are described in detail below.
|
||||
ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after | FIRST]
|
||||
```
|
||||
|
||||
Adds a new column to the table with the specified `name`, `type`, [`codec`](../../../sql-reference/statements/create/table.md#codecs) and `default_expr` (see the section [Default expressions](../../../sql-reference/statements/create/table.md#create-default-values)).
|
||||
Adds a new column to the table with the specified `name`, `type`, [`codec`](../create/table.md#codecs) and `default_expr` (see the section [Default expressions](../../../sql-reference/statements/create/table.md#create-default-values)).
|
||||
|
||||
If the `IF NOT EXISTS` clause is included, the query won’t return an error if the column already exists. If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. If you want to add a column to the beginning of the table use the `FIRST` clause. Otherwise, the column is added to the end of the table. For a chain of actions, `name_after` can be the name of a column that is added in one of the previous actions.
|
||||
|
||||
@ -64,6 +63,7 @@ Added2 UInt32
|
||||
ToDrop UInt32
|
||||
Added3 UInt32
|
||||
```
|
||||
|
||||
## DROP COLUMN {#alter_drop-column}
|
||||
|
||||
``` sql
|
||||
@ -118,7 +118,7 @@ ALTER TABLE visits CLEAR COLUMN browser IN PARTITION tuple()
|
||||
## COMMENT COLUMN {#alter_comment-column}
|
||||
|
||||
``` sql
|
||||
COMMENT COLUMN [IF EXISTS] name 'comment'
|
||||
COMMENT COLUMN [IF EXISTS] name 'Text comment'
|
||||
```
|
||||
|
||||
Adds a comment to the column. If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist.
|
||||
@ -136,7 +136,7 @@ ALTER TABLE visits COMMENT COLUMN browser 'The table shows the browser used for
|
||||
## MODIFY COLUMN {#alter_modify-column}
|
||||
|
||||
``` sql
|
||||
MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL] [AFTER name_after | FIRST]
|
||||
MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [codec] [TTL] [AFTER name_after | FIRST]
|
||||
```
|
||||
|
||||
This query changes the `name` column properties:
|
||||
@ -145,8 +145,12 @@ This query changes the `name` column properties:
|
||||
|
||||
- Default expression
|
||||
|
||||
- Compression Codec
|
||||
|
||||
- TTL
|
||||
|
||||
For examples of columns compression CODECS modifying, see [Column Compression Codecs](../create/table.md#codecs).
|
||||
|
||||
For examples of columns TTL modifying, see [Column TTL](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-column-ttl).
|
||||
|
||||
If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist.
|
||||
@ -179,6 +183,8 @@ ALTER TABLE table_name MODIFY column_name REMOVE property;
|
||||
|
||||
**Example**
|
||||
|
||||
Remove TTL:
|
||||
|
||||
```sql
|
||||
ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
|
||||
```
|
||||
@ -187,22 +193,6 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
|
||||
|
||||
- [REMOVE TTL](ttl.md).
|
||||
|
||||
## RENAME COLUMN {#alter_rename-column}
|
||||
|
||||
Renames an existing column.
|
||||
|
||||
Syntax:
|
||||
|
||||
```sql
|
||||
ALTER TABLE table_name RENAME COLUMN column_name TO new_column_name
|
||||
```
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
ALTER TABLE table_with_ttl RENAME COLUMN column_ttl TO column_ttl_new;
|
||||
```
|
||||
|
||||
## Limitations {#alter-query-limitations}
|
||||
|
||||
The `ALTER` query lets you create and delete separate elements (columns) in nested data structures, but not whole nested data structures. To add a nested data structure, you can add columns with a name like `name.nested_name` and the type `Array(T)`. A nested data structure is equivalent to multiple array columns with a name that has the same prefix before the dot.
|
||||
@ -213,4 +203,4 @@ If the `ALTER` query is not sufficient to make the table changes you need, you c
|
||||
|
||||
The `ALTER` query blocks all reads and writes for the table. In other words, if a long `SELECT` is running at the time of the `ALTER` query, the `ALTER` query will wait for it to complete. At the same time, all new queries to the same table will wait while this `ALTER` is running.
|
||||
|
||||
For tables that do not store data themselves (such as `Merge` and `Distributed`), `ALTER` just changes the table structure, and does not change the structure of subordinate tables. For example, when running ALTER for a `Distributed` table, you will also need to run `ALTER` for the tables on all remote servers.
|
||||
For tables that do not store data themselves (such as [Merge](../../../sql-reference/statements/alter/index.md) and [Distributed](../../../sql-reference/statements/alter/index.md)), `ALTER` just changes the table structure, and does not change the structure of subordinate tables. For example, when running ALTER for a `Distributed` table, you will also need to run `ALTER` for the tables on all remote servers.
|
||||
|
||||
@ -254,6 +254,20 @@ CREATE TABLE codec_example
|
||||
ENGINE = MergeTree()
|
||||
```
|
||||
|
||||
### Encryption Codecs {#create-query-encryption-codecs}
|
||||
|
||||
These codecs don't actually compress data, but instead encrypt data on disk. These are only available when an encryption key is specified by [encryption](../../../operations/server-configuration-parameters/settings.md#server-settings-encryption) settings. Note that encryption only makes sense at the end of codec pipelines, because encrypted data usually can't be compressed in any meaningful way.
|
||||
|
||||
Encryption codecs:
|
||||
|
||||
- `Encrypted('AES-128-GCM-SIV')` — Encrypts data with AES-128 in [RFC 8452](https://tools.ietf.org/html/rfc8452) GCM-SIV mode. This codec uses a fixed nonce and encryption is therefore deterministic. This makes it compatible with deduplicating engines such as [ReplicatedMergeTree](../../../engines/table-engines/mergetree-family/replication.md) but has a weakness: when the same data block is encrypted twice, the resulting ciphertext will be exactly the same so an adversary who can read the disk can see this equivalence (although only the equivalence).
|
||||
|
||||
!!! attention "Attention"
|
||||
Most engines including the "*MergeTree" family create index files on disk without applying codecs. This means plaintext will appear on disk if an encrypted column is indexed.
|
||||
|
||||
!!! attention "Attention"
|
||||
If you perform a SELECT query mentioning a specific value in an encrypted column (such as in its WHERE clause), the value may appear in [system.query_log](../../../operations/system-tables/query_log.md). You may want to disable the logging.
|
||||
|
||||
## Temporary Tables {#temporary-tables}
|
||||
|
||||
ClickHouse supports temporary tables which have the following characteristics:
|
||||
|
||||
@ -45,7 +45,7 @@ toc_title: "\u7D71\u5408"
|
||||
- 監視
|
||||
- [黒鉛](https://graphiteapp.org)
|
||||
- [グラファウス](https://github.com/yandex/graphouse)
|
||||
- [カーボンクリックハウス](https://github.com/lomik/carbon-clickhouse) +
|
||||
- [カーボンクリックハウス](https://github.com/lomik/carbon-clickhouse)
|
||||
- [グラファイト-クリック](https://github.com/lomik/graphite-clickhouse)
|
||||
- [黒鉛-ch-オプティマイザー](https://github.com/innogames/graphite-ch-optimizer) -staled仕切りを最大限に活用する [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) からのルールの場合 [ロールアップ構成](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) 応用できます
|
||||
- [グラファナ](https://grafana.com/)
|
||||
|
||||
@ -43,7 +43,7 @@ toc_title: "Библиотеки для интеграции от сторонн
|
||||
- Мониторинг
|
||||
- [Graphite](https://graphiteapp.org)
|
||||
- [graphouse](https://github.com/yandex/graphouse)
|
||||
- [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) +
|
||||
- [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse)
|
||||
- [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse)
|
||||
- [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - оптимизирует партиции таблиц [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) согласно правилам в [конфигурации rollup](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration)
|
||||
- [Grafana](https://grafana.com/)
|
||||
|
||||
@ -25,6 +25,30 @@ ClickHouse применяет настройку в тех случаях, ко
|
||||
- `global` — заменяет запрос `IN`/`JOIN` на `GLOBAL IN`/`GLOBAL JOIN.`
|
||||
- `allow` — разрешает использование таких подзапросов.
|
||||
|
||||
## prefer_global_in_and_join {#prefer-global-in-and-join}
|
||||
|
||||
Заменяет запрос `IN`/`JOIN` на `GLOBAL IN`/`GLOBAL JOIN`.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- 0 — выключена. Операторы `IN`/`JOIN` не заменяются на `GLOBAL IN`/`GLOBAL JOIN`.
|
||||
- 1 — включена. Операторы `IN`/`JOIN` заменяются на `GLOBAL IN`/`GLOBAL JOIN`.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
**Использование**
|
||||
|
||||
Настройка `SET distributed_product_mode=global` меняет поведение запросов для распределенных таблиц, но она не подходит для локальных таблиц или таблиц из внешних источников. В этих случаях удобно использовать настройку `prefer_global_in_and_join`.
|
||||
|
||||
Например, если нужно объединить все данные из локальных таблиц, которые находятся на разных узлах — для распределенной обработки необходим `GLOBAL JOIN`.
|
||||
|
||||
Другой вариант использования настройки `prefer_global_in_and_join` — регулирование обращений к таблицам из внешних источников.
|
||||
Эта настройка помогает уменьшить количество обращений к внешним ресурсам при объединении внешних таблиц: только один вызов на весь распределенный запрос.
|
||||
|
||||
**См. также:**
|
||||
|
||||
- [Распределенные подзапросы](../../sql-reference/operators/in.md#select-distributed-subqueries) `GLOBAL IN`/`GLOBAL JOIN`
|
||||
|
||||
## enable_optimize_predicate_expression {#enable-optimize-predicate-expression}
|
||||
|
||||
Включает пробрасывание предикатов в подзапросы для запросов `SELECT`.
|
||||
|
||||
@ -35,4 +35,3 @@ SELECT * FROM system.asynchronous_metrics LIMIT 10
|
||||
- [system.events](#system_tables-events) — таблица с количеством произошедших событий.
|
||||
- [system.metric_log](#system_tables-metric_log) — таблица фиксирующая историю значений метрик из `system.metrics` и `system.events`.
|
||||
|
||||
|
||||
@ -61,4 +61,3 @@ exception_code: ZOK
|
||||
2 rows in set. Elapsed: 0.025 sec.
|
||||
```
|
||||
|
||||
|
||||
@ -51,6 +51,7 @@ ClickHouse не удаляет данные из таблица автомати
|
||||
- `databases` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — имена баз данных, присутствующих в запросе.
|
||||
- `tables` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — имена таблиц, присутствующих в запросе.
|
||||
- `columns` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — имена столбцов, присутствующих в запросе.
|
||||
- `projections` ([String](../../sql-reference/data-types/string.md)) — имена проекций, использованных при выполнении запроса.
|
||||
- `exception_code` ([Int32](../../sql-reference/data-types/int-uint.md)) — код исключения.
|
||||
- `exception` ([String](../../sql-reference/data-types/string.md)) — сообщение исключения, если запрос завершился по исключению.
|
||||
- `stack_trace` ([String](../../sql-reference/data-types/string.md)) — [stack trace](https://en.wikipedia.org/wiki/Stack_trace). Пустая строка, если запрос успешно завершен.
|
||||
@ -65,6 +66,8 @@ ClickHouse не удаляет данные из таблица автомати
|
||||
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID родительского запроса.
|
||||
- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
|
||||
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт, с которого клиент сделал родительский запрос.
|
||||
- `initial_query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса (для распределенных запросов).
|
||||
- `initial_query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд (для распределенных запросов).
|
||||
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md)) — интерфейс, с которого ушёл запрос. Возможные значения:
|
||||
- 1 — TCP.
|
||||
- 2 — HTTP.
|
||||
@ -101,55 +104,77 @@ ClickHouse не удаляет данные из таблица автомати
|
||||
**Пример**
|
||||
|
||||
``` sql
|
||||
SELECT * FROM system.query_log WHERE type = 'QueryFinish' AND (query LIKE '%toDate(\'2000-12-05\')%') ORDER BY query_start_time DESC LIMIT 1 FORMAT Vertical;
|
||||
SELECT * FROM system.query_log WHERE type = 'QueryFinish' ORDER BY query_start_time DESC LIMIT 1 FORMAT Vertical;
|
||||
```
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
type: QueryStart
|
||||
event_date: 2020-09-11
|
||||
event_time: 2020-09-11 10:08:17
|
||||
event_time_microseconds: 2020-09-11 10:08:17.063321
|
||||
query_start_time: 2020-09-11 10:08:17
|
||||
query_start_time_microseconds: 2020-09-11 10:08:17.063321
|
||||
query_duration_ms: 0
|
||||
read_rows: 0
|
||||
read_bytes: 0
|
||||
type: QueryFinish
|
||||
event_date: 2021-07-28
|
||||
event_time: 2021-07-28 13:46:56
|
||||
event_time_microseconds: 2021-07-28 13:46:56.719791
|
||||
query_start_time: 2021-07-28 13:46:56
|
||||
query_start_time_microseconds: 2021-07-28 13:46:56.704542
|
||||
query_duration_ms: 14
|
||||
read_rows: 8393
|
||||
read_bytes: 374325
|
||||
written_rows: 0
|
||||
written_bytes: 0
|
||||
result_rows: 0
|
||||
result_bytes: 0
|
||||
memory_usage: 0
|
||||
result_rows: 4201
|
||||
result_bytes: 153024
|
||||
memory_usage: 4714038
|
||||
current_database: default
|
||||
query: INSERT INTO test1 VALUES
|
||||
query: SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)
|
||||
normalized_query_hash: 6666026786019643712
|
||||
query_kind: Select
|
||||
databases: ['system']
|
||||
tables: ['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']
|
||||
columns: ['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']
|
||||
projections: []
|
||||
exception_code: 0
|
||||
exception:
|
||||
stack_trace:
|
||||
is_initial_query: 1
|
||||
user: default
|
||||
query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef
|
||||
query_id: a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf
|
||||
address: ::ffff:127.0.0.1
|
||||
port: 33452
|
||||
port: 51006
|
||||
initial_user: default
|
||||
initial_query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef
|
||||
initial_query_id: a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf
|
||||
initial_address: ::ffff:127.0.0.1
|
||||
initial_port: 33452
|
||||
initial_port: 51006
|
||||
initial_query_start_time: 2021-07-28 13:46:56
|
||||
initial_query_start_time_microseconds: 2021-07-28 13:46:56.704542
|
||||
interface: 1
|
||||
os_user: bharatnc
|
||||
client_hostname: tower
|
||||
client_name: ClickHouse
|
||||
client_revision: 54437
|
||||
client_version_major: 20
|
||||
client_version_minor: 7
|
||||
client_version_patch: 2
|
||||
os_user:
|
||||
client_hostname:
|
||||
client_name: ClickHouse client
|
||||
client_revision: 54449
|
||||
client_version_major: 21
|
||||
client_version_minor: 8
|
||||
client_version_patch: 0
|
||||
http_method: 0
|
||||
http_user_agent:
|
||||
http_referer:
|
||||
forwarded_for:
|
||||
quota_key:
|
||||
revision: 54440
|
||||
thread_ids: []
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
Settings: {'background_pool_size':'32','load_balancing':'random','allow_suspicious_low_cardinality_types':'1','distributed_aggregation_memory_efficient':'1','skip_unavailable_shards':'1','log_queries':'1','max_bytes_before_external_group_by':'20000000000','max_bytes_before_external_sort':'20000000000','allow_introspection_functions':'1'}
|
||||
revision: 54453
|
||||
log_comment:
|
||||
thread_ids: [5058,22097,22110,22094]
|
||||
ProfileEvents.Names: ['Query','SelectQuery','ArenaAllocChunks','ArenaAllocBytes','FunctionExecute','NetworkSendElapsedMicroseconds','SelectedRows','SelectedBytes','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SystemTimeMicroseconds','SoftPageFaults','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSWriteChars']
|
||||
ProfileEvents.Values: [1,1,39,352256,64,360,8393,374325,412,440,34480,13108,4723,671,19,17828,8192,10240]
|
||||
Settings.Names: ['load_balancing','max_memory_usage']
|
||||
Settings.Values: ['random','10000000000']
|
||||
used_aggregate_functions: []
|
||||
used_aggregate_function_combinators: []
|
||||
used_database_engines: []
|
||||
used_data_type_families: ['UInt64','UInt8','Nullable','String','date']
|
||||
used_dictionaries: []
|
||||
used_formats: []
|
||||
used_functions: ['concat','notEmpty','extractAll']
|
||||
used_storages: []
|
||||
used_table_functions: []
|
||||
```
|
||||
|
||||
**Смотрите также**
|
||||
|
||||
@ -5,15 +5,26 @@ toc_title: "Манипуляции со столбцами"
|
||||
|
||||
# Манипуляции со столбцами {#manipuliatsii-so-stolbtsami}
|
||||
|
||||
Набор действий, позволяющих изменять структуру таблицы.
|
||||
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ...
|
||||
```
|
||||
|
||||
В запросе можно указать сразу несколько действий над одной таблицей через запятую.
|
||||
Каждое действие — это манипуляция над столбцом.
|
||||
|
||||
Существуют следующие действия:
|
||||
|
||||
- [ADD COLUMN](#alter_add-column) — добавляет столбец в таблицу;
|
||||
- [DROP COLUMN](#alter_drop-column) — удаляет столбец;
|
||||
- [RENAME COLUMN](#alter_rename-column) — переименовывает существующий столбец.
|
||||
- [CLEAR COLUMN](#alter_clear-column) — сбрасывает все значения в столбце для заданной партиции;
|
||||
- [COMMENT COLUMN](#alter_comment-column) — добавляет комментарий к столбцу;
|
||||
- [MODIFY COLUMN](#alter_modify-column) — изменяет тип столбца, выражение для значения по умолчанию и TTL.
|
||||
- [MODIFY COLUMN REMOVE](#modify-remove) — удаляет какое-либо из свойств столбца.
|
||||
- [RENAME COLUMN](#alter_rename-column) — переименовывает существующий столбец.
|
||||
|
||||
Подробное описание для каждого действия приведено ниже.
|
||||
|
||||
@ -72,6 +83,22 @@ DROP COLUMN [IF EXISTS] name
|
||||
ALTER TABLE visits DROP COLUMN browser
|
||||
```
|
||||
|
||||
## RENAME COLUMN {#alter_rename-column}
|
||||
|
||||
``` sql
|
||||
RENAME COLUMN [IF EXISTS] name to new_name
|
||||
```
|
||||
|
||||
Переименовывает столбец `name` в `new_name`. Если указано выражение `IF EXISTS`, то запрос не будет возвращать ошибку при условии, что столбец `name` не существует. Поскольку переименование не затрагивает физические данные колонки, запрос выполняется практически мгновенно.
|
||||
|
||||
**ЗАМЕЧЕНИЕ**: Столбцы, являющиеся частью основного ключа или ключа сортировки (заданные с помощью `ORDER BY` или `PRIMARY KEY`), не могут быть переименованы. Попытка переименовать эти слобцы приведет к `SQL Error [524]`.
|
||||
|
||||
Пример:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits RENAME COLUMN webBrowser TO browser
|
||||
```
|
||||
|
||||
## CLEAR COLUMN {#alter_clear-column}
|
||||
|
||||
``` sql
|
||||
@ -109,7 +136,7 @@ ALTER TABLE visits COMMENT COLUMN browser 'Столбец показывает,
|
||||
## MODIFY COLUMN {#alter_modify-column}
|
||||
|
||||
``` sql
|
||||
MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL] [AFTER name_after | FIRST]
|
||||
MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [codec] [TTL] [AFTER name_after | FIRST]
|
||||
```
|
||||
|
||||
Запрос изменяет следующие свойства столбца `name`:
|
||||
@ -118,11 +145,15 @@ MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL] [AFTER name_after | F
|
||||
|
||||
- Значение по умолчанию
|
||||
|
||||
- Кодеки сжатия
|
||||
|
||||
- TTL
|
||||
|
||||
Примеры изменения TTL столбца смотрите в разделе [TTL столбца](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-column-ttl).
|
||||
Примеры изменения кодеков сжатия смотрите в разделе [Кодеки сжатия столбцов](../create/table.md#codecs).
|
||||
|
||||
Если указано `IF EXISTS`, запрос не возвращает ошибку, если столбца не существует.
|
||||
Примеры изменения TTL столбца смотрите в разделе [TTL столбца](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-column-ttl).
|
||||
|
||||
Если указано `IF EXISTS`, запрос не возвращает ошибку при условии, что столбец не существует.
|
||||
|
||||
Запрос также может изменять порядок столбцов при помощи `FIRST | AFTER`, смотрите описание [ADD COLUMN](#alter_add-column).
|
||||
|
||||
@ -162,22 +193,6 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
|
||||
|
||||
- [REMOVE TTL](ttl.md).
|
||||
|
||||
## RENAME COLUMN {#alter_rename-column}
|
||||
|
||||
Переименовывает существующий столбец.
|
||||
|
||||
Синтаксис:
|
||||
|
||||
```sql
|
||||
ALTER TABLE table_name RENAME COLUMN column_name TO new_column_name
|
||||
```
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
ALTER TABLE table_with_ttl RENAME COLUMN column_ttl TO column_ttl_new;
|
||||
```
|
||||
|
||||
## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter}
|
||||
|
||||
Запрос `ALTER` позволяет создавать и удалять отдельные элементы (столбцы) вложенных структур данных, но не вложенные структуры данных целиком. Для добавления вложенной структуры данных, вы можете добавить столбцы с именем вида `name.nested_name` и типом `Array(T)` - вложенная структура данных полностью эквивалентна нескольким столбцам-массивам с именем, имеющим одинаковый префикс до точки.
|
||||
@ -186,7 +201,6 @@ ALTER TABLE table_with_ttl RENAME COLUMN column_ttl TO column_ttl_new;
|
||||
|
||||
Если возможностей запроса `ALTER` не хватает для нужного изменения таблицы, вы можете создать новую таблицу, скопировать туда данные с помощью запроса [INSERT SELECT](../insert-into.md#insert_query_insert-select), затем поменять таблицы местами с помощью запроса [RENAME](../misc.md#misc_operations-rename), и удалить старую таблицу. В качестве альтернативы для запроса `INSERT SELECT`, можно использовать инструмент [clickhouse-copier](../../../sql-reference/statements/alter/index.md).
|
||||
|
||||
Запрос `ALTER` блокирует все чтения и записи для таблицы. То есть, если на момент запроса `ALTER`, выполнялся долгий `SELECT`, то запрос `ALTER` сначала дождётся его выполнения. И в это время, все новые запросы к той же таблице, будут ждать, пока завершится этот `ALTER`.
|
||||
Запрос `ALTER` блокирует все чтения и записи для таблицы. То есть если на момент запроса `ALTER` выполнялся долгий `SELECT`, то запрос `ALTER` сначала дождётся его выполнения. И в это время все новые запросы к той же таблице будут ждать, пока завершится этот `ALTER`.
|
||||
|
||||
Для таблиц, которые не хранят данные самостоятельно (типа [Merge](../../../sql-reference/statements/alter/index.md) и [Distributed](../../../sql-reference/statements/alter/index.md)), `ALTER` всего лишь меняет структуру таблицы, но не меняет структуру подчинённых таблиц. Для примера, при ALTER-е таблицы типа `Distributed`, вам также потребуется выполнить запрос `ALTER` для таблиц на всех удалённых серверах.
|
||||
|
||||
|
||||
@ -62,7 +62,7 @@ CREATE TABLE example (
|
||||
materialized_value UInt32 MATERIALIZED 12345,
|
||||
aliased_value UInt32 ALIAS 2,
|
||||
PRIMARY KEY primary_key
|
||||
) ENGINE=MergeTree
|
||||
) ENGINE=MergeTree
|
||||
PARTITION BY partition_key
|
||||
ORDER BY (primary_key, secondary_key);
|
||||
```
|
||||
|
||||
@ -43,7 +43,7 @@ Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试
|
||||
- Monitoring
|
||||
- [Graphite](https://graphiteapp.org)
|
||||
- [graphouse](https://github.com/yandex/graphouse)
|
||||
- [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) +
|
||||
- [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse)
|
||||
- [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse)
|
||||
- [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - optimizes staled partitions in [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) if rules from [rollup configuration](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) could be applied
|
||||
- [Grafana](https://grafana.com/)
|
||||
|
||||
@ -6,4 +6,3 @@ toc_priority: 104
|
||||
|
||||
选择遇到的最后一个值。
|
||||
其结果和[any](../../../sql-reference/aggregate-functions/reference/any.md) 函数一样是不确定的 。
|
||||
|
||||
@ -11,7 +11,6 @@ set (CLICKHOUSE_COPIER_LINK
|
||||
clickhouse_functions
|
||||
clickhouse_table_functions
|
||||
clickhouse_aggregate_functions
|
||||
clickhouse_dictionaries
|
||||
string_utils
|
||||
|
||||
PUBLIC
|
||||
|
||||
@ -6,7 +6,6 @@ set (CLICKHOUSE_LOCAL_LINK
|
||||
clickhouse_aggregate_functions
|
||||
clickhouse_common_config
|
||||
clickhouse_common_io
|
||||
clickhouse_dictionaries
|
||||
clickhouse_functions
|
||||
clickhouse_parsers
|
||||
clickhouse_storages_system
|
||||
|
||||
@ -322,7 +322,7 @@ struct Checker
|
||||
{
|
||||
checkRequiredInstructions();
|
||||
}
|
||||
} checker;
|
||||
} checker __attribute__((init_priority(101))); /// Run before other static initializers.
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -13,7 +13,6 @@ set (CLICKHOUSE_SERVER_LINK
|
||||
clickhouse_common_config
|
||||
clickhouse_common_io
|
||||
clickhouse_common_zookeeper
|
||||
clickhouse_dictionaries
|
||||
clickhouse_functions
|
||||
clickhouse_parsers
|
||||
clickhouse_storages_system
|
||||
|
||||
@ -26,6 +26,7 @@
|
||||
#include <Common/DNSResolver.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/Macros.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
#include <Common/ZooKeeper/ZooKeeperNodeCache.h>
|
||||
@ -39,6 +40,7 @@
|
||||
#include <Common/remapExecutable.h>
|
||||
#include <Common/TLDListsHolder.h>
|
||||
#include <IO/HTTPCommon.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/UseSSL.h>
|
||||
#include <Interpreters/AsynchronousMetrics.h>
|
||||
#include <Interpreters/DDLWorker.h>
|
||||
@ -95,6 +97,9 @@
|
||||
#endif
|
||||
|
||||
#if USE_SSL
|
||||
# if USE_INTERNAL_SSL_LIBRARY
|
||||
# include <Compression/CompressionCodecEncrypted.h>
|
||||
# endif
|
||||
# include <Poco/Net/Context.h>
|
||||
# include <Poco/Net/SecureServerSocket.h>
|
||||
#endif
|
||||
@ -107,6 +112,10 @@
|
||||
# include <Server/KeeperTCPHandlerFactory.h>
|
||||
#endif
|
||||
|
||||
#if USE_BASE64
|
||||
# include <turbob64.h>
|
||||
#endif
|
||||
|
||||
#if USE_JEMALLOC
|
||||
# include <jemalloc/jemalloc.h>
|
||||
#endif
|
||||
@ -242,6 +251,7 @@ namespace ErrorCodes
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
|
||||
extern const int INCORRECT_DATA;
|
||||
extern const int INVALID_CONFIG_PARAMETER;
|
||||
extern const int SYSTEM_ERROR;
|
||||
extern const int FAILED_TO_GETPWUID;
|
||||
@ -445,6 +455,39 @@ void checkForUsersNotInMainConfig(
|
||||
}
|
||||
}
|
||||
|
||||
static void loadEncryptionKey(const std::string & key_command [[maybe_unused]], Poco::Logger * log)
|
||||
{
|
||||
#if USE_BASE64 && USE_SSL && USE_INTERNAL_SSL_LIBRARY
|
||||
|
||||
auto process = ShellCommand::execute(key_command);
|
||||
|
||||
std::string b64_key;
|
||||
readStringUntilEOF(b64_key, process->out);
|
||||
process->wait();
|
||||
|
||||
// turbob64 doesn't like whitespace characters in input. Strip
|
||||
// them before decoding.
|
||||
std::erase_if(b64_key, [](char c)
|
||||
{
|
||||
return c == ' ' || c == '\t' || c == '\r' || c == '\n';
|
||||
});
|
||||
|
||||
std::vector<char> buf(b64_key.size());
|
||||
const size_t key_size = tb64dec(reinterpret_cast<const unsigned char *>(b64_key.data()), b64_key.size(),
|
||||
reinterpret_cast<unsigned char *>(buf.data()));
|
||||
if (!key_size)
|
||||
throw Exception("Failed to decode encryption key", ErrorCodes::INCORRECT_DATA);
|
||||
else if (key_size < 16)
|
||||
LOG_WARNING(log, "The encryption key should be at least 16 octets long.");
|
||||
|
||||
const std::string_view key = std::string_view(buf.data(), key_size);
|
||||
CompressionCodecEncrypted::setMasterKey(key);
|
||||
|
||||
#else
|
||||
LOG_WARNING(log, "Server was built without Base64 or SSL support. Encryption is disabled.");
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
[[noreturn]] void forceShutdown()
|
||||
{
|
||||
@ -916,6 +959,10 @@ if (ThreadFuzzer::instance().isEffective())
|
||||
global_context->getMergeTreeSettings().sanityCheck(settings);
|
||||
global_context->getReplicatedMergeTreeSettings().sanityCheck(settings);
|
||||
|
||||
/// Set up encryption.
|
||||
if (config().has("encryption.key_command"))
|
||||
loadEncryptionKey(config().getString("encryption.key_command"), log);
|
||||
|
||||
Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0);
|
||||
|
||||
Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024));
|
||||
@ -1047,6 +1094,7 @@ if (ThreadFuzzer::instance().isEffective())
|
||||
loadMetadataSystem(global_context);
|
||||
/// After attaching system databases we can initialize system log.
|
||||
global_context->initializeSystemLogs();
|
||||
global_context->setSystemZooKeeperLogAfterInitializationIfNeeded();
|
||||
auto & database_catalog = DatabaseCatalog::instance();
|
||||
/// After the system database is created, attach virtual system tables (in addition to query_log and part_log)
|
||||
attachSystemTablesServer(*database_catalog.getSystemDatabase(), has_zookeeper);
|
||||
|
||||
@ -1002,6 +1002,16 @@
|
||||
</compression>
|
||||
-->
|
||||
|
||||
<!-- Configuration of encryption. The server executes a command to
|
||||
obtain an encryption key at startup if such a command is
|
||||
defined, or encryption codecs will be disabled otherwise. The
|
||||
command is executed through /bin/sh and is expected to write
|
||||
a Base64-encoded key to the stdout. -->
|
||||
<encryption>
|
||||
<!-- <key_command>/usr/bin/systemd-ask-password --id="clickhouse-server" --timeout=0 "Enter the ClickHouse encryption passphrase:" | base64</key_command> -->
|
||||
<!-- <key_command><![CDATA[IFS=; echo -n >/dev/tty "Enter the ClickHouse encryption passphrase: "; stty=`stty -F /dev/tty -g`; stty -F /dev/tty -echo; read k </dev/tty; stty -F /dev/tty "$stty"; echo -n $k | base64]]></key_command> -->
|
||||
</encryption>
|
||||
|
||||
<!-- Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster.
|
||||
Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -->
|
||||
<distributed_ddl>
|
||||
|
||||
@ -11,7 +11,7 @@ bool User::equal(const IAccessEntity & other) const
|
||||
const auto & other_user = typeid_cast<const User &>(other);
|
||||
return (authentication == other_user.authentication) && (allowed_client_hosts == other_user.allowed_client_hosts)
|
||||
&& (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles)
|
||||
&& (settings == other_user.settings) && (grantees == other_user.grantees);
|
||||
&& (settings == other_user.settings) && (grantees == other_user.grantees) && (default_database == other_user.default_database);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ struct User : public IAccessEntity
|
||||
RolesOrUsersSet default_roles = RolesOrUsersSet::AllTag{};
|
||||
SettingsProfileElements settings;
|
||||
RolesOrUsersSet grantees = RolesOrUsersSet::AllTag{};
|
||||
String default_database;
|
||||
|
||||
bool equal(const IAccessEntity & other) const override;
|
||||
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<User>(); }
|
||||
|
||||
@ -196,6 +196,9 @@ namespace
|
||||
user->access.revokeGrantOption(AccessType::ALL);
|
||||
}
|
||||
|
||||
String default_database = config.getString(user_config + ".default_database", "");
|
||||
user->default_database = default_database;
|
||||
|
||||
return user;
|
||||
}
|
||||
|
||||
|
||||
@ -43,9 +43,9 @@ public:
|
||||
const AggregateFunctionPtr & nested_function,
|
||||
const AggregateFunctionProperties &,
|
||||
const DataTypes & arguments,
|
||||
const Array &) const override
|
||||
const Array & params) const override
|
||||
{
|
||||
return std::make_shared<AggregateFunctionArray>(nested_function, arguments);
|
||||
return std::make_shared<AggregateFunctionArray>(nested_function, arguments, params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -29,10 +29,11 @@ private:
|
||||
size_t num_arguments;
|
||||
|
||||
public:
|
||||
AggregateFunctionArray(AggregateFunctionPtr nested_, const DataTypes & arguments)
|
||||
: IAggregateFunctionHelper<AggregateFunctionArray>(arguments, {})
|
||||
AggregateFunctionArray(AggregateFunctionPtr nested_, const DataTypes & arguments, const Array & params_)
|
||||
: IAggregateFunctionHelper<AggregateFunctionArray>(arguments, params_)
|
||||
, nested_func(nested_), num_arguments(arguments.size())
|
||||
{
|
||||
assert(parameters == nested_func->getParameters());
|
||||
for (const auto & type : arguments)
|
||||
if (!isArray(type))
|
||||
throw Exception("All arguments for aggregate function " + getName() + " must be arrays", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
@ -34,14 +34,14 @@ public:
|
||||
const AggregateFunctionPtr & nested_function,
|
||||
const AggregateFunctionProperties &,
|
||||
const DataTypes & arguments,
|
||||
const Array &) const override
|
||||
const Array & params) const override
|
||||
{
|
||||
AggregateFunctionPtr res;
|
||||
if (arguments.size() == 1)
|
||||
{
|
||||
res.reset(createWithNumericType<
|
||||
AggregateFunctionDistinct,
|
||||
AggregateFunctionDistinctSingleNumericData>(*arguments[0], nested_function, arguments));
|
||||
AggregateFunctionDistinctSingleNumericData>(*arguments[0], nested_function, arguments, params));
|
||||
|
||||
if (res)
|
||||
return res;
|
||||
@ -49,14 +49,14 @@ public:
|
||||
if (arguments[0]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
|
||||
return std::make_shared<
|
||||
AggregateFunctionDistinct<
|
||||
AggregateFunctionDistinctSingleGenericData<true>>>(nested_function, arguments);
|
||||
AggregateFunctionDistinctSingleGenericData<true>>>(nested_function, arguments, params);
|
||||
else
|
||||
return std::make_shared<
|
||||
AggregateFunctionDistinct<
|
||||
AggregateFunctionDistinctSingleGenericData<false>>>(nested_function, arguments);
|
||||
AggregateFunctionDistinctSingleGenericData<false>>>(nested_function, arguments, params);
|
||||
}
|
||||
|
||||
return std::make_shared<AggregateFunctionDistinct<AggregateFunctionDistinctMultipleGenericData>>(nested_function, arguments);
|
||||
return std::make_shared<AggregateFunctionDistinct<AggregateFunctionDistinctMultipleGenericData>>(nested_function, arguments, params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -167,8 +167,8 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
AggregateFunctionDistinct(AggregateFunctionPtr nested_func_, const DataTypes & arguments)
|
||||
: IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct>(arguments, nested_func_->getParameters())
|
||||
AggregateFunctionDistinct(AggregateFunctionPtr nested_func_, const DataTypes & arguments, const Array & params_)
|
||||
: IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct>(arguments, params_)
|
||||
, nested_func(nested_func_)
|
||||
, arguments_num(arguments.size()) {}
|
||||
|
||||
|
||||
@ -38,9 +38,9 @@ public:
|
||||
const AggregateFunctionPtr & nested_function,
|
||||
const AggregateFunctionProperties &,
|
||||
const DataTypes & arguments,
|
||||
const Array &) const override
|
||||
const Array & params) const override
|
||||
{
|
||||
return std::make_shared<AggregateFunctionForEach>(nested_function, arguments);
|
||||
return std::make_shared<AggregateFunctionForEach>(nested_function, arguments, params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -105,8 +105,8 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
AggregateFunctionForEach(AggregateFunctionPtr nested_, const DataTypes & arguments)
|
||||
: IAggregateFunctionDataHelper<AggregateFunctionForEachData, AggregateFunctionForEach>(arguments, {})
|
||||
AggregateFunctionForEach(AggregateFunctionPtr nested_, const DataTypes & arguments, const Array & params_)
|
||||
: IAggregateFunctionDataHelper<AggregateFunctionForEachData, AggregateFunctionForEach>(arguments, params_)
|
||||
, nested_func(nested_), num_arguments(arguments.size())
|
||||
{
|
||||
nested_size_of_data = nested_func->sizeOfData();
|
||||
|
||||
@ -25,8 +25,8 @@ template <typename HasLimit>
|
||||
class AggregateFunctionGroupUniqArrayDate : public AggregateFunctionGroupUniqArray<DataTypeDate::FieldType, HasLimit>
|
||||
{
|
||||
public:
|
||||
explicit AggregateFunctionGroupUniqArrayDate(const DataTypePtr & argument_type, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
: AggregateFunctionGroupUniqArray<DataTypeDate::FieldType, HasLimit>(argument_type, max_elems_) {}
|
||||
explicit AggregateFunctionGroupUniqArrayDate(const DataTypePtr & argument_type, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
: AggregateFunctionGroupUniqArray<DataTypeDate::FieldType, HasLimit>(argument_type, parameters_, max_elems_) {}
|
||||
DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<DataTypeDate>()); }
|
||||
};
|
||||
|
||||
@ -34,8 +34,8 @@ template <typename HasLimit>
|
||||
class AggregateFunctionGroupUniqArrayDateTime : public AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType, HasLimit>
|
||||
{
|
||||
public:
|
||||
explicit AggregateFunctionGroupUniqArrayDateTime(const DataTypePtr & argument_type, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
: AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType, HasLimit>(argument_type, max_elems_) {}
|
||||
explicit AggregateFunctionGroupUniqArrayDateTime(const DataTypePtr & argument_type, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
: AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType, HasLimit>(argument_type, parameters_, max_elems_) {}
|
||||
DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>()); }
|
||||
};
|
||||
|
||||
@ -102,9 +102,9 @@ AggregateFunctionPtr createAggregateFunctionGroupUniqArray(
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (!limit_size)
|
||||
return createAggregateFunctionGroupUniqArrayImpl<std::false_type>(name, argument_types[0]);
|
||||
return createAggregateFunctionGroupUniqArrayImpl<std::false_type>(name, argument_types[0], parameters);
|
||||
else
|
||||
return createAggregateFunctionGroupUniqArrayImpl<std::true_type>(name, argument_types[0], max_elems);
|
||||
return createAggregateFunctionGroupUniqArrayImpl<std::true_type>(name, argument_types[0], parameters, max_elems);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -48,9 +48,9 @@ private:
|
||||
using State = AggregateFunctionGroupUniqArrayData<T>;
|
||||
|
||||
public:
|
||||
AggregateFunctionGroupUniqArray(const DataTypePtr & argument_type, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
AggregateFunctionGroupUniqArray(const DataTypePtr & argument_type, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
: IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayData<T>,
|
||||
AggregateFunctionGroupUniqArray<T, Tlimit_num_elem>>({argument_type}, {}),
|
||||
AggregateFunctionGroupUniqArray<T, Tlimit_num_elem>>({argument_type}, parameters_),
|
||||
max_elems(max_elems_) {}
|
||||
|
||||
String getName() const override { return "groupUniqArray"; }
|
||||
@ -152,8 +152,8 @@ class AggregateFunctionGroupUniqArrayGeneric
|
||||
using State = AggregateFunctionGroupUniqArrayGenericData;
|
||||
|
||||
public:
|
||||
AggregateFunctionGroupUniqArrayGeneric(const DataTypePtr & input_data_type_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
: IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayGenericData, AggregateFunctionGroupUniqArrayGeneric<is_plain_column, Tlimit_num_elem>>({input_data_type_}, {})
|
||||
AggregateFunctionGroupUniqArrayGeneric(const DataTypePtr & input_data_type_, const Array & parameters_, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
: IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayGenericData, AggregateFunctionGroupUniqArrayGeneric<is_plain_column, Tlimit_num_elem>>({input_data_type_}, parameters_)
|
||||
, input_data_type(this->argument_types[0])
|
||||
, max_elems(max_elems_) {}
|
||||
|
||||
|
||||
@ -35,9 +35,9 @@ public:
|
||||
const AggregateFunctionPtr & nested_function,
|
||||
const AggregateFunctionProperties &,
|
||||
const DataTypes & arguments,
|
||||
const Array &) const override
|
||||
const Array & params) const override
|
||||
{
|
||||
return std::make_shared<AggregateFunctionIf>(nested_function, arguments);
|
||||
return std::make_shared<AggregateFunctionIf>(nested_function, arguments, params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -37,8 +37,8 @@ private:
|
||||
size_t num_arguments;
|
||||
|
||||
public:
|
||||
AggregateFunctionIf(AggregateFunctionPtr nested, const DataTypes & types)
|
||||
: IAggregateFunctionHelper<AggregateFunctionIf>(types, nested->getParameters())
|
||||
AggregateFunctionIf(AggregateFunctionPtr nested, const DataTypes & types, const Array & params_)
|
||||
: IAggregateFunctionHelper<AggregateFunctionIf>(types, params_)
|
||||
, nested_func(nested), num_arguments(types.size())
|
||||
{
|
||||
if (num_arguments == 0)
|
||||
|
||||
@ -39,7 +39,7 @@ public:
|
||||
const AggregateFunctionPtr & nested_function,
|
||||
const AggregateFunctionProperties &,
|
||||
const DataTypes & arguments,
|
||||
const Array &) const override
|
||||
const Array & params) const override
|
||||
{
|
||||
const DataTypePtr & argument = arguments[0];
|
||||
|
||||
@ -53,7 +53,7 @@ public:
|
||||
+ ", because it corresponds to different aggregate function: " + function->getFunctionName() + " instead of " + nested_function->getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<AggregateFunctionMerge>(nested_function, argument);
|
||||
return std::make_shared<AggregateFunctionMerge>(nested_function, argument, params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -29,15 +29,15 @@ private:
|
||||
AggregateFunctionPtr nested_func;
|
||||
|
||||
public:
|
||||
AggregateFunctionMerge(const AggregateFunctionPtr & nested_, const DataTypePtr & argument)
|
||||
: IAggregateFunctionHelper<AggregateFunctionMerge>({argument}, nested_->getParameters())
|
||||
AggregateFunctionMerge(const AggregateFunctionPtr & nested_, const DataTypePtr & argument, const Array & params_)
|
||||
: IAggregateFunctionHelper<AggregateFunctionMerge>({argument}, params_)
|
||||
, nested_func(nested_)
|
||||
{
|
||||
const DataTypeAggregateFunction * data_type = typeid_cast<const DataTypeAggregateFunction *>(argument.get());
|
||||
|
||||
if (!data_type || data_type->getFunctionName() != nested_func->getName())
|
||||
throw Exception("Illegal type " + argument->getName() + " of argument for aggregate function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
if (!data_type || !nested_func->haveSameStateRepresentation(*data_type->getFunction()))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument for aggregate function {}, "
|
||||
"expected {} or equivalent type", argument->getName(), getName(), getStateType()->getName());
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
|
||||
@ -105,6 +105,11 @@ public:
|
||||
return res;
|
||||
}
|
||||
|
||||
bool haveSameStateRepresentation(const IAggregateFunction & rhs) const override
|
||||
{
|
||||
return getName() == rhs.getName() && this->haveEqualArgumentTypes(rhs);
|
||||
}
|
||||
|
||||
bool allocatesMemoryInArena() const override { return false; }
|
||||
|
||||
void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||
|
||||
@ -179,6 +179,11 @@ public:
|
||||
this->data(place).deserialize(buf);
|
||||
}
|
||||
|
||||
bool haveSameStateRepresentation(const IAggregateFunction & rhs) const override
|
||||
{
|
||||
return this->getName() == rhs.getName() && this->haveEqualArgumentTypes(rhs);
|
||||
}
|
||||
|
||||
private:
|
||||
enum class PatternActionType
|
||||
{
|
||||
|
||||
@ -31,10 +31,10 @@ namespace
|
||||
|
||||
template <typename T>
|
||||
inline AggregateFunctionPtr createAggregateFunctionSequenceNodeImpl(
|
||||
const DataTypePtr data_type, const DataTypes & argument_types, SequenceDirection direction, SequenceBase base)
|
||||
const DataTypePtr data_type, const DataTypes & argument_types, const Array & parameters, SequenceDirection direction, SequenceBase base)
|
||||
{
|
||||
return std::make_shared<SequenceNextNodeImpl<T, NodeString<max_events_size>>>(
|
||||
data_type, argument_types, base, direction, min_required_args);
|
||||
data_type, argument_types, parameters, base, direction, min_required_args);
|
||||
}
|
||||
|
||||
AggregateFunctionPtr
|
||||
@ -116,17 +116,17 @@ createAggregateFunctionSequenceNode(const std::string & name, const DataTypes &
|
||||
|
||||
WhichDataType timestamp_type(argument_types[0].get());
|
||||
if (timestamp_type.idx == TypeIndex::UInt8)
|
||||
return createAggregateFunctionSequenceNodeImpl<UInt8>(data_type, argument_types, direction, base);
|
||||
return createAggregateFunctionSequenceNodeImpl<UInt8>(data_type, argument_types, parameters, direction, base);
|
||||
if (timestamp_type.idx == TypeIndex::UInt16)
|
||||
return createAggregateFunctionSequenceNodeImpl<UInt16>(data_type, argument_types, direction, base);
|
||||
return createAggregateFunctionSequenceNodeImpl<UInt16>(data_type, argument_types, parameters, direction, base);
|
||||
if (timestamp_type.idx == TypeIndex::UInt32)
|
||||
return createAggregateFunctionSequenceNodeImpl<UInt32>(data_type, argument_types, direction, base);
|
||||
return createAggregateFunctionSequenceNodeImpl<UInt32>(data_type, argument_types, parameters, direction, base);
|
||||
if (timestamp_type.idx == TypeIndex::UInt64)
|
||||
return createAggregateFunctionSequenceNodeImpl<UInt64>(data_type, argument_types, direction, base);
|
||||
return createAggregateFunctionSequenceNodeImpl<UInt64>(data_type, argument_types, parameters, direction, base);
|
||||
if (timestamp_type.isDate())
|
||||
return createAggregateFunctionSequenceNodeImpl<DataTypeDate::FieldType>(data_type, argument_types, direction, base);
|
||||
return createAggregateFunctionSequenceNodeImpl<DataTypeDate::FieldType>(data_type, argument_types, parameters, direction, base);
|
||||
if (timestamp_type.isDateTime())
|
||||
return createAggregateFunctionSequenceNodeImpl<DataTypeDateTime::FieldType>(data_type, argument_types, direction, base);
|
||||
return createAggregateFunctionSequenceNodeImpl<DataTypeDateTime::FieldType>(data_type, argument_types, parameters, direction, base);
|
||||
|
||||
throw Exception{"Illegal type " + argument_types.front().get()->getName()
|
||||
+ " of first argument of aggregate function " + name + ", must be Unsigned Number, Date, DateTime",
|
||||
|
||||
@ -175,11 +175,12 @@ public:
|
||||
SequenceNextNodeImpl(
|
||||
const DataTypePtr & data_type_,
|
||||
const DataTypes & arguments,
|
||||
const Array & parameters_,
|
||||
SequenceBase seq_base_kind_,
|
||||
SequenceDirection seq_direction_,
|
||||
size_t min_required_args_,
|
||||
UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
|
||||
: IAggregateFunctionDataHelper<SequenceNextNodeGeneralData<Node>, Self>({data_type_}, {})
|
||||
: IAggregateFunctionDataHelper<SequenceNextNodeGeneralData<Node>, Self>({data_type_}, parameters_)
|
||||
, seq_base_kind(seq_base_kind_)
|
||||
, seq_direction(seq_direction_)
|
||||
, min_required_args(min_required_args_)
|
||||
@ -193,6 +194,11 @@ public:
|
||||
|
||||
DataTypePtr getReturnType() const override { return data_type; }
|
||||
|
||||
bool haveSameStateRepresentation(const IAggregateFunction & rhs) const override
|
||||
{
|
||||
return this->getName() == rhs.getName() && this->haveEqualArgumentTypes(rhs);
|
||||
}
|
||||
|
||||
AggregateFunctionPtr getOwnNullAdapter(
|
||||
const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params,
|
||||
const AggregateFunctionProperties &) const override
|
||||
|
||||
@ -50,4 +50,21 @@ String IAggregateFunction::getDescription() const
|
||||
|
||||
return description;
|
||||
}
|
||||
|
||||
bool IAggregateFunction::haveEqualArgumentTypes(const IAggregateFunction & rhs) const
|
||||
{
|
||||
return std::equal(argument_types.begin(), argument_types.end(),
|
||||
rhs.argument_types.begin(), rhs.argument_types.end(),
|
||||
[](const auto & t1, const auto & t2) { return t1->equals(*t2); });
|
||||
}
|
||||
|
||||
bool IAggregateFunction::haveSameStateRepresentation(const IAggregateFunction & rhs) const
|
||||
{
|
||||
bool res = getName() == rhs.getName()
|
||||
&& parameters == rhs.parameters
|
||||
&& haveEqualArgumentTypes(rhs);
|
||||
assert(res == (getStateType()->getName() == rhs.getStateType()->getName()));
|
||||
return res;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -74,6 +74,16 @@ public:
|
||||
/// Get the data type of internal state. By default it is AggregateFunction(name(params), argument_types...).
|
||||
virtual DataTypePtr getStateType() const;
|
||||
|
||||
/// Returns true if two aggregate functions have the same state representation in memory and the same serialization,
|
||||
/// so state of one aggregate function can be safely used with another.
|
||||
/// Examples:
|
||||
/// - quantile(x), quantile(a)(x), quantile(b)(x) - parameter doesn't affect state and used for finalization only
|
||||
/// - foo(x) and fooIf(x) - If combinator doesn't affect state
|
||||
/// By default returns true only if functions have exactly the same names, combinators and parameters.
|
||||
virtual bool haveSameStateRepresentation(const IAggregateFunction & rhs) const;
|
||||
|
||||
bool haveEqualArgumentTypes(const IAggregateFunction & rhs) const;
|
||||
|
||||
/// Get type which will be used for prediction result in case if function is an ML method.
|
||||
virtual DataTypePtr getReturnTypeToPredict() const
|
||||
{
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <DataStreams/OneBlockInputStream.h>
|
||||
#include <DataStreams/OwningBlockInputStream.h>
|
||||
#include <DataStreams/formatBlock.h>
|
||||
#include <Dictionaries/DictionarySourceHelpers.h>
|
||||
#include <Processors/Formats/InputStreamFromInputFormat.h>
|
||||
#include <IO/WriteBufferFromOStream.h>
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Client/HedgedConnections.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Interpreters/ClientInfo.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
@ -21,13 +22,14 @@ namespace ErrorCodes
|
||||
|
||||
HedgedConnections::HedgedConnections(
|
||||
const ConnectionPoolWithFailoverPtr & pool_,
|
||||
const Settings & settings_,
|
||||
ContextPtr context_,
|
||||
const ConnectionTimeouts & timeouts_,
|
||||
const ThrottlerPtr & throttler_,
|
||||
PoolMode pool_mode,
|
||||
std::shared_ptr<QualifiedTableName> table_to_check_)
|
||||
: hedged_connections_factory(pool_, &settings_, timeouts_, table_to_check_)
|
||||
, settings(settings_)
|
||||
: hedged_connections_factory(pool_, &context_->getSettingsRef(), timeouts_, table_to_check_)
|
||||
, context(std::move(context_))
|
||||
, settings(context->getSettingsRef())
|
||||
, drain_timeout(settings.drain_timeout)
|
||||
, allow_changing_replica_until_first_data_packet(settings.allow_changing_replica_until_first_data_packet)
|
||||
, throttler(throttler_)
|
||||
|
||||
@ -72,7 +72,7 @@ public:
|
||||
};
|
||||
|
||||
HedgedConnections(const ConnectionPoolWithFailoverPtr & pool_,
|
||||
const Settings & settings_,
|
||||
ContextPtr context_,
|
||||
const ConnectionTimeouts & timeouts_,
|
||||
const ThrottlerPtr & throttler,
|
||||
PoolMode pool_mode,
|
||||
@ -188,6 +188,7 @@ private:
|
||||
Packet last_received_packet;
|
||||
|
||||
Epoll epoll;
|
||||
ContextPtr context;
|
||||
const Settings & settings;
|
||||
|
||||
/// The following two fields are from settings but can be referenced outside the lifetime of
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <string_view>
|
||||
#include <unordered_set>
|
||||
|
||||
@ -11,11 +12,10 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int TLD_LIST_NOT_FOUND;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
///
|
||||
/// TLDList
|
||||
///
|
||||
TLDList::TLDList(size_t size)
|
||||
: tld_container(size)
|
||||
, pool(std::make_unique<Arena>(10 << 20))
|
||||
@ -31,9 +31,7 @@ bool TLDList::has(const StringRef & host) const
|
||||
return tld_container.has(host);
|
||||
}
|
||||
|
||||
///
|
||||
/// TLDListsHolder
|
||||
///
|
||||
TLDListsHolder & TLDListsHolder::getInstance()
|
||||
{
|
||||
static TLDListsHolder instance;
|
||||
@ -62,24 +60,22 @@ size_t TLDListsHolder::parseAndAddTldList(const std::string & name, const std::s
|
||||
std::unordered_set<std::string> tld_list_tmp;
|
||||
|
||||
ReadBufferFromFile in(path);
|
||||
String line;
|
||||
while (!in.eof())
|
||||
{
|
||||
char * newline = find_first_symbols<'\n'>(in.position(), in.buffer().end());
|
||||
if (newline >= in.buffer().end())
|
||||
break;
|
||||
|
||||
std::string_view line(in.position(), newline - in.position());
|
||||
in.position() = newline + 1;
|
||||
|
||||
readEscapedStringUntilEOL(line, in);
|
||||
++in.position();
|
||||
/// Skip comments
|
||||
if (line.size() > 2 && line[0] == '/' && line[1] == '/')
|
||||
continue;
|
||||
trim(line);
|
||||
line = trim(line, [](char c) { return std::isspace(c); });
|
||||
/// Skip empty line
|
||||
if (line.empty())
|
||||
continue;
|
||||
tld_list_tmp.emplace(line);
|
||||
}
|
||||
if (!in.eof())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not all list had been read", name);
|
||||
|
||||
TLDList tld_list(tld_list_tmp.size());
|
||||
for (const auto & host : tld_list_tmp)
|
||||
|
||||
@ -135,8 +135,10 @@ void ZooKeeper::init(const std::string & implementation_, const Strings & hosts_
|
||||
}
|
||||
|
||||
ZooKeeper::ZooKeeper(const std::string & hosts_string, const std::string & identity_, int32_t session_timeout_ms_,
|
||||
int32_t operation_timeout_ms_, const std::string & chroot_, const std::string & implementation_)
|
||||
int32_t operation_timeout_ms_, const std::string & chroot_, const std::string & implementation_,
|
||||
std::shared_ptr<DB::ZooKeeperLog> zk_log_)
|
||||
{
|
||||
zk_log = std::move(zk_log_);
|
||||
Strings hosts_strings;
|
||||
splitInto<','>(hosts_strings, hosts_string);
|
||||
|
||||
@ -144,8 +146,10 @@ ZooKeeper::ZooKeeper(const std::string & hosts_string, const std::string & ident
|
||||
}
|
||||
|
||||
ZooKeeper::ZooKeeper(const Strings & hosts_, const std::string & identity_, int32_t session_timeout_ms_,
|
||||
int32_t operation_timeout_ms_, const std::string & chroot_, const std::string & implementation_)
|
||||
int32_t operation_timeout_ms_, const std::string & chroot_, const std::string & implementation_,
|
||||
std::shared_ptr<DB::ZooKeeperLog> zk_log_)
|
||||
{
|
||||
zk_log = std::move(zk_log_);
|
||||
init(implementation_, hosts_, identity_, session_timeout_ms_, operation_timeout_ms_, chroot_);
|
||||
}
|
||||
|
||||
@ -729,7 +733,7 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition &
|
||||
|
||||
ZooKeeperPtr ZooKeeper::startNewSession() const
|
||||
{
|
||||
return std::make_shared<ZooKeeper>(hosts, identity, session_timeout_ms, operation_timeout_ms, chroot, implementation);
|
||||
return std::make_shared<ZooKeeper>(hosts, identity, session_timeout_ms, operation_timeout_ms, chroot, implementation, zk_log);
|
||||
}
|
||||
|
||||
|
||||
@ -1020,6 +1024,14 @@ void ZooKeeper::finalize()
|
||||
impl->finalize();
|
||||
}
|
||||
|
||||
void ZooKeeper::setZooKeeperLog(std::shared_ptr<DB::ZooKeeperLog> zk_log_)
|
||||
{
|
||||
zk_log = std::move(zk_log_);
|
||||
if (auto * zk = dynamic_cast<Coordination::ZooKeeper *>(impl.get()))
|
||||
zk->setZooKeeperLog(zk_log);
|
||||
}
|
||||
|
||||
|
||||
size_t KeeperMultiException::getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses)
|
||||
{
|
||||
if (responses.empty())
|
||||
|
||||
@ -56,13 +56,15 @@ public:
|
||||
int32_t session_timeout_ms_ = Coordination::DEFAULT_SESSION_TIMEOUT_MS,
|
||||
int32_t operation_timeout_ms_ = Coordination::DEFAULT_OPERATION_TIMEOUT_MS,
|
||||
const std::string & chroot_ = "",
|
||||
const std::string & implementation_ = "zookeeper");
|
||||
const std::string & implementation_ = "zookeeper",
|
||||
std::shared_ptr<DB::ZooKeeperLog> zk_log_ = nullptr);
|
||||
|
||||
ZooKeeper(const Strings & hosts_, const std::string & identity_ = "",
|
||||
int32_t session_timeout_ms_ = Coordination::DEFAULT_SESSION_TIMEOUT_MS,
|
||||
int32_t operation_timeout_ms_ = Coordination::DEFAULT_OPERATION_TIMEOUT_MS,
|
||||
const std::string & chroot_ = "",
|
||||
const std::string & implementation_ = "zookeeper");
|
||||
const std::string & implementation_ = "zookeeper",
|
||||
std::shared_ptr<DB::ZooKeeperLog> zk_log_ = nullptr);
|
||||
|
||||
/** Config of the form:
|
||||
<zookeeper>
|
||||
@ -273,6 +275,8 @@ public:
|
||||
|
||||
void finalize();
|
||||
|
||||
void setZooKeeperLog(std::shared_ptr<DB::ZooKeeperLog> zk_log_);
|
||||
|
||||
private:
|
||||
friend class EphemeralNodeHolder;
|
||||
|
||||
|
||||
@ -315,9 +315,10 @@ ZooKeeper::ZooKeeper(
|
||||
std::shared_ptr<ZooKeeperLog> zk_log_)
|
||||
: root_path(root_path_),
|
||||
session_timeout(session_timeout_),
|
||||
operation_timeout(std::min(operation_timeout_, session_timeout_)),
|
||||
zk_log(std::move(zk_log_))
|
||||
operation_timeout(std::min(operation_timeout_, session_timeout_))
|
||||
{
|
||||
std::atomic_store(&zk_log, std::move(zk_log_));
|
||||
|
||||
if (!root_path.empty())
|
||||
{
|
||||
if (root_path.back() == '/')
|
||||
@ -1212,9 +1213,16 @@ void ZooKeeper::close()
|
||||
}
|
||||
|
||||
|
||||
void ZooKeeper::setZooKeeperLog(std::shared_ptr<DB::ZooKeeperLog> zk_log_)
|
||||
{
|
||||
/// logOperationIfNeeded(...) uses zk_log and can be called from different threads, so we have to use atomic shared_ptr
|
||||
std::atomic_store(&zk_log, std::move(zk_log_));
|
||||
}
|
||||
|
||||
void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response, bool finalize)
|
||||
{
|
||||
if (!zk_log)
|
||||
auto maybe_zk_log = std::atomic_load(&zk_log);
|
||||
if (!maybe_zk_log)
|
||||
return;
|
||||
|
||||
ZooKeeperLogElement::Type log_type = ZooKeeperLogElement::UNKNOWN;
|
||||
@ -1249,7 +1257,7 @@ void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const
|
||||
elem.event_time = event_time;
|
||||
elem.address = socket.peerAddress();
|
||||
elem.session_id = session_id;
|
||||
zk_log->add(elem);
|
||||
maybe_zk_log->add(elem);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -189,6 +189,8 @@ public:
|
||||
|
||||
void finalize() override { finalize(false, false); }
|
||||
|
||||
void setZooKeeperLog(std::shared_ptr<DB::ZooKeeperLog> zk_log_);
|
||||
|
||||
private:
|
||||
String root_path;
|
||||
ACLs default_acls;
|
||||
|
||||
@ -2,8 +2,10 @@
|
||||
|
||||
// .h autogenerated by cmake!
|
||||
|
||||
#cmakedefine01 USE_BASE64
|
||||
#cmakedefine01 USE_RE2_ST
|
||||
#cmakedefine01 USE_SSL
|
||||
#cmakedefine01 USE_INTERNAL_SSL_LIBRARY
|
||||
#cmakedefine01 USE_HDFS
|
||||
#cmakedefine01 USE_INTERNAL_HDFS3_LIBRARY
|
||||
#cmakedefine01 USE_AWS_S3
|
||||
|
||||
212
src/Compression/CompressionCodecEncrypted.cpp
Normal file
212
src/Compression/CompressionCodecEncrypted.cpp
Normal file
@ -0,0 +1,212 @@
|
||||
#include <Common/config.h>
|
||||
#include <Compression/CompressionFactory.h>
|
||||
#if USE_SSL && USE_INTERNAL_SSL_LIBRARY
|
||||
|
||||
#include <Compression/CompressionCodecEncrypted.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <cassert>
|
||||
#include <openssl/digest.h>
|
||||
#include <openssl/err.h>
|
||||
#include <openssl/hkdf.h>
|
||||
#include <string_view>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_CODEC_PARAMETER;
|
||||
extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
|
||||
extern const int NO_ELEMENTS_IN_CONFIG;
|
||||
extern const int OPENSSL_ERROR;
|
||||
}
|
||||
|
||||
void CompressionCodecEncrypted::setMasterKey(const std::string_view & master_key)
|
||||
{
|
||||
keys.emplace(master_key);
|
||||
}
|
||||
|
||||
CompressionCodecEncrypted::KeyHolder::KeyHolder(const std::string_view & master_key)
|
||||
{
|
||||
// Derive a key from it.
|
||||
keygen_key = deriveKey(master_key);
|
||||
|
||||
// EVP_AEAD_CTX is not stateful so we can create an
|
||||
// instance now.
|
||||
EVP_AEAD_CTX_zero(&ctx);
|
||||
const int ok = EVP_AEAD_CTX_init(&ctx, EVP_aead_aes_128_gcm(),
|
||||
reinterpret_cast<const uint8_t*>(keygen_key.data()), keygen_key.size(),
|
||||
16 /* tag size */, nullptr);
|
||||
if (!ok)
|
||||
throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
}
|
||||
|
||||
CompressionCodecEncrypted::KeyHolder::~KeyHolder()
|
||||
{
|
||||
EVP_AEAD_CTX_cleanup(&ctx);
|
||||
}
|
||||
|
||||
const CompressionCodecEncrypted::KeyHolder & CompressionCodecEncrypted::getKeys()
|
||||
{
|
||||
if (keys)
|
||||
return *keys;
|
||||
else
|
||||
throw Exception("There is no configuration for encryption in the server config",
|
||||
ErrorCodes::NO_ELEMENTS_IN_CONFIG);
|
||||
}
|
||||
|
||||
CompressionCodecEncrypted::CompressionCodecEncrypted(const std::string_view & cipher)
|
||||
{
|
||||
setCodecDescription("Encrypted", {std::make_shared<ASTLiteral>(cipher)});
|
||||
}
|
||||
|
||||
uint8_t CompressionCodecEncrypted::getMethodByte() const
|
||||
{
|
||||
return static_cast<uint8_t>(CompressionMethodByte::Encrypted);
|
||||
}
|
||||
|
||||
void CompressionCodecEncrypted::updateHash(SipHash & hash) const
|
||||
{
|
||||
getCodecDesc()->updateTreeHash(hash);
|
||||
}
|
||||
|
||||
UInt32 CompressionCodecEncrypted::getMaxCompressedDataSize(UInt32 uncompressed_size) const
|
||||
{
|
||||
// The GCM mode is a stream cipher. No paddings are
|
||||
// involved. There will be a tag at the end of ciphertext (16
|
||||
// octets).
|
||||
return uncompressed_size + 16;
|
||||
}
|
||||
|
||||
UInt32 CompressionCodecEncrypted::doCompressData(const char * source, UInt32 source_size, char * dest) const
|
||||
{
|
||||
// Generate an IV out of the data block and the key-generation
|
||||
// key. It is completely deterministic, but does not leak any
|
||||
// information about the data block except for equivalence of
|
||||
// identical blocks (under the same master key). The IV will
|
||||
// be used as an authentication tag. The ciphertext and the
|
||||
// tag will be written directly in the dest buffer.
|
||||
const std::string_view plaintext = std::string_view(source, source_size);
|
||||
|
||||
encrypt(plaintext, dest);
|
||||
return source_size + 16;
|
||||
}
|
||||
|
||||
void CompressionCodecEncrypted::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size [[maybe_unused]]) const
|
||||
{
|
||||
// Extract the IV from the encrypted data block. Decrypt the
|
||||
// block with the extracted IV, and compare the tag. Throw an
|
||||
// exception if tags don't match.
|
||||
const std::string_view ciphertext_and_tag = std::string_view(source, source_size);
|
||||
assert(ciphertext_and_tag.size() == uncompressed_size + 16);
|
||||
|
||||
decrypt(ciphertext_and_tag, dest);
|
||||
}
|
||||
|
||||
std::string CompressionCodecEncrypted::lastErrorString()
|
||||
{
|
||||
std::array<char, 1024> buffer;
|
||||
ERR_error_string_n(ERR_get_error(), buffer.data(), buffer.size());
|
||||
return std::string(buffer.data());
|
||||
}
|
||||
|
||||
std::string CompressionCodecEncrypted::deriveKey(const std::string_view & master_key)
|
||||
{
|
||||
std::string_view salt(""); // No salt: derive keys in a deterministic manner.
|
||||
std::string_view info("Codec Encrypted('AES-128-GCM-SIV') key generation key");
|
||||
std::array<char, 32> result;
|
||||
|
||||
const int ok = HKDF(reinterpret_cast<uint8_t *>(result.data()), result.size(),
|
||||
EVP_sha256(),
|
||||
reinterpret_cast<const uint8_t *>(master_key.data()), master_key.size(),
|
||||
reinterpret_cast<const uint8_t *>(salt.data()), salt.size(),
|
||||
reinterpret_cast<const uint8_t *>(info.data()), info.size());
|
||||
if (!ok)
|
||||
throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
return std::string(result.data(), 16);
|
||||
}
|
||||
|
||||
void CompressionCodecEncrypted::encrypt(const std::string_view & plaintext, char * ciphertext_and_tag)
|
||||
{
|
||||
// Fixed nonce. Yes this is unrecommended, but we have to live
|
||||
// with it.
|
||||
std::string_view nonce("\0\0\0\0\0\0\0\0\0\0\0\0", 12);
|
||||
|
||||
size_t out_len;
|
||||
const int ok = EVP_AEAD_CTX_seal(&getKeys().ctx,
|
||||
reinterpret_cast<uint8_t *>(ciphertext_and_tag),
|
||||
&out_len, plaintext.size() + 16,
|
||||
reinterpret_cast<const uint8_t *>(nonce.data()), nonce.size(),
|
||||
reinterpret_cast<const uint8_t *>(plaintext.data()), plaintext.size(),
|
||||
nullptr, 0);
|
||||
if (!ok)
|
||||
throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
assert(out_len == plaintext.size() + 16);
|
||||
}
|
||||
|
||||
void CompressionCodecEncrypted::decrypt(const std::string_view & ciphertext, char * plaintext)
|
||||
{
|
||||
std::string_view nonce("\0\0\0\0\0\0\0\0\0\0\0\0", 12);
|
||||
|
||||
size_t out_len;
|
||||
const int ok = EVP_AEAD_CTX_open(&getKeys().ctx,
|
||||
reinterpret_cast<uint8_t *>(plaintext),
|
||||
&out_len, ciphertext.size(),
|
||||
reinterpret_cast<const uint8_t *>(nonce.data()), nonce.size(),
|
||||
reinterpret_cast<const uint8_t *>(ciphertext.data()), ciphertext.size(),
|
||||
nullptr, 0);
|
||||
if (!ok)
|
||||
throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
assert(out_len == ciphertext.size() - 16);
|
||||
}
|
||||
|
||||
void registerCodecEncrypted(CompressionCodecFactory & factory)
|
||||
{
|
||||
const auto method_code = uint8_t(CompressionMethodByte::Encrypted);
|
||||
factory.registerCompressionCodec("Encrypted", method_code, [&](const ASTPtr & arguments) -> CompressionCodecPtr
|
||||
{
|
||||
if (arguments)
|
||||
{
|
||||
if (arguments->children.size() != 1)
|
||||
throw Exception("Codec Encrypted() must have 1 parameter, given " +
|
||||
std::to_string(arguments->children.size()),
|
||||
ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE);
|
||||
|
||||
const auto children = arguments->children;
|
||||
const auto * literal = children[0]->as<ASTLiteral>();
|
||||
if (!literal)
|
||||
throw Exception("Wrong argument for codec Encrypted(). Expected a string literal",
|
||||
ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE);
|
||||
|
||||
const String cipher = literal->value.safeGet<String>();
|
||||
if (cipher == "AES-128-GCM-SIV")
|
||||
return std::make_shared<CompressionCodecEncrypted>(cipher);
|
||||
else
|
||||
throw Exception("Cipher '" + cipher + "' is not supported",
|
||||
ErrorCodes::ILLEGAL_CODEC_PARAMETER);
|
||||
}
|
||||
else
|
||||
{
|
||||
/* The factory is asking us to construct the codec
|
||||
* only from the method code. How can that be
|
||||
* possible? For now we only support a single cipher
|
||||
* so it's not really a problem, but if we were to
|
||||
* support more ciphers it would be catastrophic. */
|
||||
return std::make_shared<CompressionCodecEncrypted>("AES-128-GCM-SIV");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#else /* USE_SSL && USE_INTERNAL_SSL_LIBRARY */
|
||||
|
||||
namespace DB
|
||||
{
|
||||
void registerCodecEncrypted(CompressionCodecFactory &)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* USE_SSL && USE_INTERNAL_SSL_LIBRARY */
|
||||
104
src/Compression/CompressionCodecEncrypted.h
Normal file
104
src/Compression/CompressionCodecEncrypted.h
Normal file
@ -0,0 +1,104 @@
|
||||
#pragma once
|
||||
|
||||
// This depends on BoringSSL-specific API, notably <openssl/aead.h>.
|
||||
#include <Common/config.h>
|
||||
#if USE_SSL && USE_INTERNAL_SSL_LIBRARY
|
||||
|
||||
#include <Compression/ICompressionCodec.h>
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <openssl/aead.h>
|
||||
#include <optional>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
/** This codec encrypts and decrypts blocks with AES-128 in
|
||||
* GCM-SIV mode (RFC-8452), which is the only cipher currently
|
||||
* supported. Although it is implemented as a compression codec
|
||||
* it doesn't actually compress data. In fact encrypted data will
|
||||
* no longer be compressible in any meaningful way. This means if
|
||||
* you want to apply both compression and encryption to your
|
||||
* columns, you need to put this codec at the end of the chain
|
||||
* like "column Int32 Codec(Delta, LZ4,
|
||||
* Encrypted('AES-128-GCM-SIV'))".
|
||||
*
|
||||
* The key is obtained by executing a command specified in the
|
||||
* configuration file at startup, and if it doesn't specify a
|
||||
* command the codec refuses to process any data. The command is
|
||||
* expected to write a Base64-encoded key of any length, and we
|
||||
* apply HKDF-SHA-256 to derive a 128-bit key-generation key
|
||||
* (only the first half of the result is used). We then encrypt
|
||||
* blocks in AES-128-GCM-SIV with a universally fixed nonce (12
|
||||
* repeated NUL characters).
|
||||
*
|
||||
* This construct has a weakness due to the nonce being fixed at
|
||||
* all times: when the same data block is encrypted twice, the
|
||||
* resulting ciphertext will be exactly the same. We have to live
|
||||
* with this weakness because ciphertext must be deterministic,
|
||||
* as otherwise our engines like ReplicatedMergeTree cannot
|
||||
* deduplicate data blocks.
|
||||
*/
|
||||
class CompressionCodecEncrypted : public ICompressionCodec
|
||||
{
|
||||
public:
|
||||
/** If a master key is available, the server is supposed to
|
||||
* invoke this static method at the startup. The codec will
|
||||
* refuse to compress or decompress any data until that. The
|
||||
* key can be an arbitrary octet string, but it is
|
||||
* recommended that the key is at least 16 octets long.
|
||||
*
|
||||
* Note that the master key is currently not guarded by a
|
||||
* mutex. This method should be invoked no more than once.
|
||||
*/
|
||||
static void setMasterKey(const std::string_view & master_key);
|
||||
|
||||
CompressionCodecEncrypted(const std::string_view & cipher);
|
||||
|
||||
uint8_t getMethodByte() const override;
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
bool isCompression() const override
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
bool isGenericCompression() const override
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
bool isPostProcessing() const override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
protected:
|
||||
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
|
||||
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||
|
||||
private:
|
||||
static std::string lastErrorString();
|
||||
static std::string deriveKey(const std::string_view & master_key);
|
||||
static void encrypt(const std::string_view & plaintext, char * ciphertext_and_tag);
|
||||
static void decrypt(const std::string_view & ciphertext_and_tag, char * plaintext);
|
||||
|
||||
/** A private class that holds keys derived from the master
|
||||
* key.
|
||||
*/
|
||||
struct KeyHolder : private boost::noncopyable
|
||||
{
|
||||
KeyHolder(const std::string_view & master_key);
|
||||
~KeyHolder();
|
||||
|
||||
std::string keygen_key;
|
||||
EVP_AEAD_CTX ctx;
|
||||
};
|
||||
|
||||
static const KeyHolder & getKeys();
|
||||
|
||||
static inline std::optional<KeyHolder> keys;
|
||||
};
|
||||
}
|
||||
|
||||
#endif /* USE_SSL && USE_INTERNAL_SSL_LIBRARY */
|
||||
@ -79,6 +79,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
bool is_compression = false;
|
||||
bool has_none = false;
|
||||
std::optional<size_t> generic_compression_codec_pos;
|
||||
std::set<size_t> post_processing_codecs;
|
||||
|
||||
bool can_substitute_codec_arguments = true;
|
||||
for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i)
|
||||
@ -156,6 +157,9 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
|
||||
if (!generic_compression_codec_pos && result_codec->isGenericCompression())
|
||||
generic_compression_codec_pos = i;
|
||||
|
||||
if (result_codec->isPostProcessing())
|
||||
post_processing_codecs.insert(i);
|
||||
}
|
||||
|
||||
String codec_description = queryToString(codecs_descriptions);
|
||||
@ -170,7 +174,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
|
||||
/// Allow to explicitly specify single NONE codec if user don't want any compression.
|
||||
/// But applying other transformations solely without compression (e.g. Delta) does not make sense.
|
||||
if (!is_compression && !has_none)
|
||||
/// It's okay to apply post-processing codecs solely without anything else.
|
||||
if (!is_compression && !has_none && post_processing_codecs.size() != codecs_descriptions->children.size())
|
||||
throw Exception(
|
||||
"Compression codec " + codec_description
|
||||
+ " does not compress anything."
|
||||
@ -180,9 +185,19 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
|
||||
ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
/// It does not make sense to apply any non-post-processing codecs
|
||||
/// after post-processing one.
|
||||
if (!post_processing_codecs.empty() &&
|
||||
*post_processing_codecs.begin() != codecs_descriptions->children.size() - post_processing_codecs.size())
|
||||
throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
|
||||
" because it does not make sense to apply any non-post-processing codecs after"
|
||||
" post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs'"
|
||||
" to skip this check).", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
/// It does not make sense to apply any transformations after generic compression algorithm
|
||||
/// So, generic compression can be only one and only at the end.
|
||||
if (generic_compression_codec_pos && *generic_compression_codec_pos != codecs_descriptions->children.size() - 1)
|
||||
if (generic_compression_codec_pos &&
|
||||
*generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - post_processing_codecs.size())
|
||||
throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
|
||||
" because it does not make sense to apply any transformations after generic compression algorithm."
|
||||
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
|
||||
@ -337,6 +352,7 @@ void registerCodecDelta(CompressionCodecFactory & factory);
|
||||
void registerCodecT64(CompressionCodecFactory & factory);
|
||||
void registerCodecDoubleDelta(CompressionCodecFactory & factory);
|
||||
void registerCodecGorilla(CompressionCodecFactory & factory);
|
||||
void registerCodecEncrypted(CompressionCodecFactory & factory);
|
||||
void registerCodecMultiple(CompressionCodecFactory & factory);
|
||||
|
||||
CompressionCodecFactory::CompressionCodecFactory()
|
||||
@ -349,6 +365,7 @@ CompressionCodecFactory::CompressionCodecFactory()
|
||||
registerCodecT64(*this);
|
||||
registerCodecDoubleDelta(*this);
|
||||
registerCodecGorilla(*this);
|
||||
registerCodecEncrypted(*this);
|
||||
registerCodecMultiple(*this);
|
||||
|
||||
default_codec = get("LZ4", {});
|
||||
|
||||
@ -43,6 +43,7 @@ enum class CompressionMethodByte : uint8_t
|
||||
T64 = 0x93,
|
||||
DoubleDelta = 0x94,
|
||||
Gorilla = 0x95,
|
||||
Encrypted = 0x96,
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -73,6 +73,9 @@ public:
|
||||
/// Is it a generic compression algorithm like lz4, zstd. Usually it does not make sense to apply generic compression more than single time.
|
||||
virtual bool isGenericCompression() const = 0;
|
||||
|
||||
/// If it is a post-processing codec such as encryption. Usually it does not make sense to apply non-post-processing codecs after this.
|
||||
virtual bool isPostProcessing() const { return false; }
|
||||
|
||||
/// It is a codec available only for evaluation purposes and not meant to be used in production.
|
||||
/// It will not be allowed to use unless the user will turn off the safety switch.
|
||||
virtual bool isExperimental() const { return false; }
|
||||
|
||||
@ -24,6 +24,7 @@ SRCS(
|
||||
CompressedWriteBuffer.cpp
|
||||
CompressionCodecDelta.cpp
|
||||
CompressionCodecDoubleDelta.cpp
|
||||
CompressionCodecEncrypted.cpp
|
||||
CompressionCodecGorilla.cpp
|
||||
CompressionCodecLZ4.cpp
|
||||
CompressionCodecMultiple.cpp
|
||||
|
||||
@ -481,6 +481,7 @@ class IColumn;
|
||||
M(Bool, query_plan_enable_optimizations, true, "Apply optimizations to query plan", 0) \
|
||||
M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \
|
||||
M(Bool, query_plan_filter_push_down, true, "Allow to push down filter by predicate query plan step", 0) \
|
||||
M(UInt64, regexp_max_matches_per_row, 1000, "Max matches of any single regexp per row, used to safeguard 'extractAllGroupsHorizontal' against consuming too much memory with greedy RE.", 0) \
|
||||
\
|
||||
M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \
|
||||
M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \
|
||||
|
||||
@ -102,7 +102,7 @@ RemoteQueryExecutor::RemoteQueryExecutor(
|
||||
if (main_table)
|
||||
table_to_check = std::make_shared<QualifiedTableName>(main_table.getQualifiedName());
|
||||
|
||||
return std::make_shared<HedgedConnections>(pool, current_settings, timeouts, throttler, pool_mode, table_to_check);
|
||||
return std::make_shared<HedgedConnections>(pool, context, timeouts, throttler, pool_mode, table_to_check);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
15
src/DataStreams/formatBlock.cpp
Normal file
15
src/DataStreams/formatBlock.cpp
Normal file
@ -0,0 +1,15 @@
|
||||
#include <Core/Block.h>
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <DataStreams/formatBlock.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
void formatBlock(BlockOutputStreamPtr & out, const Block & block)
|
||||
{
|
||||
out->writePrefix();
|
||||
out->write(block);
|
||||
out->writeSuffix();
|
||||
out->flush();
|
||||
}
|
||||
|
||||
}
|
||||
9
src/DataStreams/formatBlock.h
Normal file
9
src/DataStreams/formatBlock.h
Normal file
@ -0,0 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataStreams/IBlockStream_fwd.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
void formatBlock(BlockOutputStreamPtr & out, const Block & block);
|
||||
|
||||
}
|
||||
@ -33,6 +33,8 @@ public:
|
||||
const char * getFamilyName() const override { return "AggregateFunction"; }
|
||||
TypeIndex getTypeId() const override { return TypeIndex::AggregateFunction; }
|
||||
|
||||
Array getParameters() const { return parameters; }
|
||||
|
||||
bool canBeInsideNullable() const override { return false; }
|
||||
|
||||
DataTypePtr getReturnType() const { return function->getReturnType(); }
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#include "DictionarySourceHelpers.h"
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Core/ColumnWithTypeAndName.h>
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <DataStreams/IBlockStream_fwd.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include "DictionaryStructure.h"
|
||||
@ -18,14 +18,6 @@ namespace ErrorCodes
|
||||
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
void formatBlock(BlockOutputStreamPtr & out, const Block & block)
|
||||
{
|
||||
out->writePrefix();
|
||||
out->write(block);
|
||||
out->writeSuffix();
|
||||
out->flush();
|
||||
}
|
||||
|
||||
/// For simple key
|
||||
|
||||
Block blockForIds(
|
||||
|
||||
@ -13,15 +13,8 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class IBlockOutputStream;
|
||||
using BlockOutputStreamPtr = std::shared_ptr<IBlockOutputStream>;
|
||||
|
||||
struct DictionaryStructure;
|
||||
|
||||
/// Write keys to block output stream.
|
||||
|
||||
void formatBlock(BlockOutputStreamPtr & out, const Block & block);
|
||||
|
||||
/// For simple key
|
||||
|
||||
Block blockForIds(
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <common/scope_guard.h>
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <DataStreams/OwningBlockInputStream.h>
|
||||
#include <DataStreams/formatBlock.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <functional>
|
||||
#include <common/scope_guard.h>
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <DataStreams/formatBlock.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include "HTTPDictionarySource.h"
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <DataStreams/OwningBlockInputStream.h>
|
||||
#include <DataStreams/formatBlock.h>
|
||||
#include <IO/ConnectionTimeouts.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <IO/ReadWriteBufferFromHTTP.h>
|
||||
|
||||
@ -7,6 +7,8 @@
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/Regexps.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Core/Settings.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
@ -47,11 +49,17 @@ enum class ExtractAllGroupsResultKind
|
||||
template <typename Impl>
|
||||
class FunctionExtractAllGroups : public IFunction
|
||||
{
|
||||
ContextPtr context;
|
||||
|
||||
public:
|
||||
static constexpr auto Kind = Impl::Kind;
|
||||
static constexpr auto name = Impl::Name;
|
||||
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionExtractAllGroups>(); }
|
||||
FunctionExtractAllGroups(ContextPtr context_)
|
||||
: context(context_)
|
||||
{}
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionExtractAllGroups>(context); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
@ -147,6 +155,9 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Additional limit to fail fast on supposedly incorrect usage.
|
||||
const auto max_matches_per_row = context->getSettingsRef().regexp_max_matches_per_row;
|
||||
|
||||
PODArray<StringPiece, 0> all_matches;
|
||||
/// Number of times RE matched on each row of haystack column.
|
||||
PODArray<size_t, 0> number_of_matches_per_row;
|
||||
@ -172,16 +183,13 @@ public:
|
||||
for (size_t group = 1; group <= groups_count; ++group)
|
||||
all_matches.push_back(matched_groups[group]);
|
||||
|
||||
/// Additional limit to fail fast on supposedly incorrect usage, arbitrary value.
|
||||
static constexpr size_t MAX_MATCHES_PER_ROW = 1000;
|
||||
if (matches_per_row > MAX_MATCHES_PER_ROW)
|
||||
++matches_per_row;
|
||||
if (matches_per_row > max_matches_per_row)
|
||||
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
|
||||
"Too many matches per row (> {}) in the result of function {}",
|
||||
MAX_MATCHES_PER_ROW, getName());
|
||||
max_matches_per_row, getName());
|
||||
|
||||
pos = matched_groups[0].data() + std::max<size_t>(1, matched_groups[0].size());
|
||||
|
||||
++matches_per_row;
|
||||
}
|
||||
|
||||
number_of_matches_per_row.push_back(matches_per_row);
|
||||
|
||||
@ -803,6 +803,9 @@ void Context::setUser(const Credentials & credentials, const Poco::Net::SocketAd
|
||||
auto user = access->getUser();
|
||||
current_roles = std::make_shared<std::vector<UUID>>(user->granted_roles.findGranted(user->default_roles));
|
||||
|
||||
if (!user->default_database.empty())
|
||||
setCurrentDatabase(user->default_database);
|
||||
|
||||
auto default_profile_info = access->getDefaultProfileInfo();
|
||||
settings_constraints_and_current_profiles = default_profile_info->getConstraintsAndProfileIDs();
|
||||
applySettingsChanges(default_profile_info->settings);
|
||||
@ -1752,6 +1755,24 @@ zkutil::ZooKeeperPtr Context::getZooKeeper() const
|
||||
return shared->zookeeper;
|
||||
}
|
||||
|
||||
void Context::setSystemZooKeeperLogAfterInitializationIfNeeded()
|
||||
{
|
||||
/// It can be nearly impossible to understand in which order global objects are initialized on server startup.
|
||||
/// If getZooKeeper() is called before initializeSystemLogs(), then zkutil::ZooKeeper gets nullptr
|
||||
/// instead of pointer to system table and it logs nothing.
|
||||
/// This method explicitly sets correct pointer to system log after its initialization.
|
||||
/// TODO get rid of this if possible
|
||||
|
||||
std::lock_guard lock(shared->zookeeper_mutex);
|
||||
if (!shared->system_logs || !shared->system_logs->zookeeper_log)
|
||||
return;
|
||||
|
||||
if (shared->zookeeper)
|
||||
shared->zookeeper->setZooKeeperLog(shared->system_logs->zookeeper_log);
|
||||
|
||||
for (auto & zk : shared->auxiliary_zookeepers)
|
||||
zk.second->setZooKeeperLog(shared->system_logs->zookeeper_log);
|
||||
}
|
||||
|
||||
void Context::initializeKeeperStorageDispatcher() const
|
||||
{
|
||||
|
||||
@ -649,6 +649,8 @@ public:
|
||||
// Reload Zookeeper
|
||||
void reloadZooKeeperIfChanged(const ConfigurationPtr & config) const;
|
||||
|
||||
void setSystemZooKeeperLogAfterInitializationIfNeeded();
|
||||
|
||||
/// Create a cache of uncompressed blocks of specified size. This can be done only once.
|
||||
void setUncompressedCache(size_t max_size_in_bytes);
|
||||
std::shared_ptr<UncompressedCache> getUncompressedCache() const;
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <Parsers/ASTCreateUserQuery.h>
|
||||
#include <Parsers/ASTUserNameWithHost.h>
|
||||
#include <Parsers/ASTRolesOrUsersSet.h>
|
||||
#include <Parsers/ASTDatabaseOrNone.h>
|
||||
#include <Access/AccessControlManager.h>
|
||||
#include <Access/User.h>
|
||||
#include <Access/ContextAccess.h>
|
||||
@ -59,6 +60,9 @@ namespace
|
||||
else if (query.default_roles)
|
||||
set_default_roles(*query.default_roles);
|
||||
|
||||
if (query.default_database)
|
||||
user.default_database = query.default_database->database_name;
|
||||
|
||||
if (override_settings)
|
||||
user.settings = *override_settings;
|
||||
else if (query.settings)
|
||||
|
||||
@ -82,6 +82,13 @@ namespace
|
||||
query->grantees->use_keyword_any = true;
|
||||
}
|
||||
|
||||
if (!user.default_database.empty())
|
||||
{
|
||||
auto ast = std::make_shared<ASTDatabaseOrNone>();
|
||||
ast->database_name = user.default_database;
|
||||
query->default_database = ast;
|
||||
}
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
|
||||
@ -210,6 +210,12 @@ namespace
|
||||
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " GRANTEES " << (settings.hilite ? IAST::hilite_none : "");
|
||||
grantees.format(settings);
|
||||
}
|
||||
|
||||
void formatDefaultDatabase(const ASTDatabaseOrNone & default_database, const IAST::FormatSettings & settings)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " DEFAULT DATABASE " << (settings.hilite ? IAST::hilite_none : "");
|
||||
default_database.format(settings);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -262,6 +268,9 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState &
|
||||
if (remove_hosts)
|
||||
formatHosts("DROP", *remove_hosts, format);
|
||||
|
||||
if (default_database)
|
||||
formatDefaultDatabase(*default_database, format);
|
||||
|
||||
if (default_roles)
|
||||
formatDefaultRoles(*default_roles, format);
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Parsers/ASTQueryWithOnCluster.h>
|
||||
#include <Parsers/ASTDatabaseOrNone.h>
|
||||
#include <Access/Authentication.h>
|
||||
#include <Access/AllowedClientHosts.h>
|
||||
|
||||
@ -10,12 +11,14 @@ namespace DB
|
||||
{
|
||||
class ASTUserNamesWithHost;
|
||||
class ASTRolesOrUsersSet;
|
||||
class ASTDatabaseOrNone;
|
||||
class ASTSettingsProfileElements;
|
||||
|
||||
/** CREATE USER [IF NOT EXISTS | OR REPLACE] name
|
||||
* [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}]
|
||||
* [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
* [DEFAULT ROLE role [,...]]
|
||||
* [DEFAULT DATABASE database | NONE]
|
||||
* [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
* [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
|
||||
*
|
||||
@ -24,6 +27,7 @@ class ASTSettingsProfileElements;
|
||||
* [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}]
|
||||
* [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
* [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||
* [DEFAULT DATABASE database | NONE]
|
||||
* [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
* [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
|
||||
*/
|
||||
@ -51,6 +55,8 @@ public:
|
||||
std::shared_ptr<ASTSettingsProfileElements> settings;
|
||||
std::shared_ptr<ASTRolesOrUsersSet> grantees;
|
||||
|
||||
std::shared_ptr<ASTDatabaseOrNone> default_database;
|
||||
|
||||
String getID(char) const override;
|
||||
ASTPtr clone() const override;
|
||||
void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override;
|
||||
|
||||
16
src/Parsers/ASTDatabaseOrNone.cpp
Normal file
16
src/Parsers/ASTDatabaseOrNone.cpp
Normal file
@ -0,0 +1,16 @@
|
||||
#include <Parsers/ASTDatabaseOrNone.h>
|
||||
#include <IO/Operators.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
void ASTDatabaseOrNone::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const
|
||||
{
|
||||
if (none)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : "");
|
||||
return;
|
||||
}
|
||||
settings.ostr << database_name;
|
||||
}
|
||||
|
||||
}
|
||||
21
src/Parsers/ASTDatabaseOrNone.h
Normal file
21
src/Parsers/ASTDatabaseOrNone.h
Normal file
@ -0,0 +1,21 @@
|
||||
#pragma once
|
||||
|
||||
#include <Parsers/IAST.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ASTDatabaseOrNone : public IAST
|
||||
{
|
||||
public:
|
||||
bool none = false;
|
||||
String database_name;
|
||||
|
||||
bool isNone() const { return none; }
|
||||
String getID(char) const override { return "DatabaseOrNone"; }
|
||||
ASTPtr clone() const override { return std::make_shared<ASTDatabaseOrNone>(*this); }
|
||||
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <Parsers/ParserRolesOrUsersSet.h>
|
||||
#include <Parsers/ASTSettingsProfileElement.h>
|
||||
#include <Parsers/ParserSettingsProfileElement.h>
|
||||
#include <Parsers/ParserDatabaseOrNone.h>
|
||||
#include <common/range.h>
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
#include <boost/range/algorithm_ext/push_back.hpp>
|
||||
@ -300,6 +301,23 @@ namespace
|
||||
return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected);
|
||||
});
|
||||
}
|
||||
|
||||
bool parseDefaultDatabase(IParserBase::Pos & pos, Expected & expected, std::shared_ptr<ASTDatabaseOrNone> & default_database)
|
||||
{
|
||||
return IParserBase::wrapParseImpl(pos, [&]
|
||||
{
|
||||
if (!ParserKeyword{"DEFAULT DATABASE"}.ignore(pos, expected))
|
||||
return false;
|
||||
|
||||
ASTPtr ast;
|
||||
ParserDatabaseOrNone database_p;
|
||||
if (!database_p.parse(pos, ast, expected))
|
||||
return false;
|
||||
|
||||
default_database = typeid_cast<std::shared_ptr<ASTDatabaseOrNone>>(ast);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -349,6 +367,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
|
||||
std::shared_ptr<ASTRolesOrUsersSet> default_roles;
|
||||
std::shared_ptr<ASTSettingsProfileElements> settings;
|
||||
std::shared_ptr<ASTRolesOrUsersSet> grantees;
|
||||
std::shared_ptr<ASTDatabaseOrNone> default_database;
|
||||
String cluster;
|
||||
|
||||
while (true)
|
||||
@ -390,6 +409,9 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
|
||||
if (!grantees && parseGrantees(pos, expected, attach_mode, grantees))
|
||||
continue;
|
||||
|
||||
if (!default_database && parseDefaultDatabase(pos, expected, default_database))
|
||||
continue;
|
||||
|
||||
if (alter)
|
||||
{
|
||||
if (new_name.empty() && (names->size() == 1) && parseRenameTo(pos, expected, new_name))
|
||||
@ -445,6 +467,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
|
||||
query->default_roles = std::move(default_roles);
|
||||
query->settings = std::move(settings);
|
||||
query->grantees = std::move(grantees);
|
||||
query->default_database = std::move(default_database);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
29
src/Parsers/ParserDatabaseOrNone.cpp
Normal file
29
src/Parsers/ParserDatabaseOrNone.cpp
Normal file
@ -0,0 +1,29 @@
|
||||
#include <Parsers/ParserDatabaseOrNone.h>
|
||||
#include <Parsers/parseIdentifierOrStringLiteral.h>
|
||||
#include <Parsers/ASTDatabaseOrNone.h>
|
||||
#include <Parsers/CommonParsers.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
bool ParserDatabaseOrNone::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
{
|
||||
auto result = std::make_shared<ASTDatabaseOrNone>();
|
||||
node = result;
|
||||
|
||||
if (ParserKeyword{"NONE"}.ignore(pos, expected))
|
||||
{
|
||||
result->none = true;
|
||||
return true;
|
||||
}
|
||||
String database_name;
|
||||
if (parseIdentifierOrStringLiteral(pos, expected, database_name))
|
||||
{
|
||||
result->database_name = database_name;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
17
src/Parsers/ParserDatabaseOrNone.h
Normal file
17
src/Parsers/ParserDatabaseOrNone.h
Normal file
@ -0,0 +1,17 @@
|
||||
#pragma once
|
||||
#include <Parsers/IParserBase.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ParserDatabaseOrNone : public IParserBase
|
||||
{
|
||||
protected:
|
||||
const char * getName() const override { return "DatabaseOrNone"; }
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -358,8 +358,8 @@ void WindowTransform::advancePartitionEnd()
|
||||
assert(end.block == partition_end.block + 1);
|
||||
|
||||
// Try to advance the partition end pointer.
|
||||
const size_t n = partition_by_indices.size();
|
||||
if (n == 0)
|
||||
const size_t partition_by_columns = partition_by_indices.size();
|
||||
if (partition_by_columns == 0)
|
||||
{
|
||||
// No PARTITION BY. All input is one partition, which will end when the
|
||||
// input ends.
|
||||
@ -370,27 +370,44 @@ void WindowTransform::advancePartitionEnd()
|
||||
// Check for partition end.
|
||||
// The partition ends when the PARTITION BY columns change. We need
|
||||
// some reference columns for comparison. We might have already
|
||||
// dropped the blocks where the partition starts, but any row in the
|
||||
// partition will do. We use the current_row for this. It might be the same
|
||||
// as the partition_end if we're at the first row of the first partition, so
|
||||
// we will compare it to itself, but it still works correctly.
|
||||
// dropped the blocks where the partition starts, but any other row in the
|
||||
// partition will do. We can't use frame_start or frame_end or current_row (the next row
|
||||
// for which we are calculating the window functions), because they all might be
|
||||
// past the end of the partition. prev_frame_start is suitable, because it
|
||||
// is a pointer to the first row of the previous frame that must have been
|
||||
// valid, or to the first row of the partition, and we make sure not to drop
|
||||
// its block.
|
||||
assert(partition_start <= prev_frame_start);
|
||||
// The frame start should be inside the prospective partition, except the
|
||||
// case when it still has no rows.
|
||||
assert(prev_frame_start < partition_end || partition_start == partition_end);
|
||||
assert(first_block_number <= prev_frame_start.block);
|
||||
const auto block_rows = blockRowsNumber(partition_end);
|
||||
for (; partition_end.row < block_rows; ++partition_end.row)
|
||||
{
|
||||
// fmt::print(stderr, "compare reference '{}' to compared '{}'\n",
|
||||
// prev_frame_start, partition_end);
|
||||
|
||||
size_t i = 0;
|
||||
for (; i < n; i++)
|
||||
for (; i < partition_by_columns; i++)
|
||||
{
|
||||
const auto * ref = inputAt(current_row)[partition_by_indices[i]].get();
|
||||
const auto * c = inputAt(partition_end)[partition_by_indices[i]].get();
|
||||
if (c->compareAt(partition_end.row,
|
||||
current_row.row, *ref,
|
||||
const auto * reference_column
|
||||
= inputAt(prev_frame_start)[partition_by_indices[i]].get();
|
||||
const auto * compared_column
|
||||
= inputAt(partition_end)[partition_by_indices[i]].get();
|
||||
|
||||
// fmt::print(stderr, "reference '{}', compared '{}'\n",
|
||||
// (*reference_column)[prev_frame_start.row],
|
||||
// (*compared_column)[partition_end.row]);
|
||||
if (compared_column->compareAt(partition_end.row,
|
||||
prev_frame_start.row, *reference_column,
|
||||
1 /* nan_direction_hint */) != 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (i < n)
|
||||
if (i < partition_by_columns)
|
||||
{
|
||||
partition_ended = true;
|
||||
return;
|
||||
@ -976,6 +993,9 @@ void WindowTransform::writeOutCurrentRow()
|
||||
a->insertResultInto(buf, *result_column, arena.get());
|
||||
}
|
||||
}
|
||||
|
||||
// fmt::print(stderr, "wrote out aggregation state for current row '{}'\n",
|
||||
// current_row);
|
||||
}
|
||||
|
||||
static void assertSameColumns(const Columns & left_all,
|
||||
@ -1190,7 +1210,7 @@ void WindowTransform::appendChunk(Chunk & chunk)
|
||||
peer_group_number = 1;
|
||||
|
||||
// fmt::print(stderr, "reinitialize agg data at start of {}\n",
|
||||
// new_partition_start);
|
||||
// partition_start);
|
||||
// Reinitialize the aggregate function states because the new partition
|
||||
// has started.
|
||||
for (auto & ws : workspaces)
|
||||
@ -1367,13 +1387,16 @@ void WindowTransform::work()
|
||||
}
|
||||
|
||||
// We don't really have to keep the entire partition, and it can be big, so
|
||||
// we want to drop the starting blocks to save memory.
|
||||
// We can drop the old blocks if we already returned them as output, and the
|
||||
// frame and the current row are already past them. Note that the frame
|
||||
// start can be further than current row for some frame specs (e.g. EXCLUDE
|
||||
// CURRENT ROW), so we have to check both.
|
||||
// we want to drop the starting blocks to save memory. We can drop the old
|
||||
// blocks if we already returned them as output, and the frame and the
|
||||
// current row are already past them. We also need to keep the previous
|
||||
// frame start because we use it as the partition etalon. It is always less
|
||||
// than the current frame start, so we don't have to check the latter. Note
|
||||
// that the frame start can be further than current row for some frame specs
|
||||
// (e.g. EXCLUDE CURRENT ROW), so we have to check both.
|
||||
assert(prev_frame_start <= frame_start);
|
||||
const auto first_used_block = std::min(next_output_block_number,
|
||||
std::min(frame_start.block, current_row.block));
|
||||
std::min(prev_frame_start.block, current_row.block));
|
||||
|
||||
if (first_block_number < first_used_block)
|
||||
{
|
||||
@ -1386,6 +1409,7 @@ void WindowTransform::work()
|
||||
|
||||
assert(next_output_block_number >= first_block_number);
|
||||
assert(frame_start.block >= first_block_number);
|
||||
assert(prev_frame_start.block >= first_block_number);
|
||||
assert(current_row.block >= first_block_number);
|
||||
assert(peer_group_start.block >= first_block_number);
|
||||
}
|
||||
|
||||
@ -2395,7 +2395,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c
|
||||
/// It's a DROP PART and it's already executed by fetching some covering part
|
||||
bool is_drop_part = !drop_range.isFakeDropRangePart() && drop_range.min_block;
|
||||
|
||||
if (is_drop_part && (part->info.min_block != drop_range.min_block || part->info.max_block != drop_range.max_block))
|
||||
if (is_drop_part && (part->info.min_block != drop_range.min_block || part->info.max_block != drop_range.max_block || part->info.getDataVersion() != drop_range.getDataVersion()))
|
||||
{
|
||||
/// Why we check only min and max blocks here without checking merge
|
||||
/// level? It's a tricky situation which can happen on a stale
|
||||
@ -2412,9 +2412,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c
|
||||
/// So here we just check that all_1_3_1 covers blocks from drop
|
||||
/// all_2_2_2.
|
||||
///
|
||||
/// NOTE: this helps only to avoid logical error during drop part.
|
||||
/// We still get intersecting "parts" in queue.
|
||||
bool is_covered_by_min_max_block = part->info.min_block <= drop_range.min_block && part->info.max_block >= drop_range.max_block;
|
||||
bool is_covered_by_min_max_block = part->info.min_block <= drop_range.min_block && part->info.max_block >= drop_range.max_block && part->info.getDataVersion() >= drop_range.getDataVersion();
|
||||
if (is_covered_by_min_max_block)
|
||||
{
|
||||
LOG_INFO(log, "Skipping drop range for part {} because covering part {} already exists", drop_range.getPartName(), part->name);
|
||||
@ -3214,7 +3212,11 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr loc
|
||||
const auto & partition_ast = ast->as<ASTPartition &>();
|
||||
|
||||
if (!partition_ast.value)
|
||||
{
|
||||
if (!MergeTreePartInfo::validatePartitionID(partition_ast.id, format_version))
|
||||
throw Exception("Invalid partition format: " + partition_ast.id, ErrorCodes::INVALID_PARTITION_VALUE);
|
||||
return partition_ast.id;
|
||||
}
|
||||
|
||||
if (format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
|
||||
{
|
||||
|
||||
@ -21,6 +21,40 @@ MergeTreePartInfo MergeTreePartInfo::fromPartName(const String & part_name, Merg
|
||||
}
|
||||
|
||||
|
||||
bool MergeTreePartInfo::validatePartitionID(const String & partition_id, MergeTreeDataFormatVersion format_version)
|
||||
{
|
||||
if (partition_id.empty())
|
||||
return false;
|
||||
|
||||
ReadBufferFromString in(partition_id);
|
||||
|
||||
if (format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
|
||||
{
|
||||
UInt32 min_yyyymmdd = 0;
|
||||
UInt32 max_yyyymmdd = 0;
|
||||
if (!tryReadIntText(min_yyyymmdd, in)
|
||||
|| !checkChar('_', in)
|
||||
|| !tryReadIntText(max_yyyymmdd, in)
|
||||
|| !checkChar('_', in))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
while (!in.eof())
|
||||
{
|
||||
char c;
|
||||
readChar(c, in);
|
||||
|
||||
if (c == '_')
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return in.eof();
|
||||
}
|
||||
|
||||
bool MergeTreePartInfo::tryParsePartName(const String & part_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version)
|
||||
{
|
||||
ReadBufferFromString in(part_name);
|
||||
|
||||
@ -86,6 +86,9 @@ struct MergeTreePartInfo
|
||||
return static_cast<UInt64>(max_block - min_block + 1);
|
||||
}
|
||||
|
||||
/// Simple sanity check for partition ID. Checking that it's not too long or too short, doesn't contain a lot of '_'.
|
||||
static bool validatePartitionID(const String & partition_id, MergeTreeDataFormatVersion format_version);
|
||||
|
||||
static MergeTreePartInfo fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version); // -V1071
|
||||
|
||||
static bool tryParsePartName(const String & part_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version);
|
||||
|
||||
@ -281,6 +281,8 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval(
|
||||
current_parts.remove(*drop_range_part_name);
|
||||
|
||||
virtual_parts.remove(*drop_range_part_name);
|
||||
|
||||
removeCoveredPartsFromMutations(*drop_range_part_name, /*remove_part = */ true, /*remove_covered_parts = */ false);
|
||||
}
|
||||
|
||||
if (entry->type == LogEntry::DROP_RANGE)
|
||||
@ -303,6 +305,9 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval(
|
||||
|
||||
for (const String & virtual_part_name : entry->getVirtualPartNames(format_version))
|
||||
{
|
||||
/// This part will never appear, so remove it from virtual parts
|
||||
virtual_parts.remove(virtual_part_name);
|
||||
|
||||
/// Because execution of the entry is unsuccessful,
|
||||
/// `virtual_part_name` will never appear so we won't need to mutate
|
||||
/// it.
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <Interpreters/IdentifierSemantic.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/parseRemoteDescription.h>
|
||||
#include <Common/Macros.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <Core/Defines.h>
|
||||
#include <common/range.h>
|
||||
@ -156,10 +157,11 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr
|
||||
if (!cluster_name.empty())
|
||||
{
|
||||
/// Use an existing cluster from the main config
|
||||
String cluster_name_expanded = context->getMacros()->expand(cluster_name);
|
||||
if (name != "clusterAllReplicas")
|
||||
cluster = context->getCluster(cluster_name);
|
||||
cluster = context->getCluster(cluster_name_expanded);
|
||||
else
|
||||
cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettings());
|
||||
cluster = context->getCluster(cluster_name_expanded)->getClusterWithReplicasAsShards(context->getSettings());
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -175,7 +175,8 @@ def configure_testcase_args(args, case_file, suite_tmp_dir, stderr_file):
|
||||
database = 'test_{suffix}'.format(suffix=random_str())
|
||||
|
||||
with open(stderr_file, 'w') as stderr:
|
||||
clickhouse_proc_create = Popen(shlex.split(testcase_args.testcase_client), stdin=PIPE, stdout=PIPE, stderr=stderr, universal_newlines=True)
|
||||
client_cmd = testcase_args.testcase_client + " " + get_additional_client_options(args)
|
||||
clickhouse_proc_create = Popen(shlex.split(client_cmd), stdin=PIPE, stdout=PIPE, stderr=stderr, universal_newlines=True)
|
||||
try:
|
||||
clickhouse_proc_create.communicate(("CREATE DATABASE " + database + get_db_engine(testcase_args, database)), timeout=testcase_args.timeout)
|
||||
except TimeoutExpired:
|
||||
@ -937,7 +938,8 @@ def main(args):
|
||||
def create_common_database(args, db_name):
|
||||
create_database_retries = 0
|
||||
while create_database_retries < MAX_RETRIES:
|
||||
clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True)
|
||||
client_cmd = args.client + " " + get_additional_client_options(args)
|
||||
clickhouse_proc_create = Popen(shlex.split(client_cmd), stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True)
|
||||
(stdout, stderr) = clickhouse_proc_create.communicate(("CREATE DATABASE IF NOT EXISTS " + db_name + get_db_engine(args, db_name)))
|
||||
if not need_retry(stdout, stderr):
|
||||
break
|
||||
|
||||
@ -1,167 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -x
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)
|
||||
DATA_DIR=${DATA_DIR:=`mktemp -d /tmp/clickhouse.test..XXXXX`}
|
||||
DATA_DIR_PATTERN=${DATA_DIR_PATTERN:=/tmp/clickhouse} # path from config file, will be replaced to temporary
|
||||
LOG_DIR=${LOG_DIR:=$DATA_DIR/log}
|
||||
export CLICKHOUSE_BINARY_NAME=${CLICKHOUSE_BINARY_NAME:="clickhouse"}
|
||||
( [ -x "$ROOT_DIR/programs/${CLICKHOUSE_BINARY_NAME}-server" ] || [ -x "$ROOT_DIR/programs/${CLICKHOUSE_BINARY_NAME}" ] ) && BUILD_DIR=${BUILD_DIR:=$ROOT_DIR} # Build without separate build dir
|
||||
[ -d "$ROOT_DIR/build${BUILD_TYPE}" ] && BUILD_DIR=${BUILD_DIR:=$ROOT_DIR/build${BUILD_TYPE}}
|
||||
BUILD_DIR=${BUILD_DIR:=$ROOT_DIR}
|
||||
[ -x ${CLICKHOUSE_BINARY_NAME}-server" ] && [ -x ${CLICKHOUSE_BINARY_NAME}-client" ] && BIN_DIR= # Allow run in /usr/bin
|
||||
( [ -x "$BUILD_DIR/programs/${CLICKHOUSE_BINARY_NAME}" ] || [ -x "$BUILD_DIR/programs/${CLICKHOUSE_BINARY_NAME}-server" ] ) && BIN_DIR=${BIN_DIR:=$BUILD_DIR/programs/}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-server" ] && CLICKHOUSE_SERVER=${CLICKHOUSE_SERVER:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-server}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}" ] && CLICKHOUSE_SERVER=${CLICKHOUSE_SERVER:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME} server}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-client" ] && CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-client}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}" ] && CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME} client}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-extract-from-config" ] && CLICKHOUSE_EXTRACT=${CLICKHOUSE_EXTRACT:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-extract-from-config}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}" ] && CLICKHOUSE_EXTRACT=${CLICKHOUSE_EXTRACT:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME} extract-from-config}
|
||||
|
||||
[ -f "$CUR_DIR/server-test.xml" ] && CONFIG_DIR=${CONFIG_DIR=$CUR_DIR}/
|
||||
CONFIG_CLIENT_DIR=${CONFIG_CLIENT_DIR=$CONFIG_DIR}
|
||||
CONFIG_SERVER_DIR=${CONFIG_SERVER_DIR=$CONFIG_DIR}
|
||||
[ ! -f "${CONFIG_CLIENT_DIR}client-test.xml" ] && CONFIG_CLIENT_DIR=${CONFIG_CLIENT_DIR:=/etc/clickhouse-client/}
|
||||
[ ! -f "${CONFIG_SERVER_DIR}server-test.xml" ] && CONFIG_SERVER_DIR=${CONFIG_SERVER_DIR:=/etc/clickhouse-server/}
|
||||
export CLICKHOUSE_CONFIG_CLIENT=${CLICKHOUSE_CONFIG_CLIENT:=${CONFIG_CLIENT_DIR}client-test.xml}
|
||||
export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:=${CONFIG_SERVER_DIR}server-test.xml}
|
||||
CLICKHOUSE_CONFIG_USERS=${CONFIG_SERVER_DIR}users.xml
|
||||
[ ! -f "$CLICKHOUSE_CONFIG_USERS" ] && CLICKHOUSE_CONFIG_USERS=$CUR_DIR/../programs/server/users.xml
|
||||
CLICKHOUSE_CONFIG_USERS_D=${CONFIG_SERVER_DIR}users.d
|
||||
[ ! -d "$CLICKHOUSE_CONFIG_USERS_D" ] && CLICKHOUSE_CONFIG_USERS_D=$CUR_DIR/../programs/server/users.d
|
||||
[ -x "$CUR_DIR/clickhouse-test" ] && TEST_DIR=${TEST_DIR=$CUR_DIR/}
|
||||
[ -d "$CUR_DIR/queries" ] && QUERIES_DIR=${QUERIES_DIR=$CUR_DIR/queries}
|
||||
[ ! -d "$QUERIES_DIR" ] && [ -d "/usr/local/share/clickhouse-test/queries" ] && QUERIES_DIR=${QUERIES_DIR=/usr/local/share/clickhouse-test/queries}
|
||||
[ ! -d "$QUERIES_DIR" ] && [ -d "/usr/share/clickhouse-test/queries" ] && QUERIES_DIR=${QUERIES_DIR=/usr/share/clickhouse-test/queries}
|
||||
|
||||
TEST_PORT_RANDOM=${TEST_PORT_RANDOM=1}
|
||||
if [ "${TEST_PORT_RANDOM}" ]; then
|
||||
CLICKHOUSE_PORT_BASE=${CLICKHOUSE_PORT_BASE:=$(( ( RANDOM % 50000 ) + 10000 ))}
|
||||
CLICKHOUSE_PORT_TCP=${CLICKHOUSE_PORT_TCP:=$(($CLICKHOUSE_PORT_BASE + 1))}
|
||||
CLICKHOUSE_PORT_HTTP=${CLICKHOUSE_PORT_HTTP:=$(($CLICKHOUSE_PORT_BASE + 2))}
|
||||
CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:=$(($CLICKHOUSE_PORT_BASE + 3))}
|
||||
CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=$(($CLICKHOUSE_PORT_BASE + 4))}
|
||||
CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=$(($CLICKHOUSE_PORT_BASE + 5))}
|
||||
CLICKHOUSE_PORT_ODBC_BRIDGE=${CLICKHOUSE_ODBC_BRIDGE:=$(($CLICKHOUSE_PORT_BASE + 6))}
|
||||
fi
|
||||
|
||||
rm -rf $DATA_DIR || true
|
||||
mkdir -p $LOG_DIR $DATA_DIR/etc || true
|
||||
|
||||
if [ "$DATA_DIR_PATTERN" != "$DATA_DIR" ]; then
|
||||
cat $CLICKHOUSE_CONFIG | sed -e s!$DATA_DIR_PATTERN!$DATA_DIR! > $DATA_DIR/etc/server-config.xml
|
||||
export CLICKHOUSE_CONFIG=$DATA_DIR/etc/server-config.xml
|
||||
cp $CLICKHOUSE_CONFIG_USERS $DATA_DIR/etc
|
||||
cp -R -L $CLICKHOUSE_CONFIG_USERS_D $DATA_DIR/etc
|
||||
cat ${CONFIG_SERVER_DIR}/ints_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/ints_dictionary.xml
|
||||
cat ${CONFIG_SERVER_DIR}/strings_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/strings_dictionary.xml
|
||||
cat ${CONFIG_SERVER_DIR}/decimals_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/decimals_dictionary.xml
|
||||
cat ${CONFIG_SERVER_DIR}/executable_pool_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/executable_pool_dictionary.xml
|
||||
fi
|
||||
|
||||
CLICKHOUSE_EXTRACT_CONFIG=${CLICKHOUSE_EXTRACT_CONFIG:="${CLICKHOUSE_EXTRACT} --config=$CLICKHOUSE_CONFIG"}
|
||||
CLICKHOUSE_LOG=${CLICKHOUSE_LOG:=${LOG_DIR}clickhouse-server.log}
|
||||
export CLICKHOUSE_PORT_TCP=${CLICKHOUSE_PORT_TCP:=`$CLICKHOUSE_EXTRACT_CONFIG --key=tcp_port || echo 9000`}
|
||||
export CLICKHOUSE_PORT_HTTP=${CLICKHOUSE_PORT_HTTP:=`$CLICKHOUSE_EXTRACT_CONFIG --key=http_port || echo 8123`}
|
||||
export CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:=`$CLICKHOUSE_EXTRACT_CONFIG --key=interserver_http_port || echo 9009`}
|
||||
export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=`$CLICKHOUSE_EXTRACT_CONFIG --key=tcp_port_secure`}
|
||||
export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=`$CLICKHOUSE_EXTRACT_CONFIG --key=https_port`}
|
||||
export CLICKHOUSE_ODBC_BRIDGE=${CLICKHOUSE_ODBC_BRIDGE:=`$CLICKHOUSE_EXTRACT_CONFIG --key=odbc_bridge.port || echo 9018`}
|
||||
|
||||
DHPARAM=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.dhParamsFile`
|
||||
PRIVATEKEY=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.privateKeyFile`
|
||||
CERT=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.certificateFile`
|
||||
# Do not generate in case broken extract-config
|
||||
[ -n "$DHPARAM" ] && openssl dhparam -out $DHPARAM 256
|
||||
[ -n "$PRIVATEKEY" ] && [ -n "$CERT" ] && openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout $PRIVATEKEY -out $CERT
|
||||
|
||||
if [ "$TEST_GDB" ] || [ "$GDB" ]; then
|
||||
echo -e "run \nset pagination off \nset logging file $LOG_DIR/server.gdb.log \nset logging on \nbacktrace \nthread apply all backtrace \nbacktrace \ndetach \nquit " > $DATA_DIR/gdb.cmd
|
||||
GDB=${GDB:="gdb -x $DATA_DIR/gdb.cmd --args "}
|
||||
fi
|
||||
|
||||
# Start a local clickhouse server which will be used to run tests
|
||||
|
||||
# TODO: fix change shard ports:
|
||||
# --remote_servers.test_shard_localhost_secure.shard.replica.port=$CLICKHOUSE_PORT_TCP_SECURE \
|
||||
# --remote_servers.test_shard_localhost.shard.replica.port=$CLICKHOUSE_PORT_TCP \
|
||||
|
||||
VERSION=`$CLICKHOUSE_CLIENT --version-clean`
|
||||
# If run from compile dir - use in-place compile binary and headers
|
||||
[ -n "$BIN_DIR" ] && INTERNAL_COMPILER_PARAMS="--compiler_executable_root=${INTERNAL_COMPILER_BIN_ROOT:=$BUILD_DIR/programs/} --compiler_headers=$BUILD_DIR/programs/clang/headers/$VERSION/ --compiler_headers_root=$BUILD_DIR/programs/clang/headers/$VERSION/"
|
||||
|
||||
$GDB $CLICKHOUSE_SERVER --config-file=$CLICKHOUSE_CONFIG --log=$CLICKHOUSE_LOG $TEST_SERVER_PARAMS -- \
|
||||
--http_port=$CLICKHOUSE_PORT_HTTP \
|
||||
--tcp_port=$CLICKHOUSE_PORT_TCP \
|
||||
--https_port=$CLICKHOUSE_PORT_HTTPS \
|
||||
--tcp_port_secure=$CLICKHOUSE_PORT_TCP_SECURE \
|
||||
--interserver_http_port=$CLICKHOUSE_PORT_INTERSERVER \
|
||||
--odbc_bridge.port=$CLICKHOUSE_ODBC_BRIDGE \
|
||||
$INTERNAL_COMPILER_PARAMS \
|
||||
$TEST_SERVER_CONFIG_PARAMS \
|
||||
2>&1 > $LOG_DIR/server.stdout.log &
|
||||
CH_PID=$!
|
||||
sleep ${TEST_SERVER_STARTUP_WAIT:=5}
|
||||
|
||||
if [ "$GDB" ]; then
|
||||
# Long symbols read
|
||||
sleep ${TEST_GDB_SLEEP:=60}
|
||||
fi
|
||||
|
||||
tail -n50 $LOG_DIR/*.log || true
|
||||
|
||||
# Define needed stuff to kill test clickhouse server after tests completion
|
||||
function finish {
|
||||
kill $CH_PID || true
|
||||
wait
|
||||
tail -n 50 $LOG_DIR/*.log || true
|
||||
if [ "$GDB" ]; then
|
||||
cat $LOG_DIR/server.gdb.log || true
|
||||
fi
|
||||
rm -rf $DATA_DIR
|
||||
}
|
||||
trap finish EXIT SIGINT SIGQUIT SIGTERM
|
||||
|
||||
# Do tests
|
||||
if [ -n "$*" ]; then
|
||||
$*
|
||||
else
|
||||
TEST_RUN=${TEST_RUN=1}
|
||||
TEST_DICT=${TEST_DICT=1}
|
||||
CLICKHOUSE_CLIENT_QUERY="${CLICKHOUSE_CLIENT} --config ${CLICKHOUSE_CONFIG_CLIENT} --port $CLICKHOUSE_PORT_TCP -m -n -q"
|
||||
$CLICKHOUSE_CLIENT_QUERY 'SELECT * from system.build_options; SELECT * FROM system.clusters;'
|
||||
CLICKHOUSE_TEST="env ${TEST_DIR}clickhouse-test --force-color --binary ${BIN_DIR}${CLICKHOUSE_BINARY_NAME} --configclient $CLICKHOUSE_CONFIG_CLIENT --configserver $CLICKHOUSE_CONFIG --tmp $DATA_DIR/tmp --queries $QUERIES_DIR $TEST_OPT0 $TEST_OPT"
|
||||
if [ "${TEST_RUN_STRESS}" ]; then
|
||||
# Running test in parallel will fail some results (tests can create/fill/drop same tables)
|
||||
TEST_NPROC=${TEST_NPROC:=$(( `nproc || sysctl -n hw.ncpu || echo 2` * 2))}
|
||||
for i in `seq 1 ${TEST_NPROC}`; do
|
||||
$CLICKHOUSE_TEST --order=random --testname --tmp=$DATA_DIR/tmp/tmp${i} &
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "${TEST_RUN_PARALLEL}" ]; then
|
||||
# Running test in parallel will fail some results (tests can create/fill/drop same tables)
|
||||
TEST_NPROC=${TEST_NPROC:=$(( `nproc || sysctl -n hw.ncpu || echo 2` * 2))}
|
||||
for i in `seq 1 ${TEST_NPROC}`; do
|
||||
$CLICKHOUSE_TEST --testname --tmp=$DATA_DIR/tmp/tmp${i} --database=test${i} --parallel=${i}/${TEST_NPROC} &
|
||||
done
|
||||
for job in `jobs -p`; do
|
||||
#echo wait $job
|
||||
wait $job || let "FAIL+=1"
|
||||
done
|
||||
|
||||
#echo $FAIL
|
||||
if [ "$FAIL" != "0" ]; then
|
||||
return $FAIL
|
||||
fi
|
||||
else
|
||||
( [ "$TEST_RUN" ] && $CLICKHOUSE_TEST ) || ${TEST_TRUE:=false}
|
||||
fi
|
||||
|
||||
$CLICKHOUSE_CLIENT_QUERY "SELECT event, value FROM system.events; SELECT metric, value FROM system.metrics; SELECT metric, value FROM system.asynchronous_metrics;"
|
||||
$CLICKHOUSE_CLIENT_QUERY "SELECT 'Still alive'"
|
||||
fi
|
||||
6
tests/config/config.d/encryption.xml
Normal file
6
tests/config/config.d/encryption.xml
Normal file
@ -0,0 +1,6 @@
|
||||
<yandex>
|
||||
<encryption>
|
||||
<!-- "Some fixed key that is at least 16 bytes long" in Base64 -->
|
||||
<key_command>echo "U29tZSBmaXhlZCBrZXkgdGhhdCBpcyBhdCBsZWFzdCAxNiBieXRlcyBsb25n"</key_command>
|
||||
</encryption>
|
||||
</yandex>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user