Merge branch 'master' into in_memory_raft

This commit is contained in:
alesapin 2021-01-19 10:27:21 +03:00
commit 6883143994
67 changed files with 6912 additions and 5622 deletions

View File

@ -12,6 +12,9 @@ assignees: ''
**Describe the bug**
A clear and concise description of what works not as it is supposed to.
**Does it reproduce on recent release?**
[The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv)
**How to reproduce**
* Which ClickHouse server version to use
* Which interface to use, if matters

File diff suppressed because it is too large Load Diff

View File

@ -17,3 +17,4 @@ ClickHouse® is an open-source column-oriented database management system that a
## Upcoming Events
* [SF Bay Area ClickHouse Virtual Office Hours (online)](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/274273549/) on 20 January 2020.
* [Chinese ClickHouse Meetup (online)](http://hdxu.cn/8KxZE) on 6 February 2020.

View File

@ -4,6 +4,12 @@
#include "syscall.h"
#include "atomic.h"
#if defined(__has_feature)
#if __has_feature(memory_sanitizer)
#include <sanitizer/msan_interface.h>
#endif
#endif
#ifdef VDSO_GETCPU_SYM
static void *volatile vdso_func;
@ -37,6 +43,13 @@ int sched_getcpu(void)
#endif
r = __syscall(SYS_getcpu, &cpu, 0, 0);
if (!r) return cpu;
if (!r) {
#if defined(__has_feature)
#if __has_feature(memory_sanitizer)
__msan_unpoison(&cpu, sizeof(cpu));
#endif
#endif
return cpu;
}
return __syscall_ret(r);
}

View File

@ -119,12 +119,6 @@ if (USE_INTERNAL_LDAP_LIBRARY)
add_subdirectory (openldap-cmake)
endif ()
# Should go before:
# - mariadb-connector-c
# - aws-s3-cmake
# - sentry-native
add_subdirectory (curl-cmake)
function(mysql_support)
set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC)
set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC)
@ -142,6 +136,7 @@ function(mysql_support)
set(ZLIB_LIBRARY ${ZLIB_LIBRARIES})
set(WITH_EXTERNAL_ZLIB ON)
endif()
set(WITH_CURL OFF)
add_subdirectory (mariadb-connector-c)
endfunction()
if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
@ -288,6 +283,10 @@ if (USE_CASSANDRA)
add_subdirectory (cassandra)
endif()
# Should go before:
# - sentry-native
add_subdirectory (curl-cmake)
if (USE_SENTRY)
add_subdirectory (sentry-native)
endif()

2
contrib/dragonbox vendored

@ -1 +1 @@
Subproject commit b2751c65c0592c0239aec3becd53d0ea2fde9329
Subproject commit 923705af6fd953aa948fc175f6020b15f7359838

2
contrib/krb5 vendored

@ -1 +1 @@
Subproject commit 90ff6f4f8c695d6bf1aaba78a9b8942be92141c2
Subproject commit 5149dea4e2be0f67707383d2682b897c14631374

View File

@ -1,4 +1,6 @@
#!/bin/bash
# shellcheck disable=SC2086
set -eux
set -o pipefail
trap "exit" INT TERM
@ -20,6 +22,7 @@ function clone
git init
git remote add origin https://github.com/ClickHouse/ClickHouse
git fetch --depth=1 origin "$SHA_TO_TEST"
git fetch --depth=1 origin master # Used to obtain the list of modified or added tests
# If not master, try to fetch pull/.../{head,merge}
if [ "$PR_TO_TEST" != "0" ]
@ -33,9 +36,6 @@ function clone
function download
{
# wget -O- -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/performance/performance.tgz" \
# | tar --strip-components=1 -zxv
wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"
chmod +x clickhouse
ln -s ./clickhouse ./clickhouse-server
@ -73,6 +73,17 @@ function watchdog
function fuzz
{
# Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests.
cd ch
NEW_TESTS=$(git diff --name-only master | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R)
cd ..
if [[ -n "$NEW_TESTS" ]]
then
NEW_TESTS_OPT="--interleave-queries-file ${NEW_TESTS}"
else
NEW_TESTS_OPT=""
fi
./clickhouse-server --config-file db/config.xml -- --path db 2>&1 | tail -100000 > server.log &
server_pid=$!
kill -0 $server_pid
@ -85,7 +96,7 @@ function fuzz
# SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric.
# SC2046: Quote this to prevent word splitting. Actually I need word splitting.
# shellcheck disable=SC2012,SC2046
./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \
./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) $NEW_TESTS_OPT \
> >(tail -n 100000 > fuzzer.log) \
2>&1 \
|| fuzzer_exit_code=$?
@ -107,7 +118,7 @@ function fuzz
case "$stage" in
"")
;&
;& # Did you know? This is "fallthrough" in bash. https://stackoverflow.com/questions/12010686/case-statement-fallthrough
"clone")
time clone
if [ -v FUZZ_LOCAL_SCRIPT ]

View File

@ -25,6 +25,7 @@ Ways to configure settings, in order of priority:
- When starting the ClickHouse console client in non-interactive mode, set the startup parameter `--setting=value`.
- When using the HTTP API, pass CGI parameters (`URL?setting_1=value&setting_2=value...`).
- Make settings in the [SETTINGS](../../sql-reference/statements/select/index.md#settings-in-select) clause of the SELECT query. The setting value is applied only to that query and is reset to default or previous value after the query is executed.
Settings that can only be made in the server config file are not covered in this section.

View File

@ -0,0 +1,190 @@
# MergeTree tables settings {#merge-tree-settings}
The values of `merge_tree` settings (for all MergeTree tables) can be viewed in the table `system.merge_tree_settings`, they can be overridden in `config.xml` in the `merge_tree` section, or set in the `SETTINGS` section of each table.
Override example in `config.xml`:
``` text
<merge_tree>
<max_suspicious_broken_parts>5</max_suspicious_broken_parts>
</merge_tree>
```
An example to set in `SETTINGS` for a particular table:
``` sql
CREATE TABLE foo
(
`A` Int64
)
ENGINE = MergeTree
ORDER BY tuple()
SETTINGS max_suspicious_broken_parts = 500;
```
An example of changing the settings for a specific table with the `ALTER TABLE ... MODIFY SETTING` command:
``` sql
ALTER TABLE foo
MODIFY SETTING max_suspicious_broken_parts = 100;
```
## parts_to_throw_insert {#parts-to-throw-insert}
If the number of active parts in a single partition exceeds the `parts_to_throw_insert` value, `INSERT` is interrupted with the `Too many parts (N). Merges are processing significantly slower than inserts` exception.
Possible values:
- Any positive integer.
Default value: 300.
To achieve maximum performance of `SELECT` queries, it is necessary to minimize the number of parts processed, see [Merge Tree](../../development/architecture.md#merge-tree).
You can set a larger value to 600 (1200), this will reduce the probability of the `Too many parts` error, but at the same time `SELECT` performance might degrade. Also in case of a merge issue (for example, due to insufficient disk space) you will notice it later than it could be with the original 300.
## parts_to_delay_insert {#parts-to-delay-insert}
If the number of active parts in a single partition exceeds the `parts_to_delay_insert` value, an `INSERT` artificially slows down.
Possible values:
- Any positive integer.
Default value: 150.
ClickHouse artificially executes `INSERT` longer (adds sleep) so that the background merge process can merge parts faster than they are added.
## max_delay_to_insert {#max-delay-to-insert}
The value in seconds, which is used to calculate the `INSERT` delay, if the number of active parts in a single partition exceeds the [parts_to_delay_insert](#parts-to-delay-insert) value.
Possible values:
- Any positive integer.
Default value: 1.
The delay (in milliseconds) for `INSERT` is calculated by the formula:
```code
max_k = parts_to_throw_insert - parts_to_delay_insert
k = 1 + parts_count_in_partition - parts_to_delay_insert
delay_milliseconds = pow(max_delay_to_insert * 1000, k / max_k)
```
For example if a partition has 299 active parts and parts_to_throw_insert = 300, parts_to_delay_insert = 150, max_delay_to_insert = 1, `INSERT` is delayed for `pow( 1 * 1000, (1 + 299 - 150) / (300 - 150) ) = 1000` milliseconds.
## max_parts_in_total {#max-parts-in-total}
If the total number of active parts in all partitions of a table exceeds the `max_parts_in_total` value `INSERT` is interrupted with the `Too many parts (N)` exception.
Possible values:
- Any positive integer.
Default value: 100000.
A large number of parts in a table reduces performance of ClickHouse queries and increases ClickHouse boot time. Most often this is a consequence of an incorrect design (mistakes when choosing a partitioning strategy - too small partitions).
## replicated_deduplication_window {#replicated-deduplication-window}
The number of most recently inserted blocks for which Zookeeper stores hash sums to check for duplicates.
Possible values:
- Any positive integer.
- 0 (disable deduplication)
Default value: 100.
The `Insert` command creates one or more blocks (parts). When inserting into Replicated tables, ClickHouse for [insert deduplication](../../engines/table-engines/mergetree-family/replication/) writes the hash sums of the created parts into Zookeeper. Hash sums are stored only for the most recent `replicated_deduplication_window` blocks. The oldest hash sums are removed from Zookeeper.
A large number of `replicated_deduplication_window` slows down `Inserts` because it needs to compare more entries.
The hash sum is calculated from the composition of the field names and types and the data of the inserted part (stream of bytes).
## replicated_deduplication_window_seconds {#replicated-deduplication-window-seconds}
The number of seconds after which the hash sums of the inserted blocks are removed from Zookeeper.
Possible values:
- Any positive integer.
Default value: 604800 (1 week).
Similar to [replicated_deduplication_window](#replicated-deduplication-window), `replicated_deduplication_window_seconds` specifies how long to store hash sums of blocks for insert deduplication. Hash sums older than `replicated_deduplication_window_seconds` are removed from Zookeeper, even if they are less than ` replicated_deduplication_window`.
## old_parts_lifetime {#old-parts-lifetime}
The time (in seconds) of storing inactive parts to protect against data loss during spontaneous server reboots.
Possible values:
- Any positive integer.
Default value: 480.
`fsync` is not called for new parts, so for some time new parts exist only in the server's RAM (OS cache). If the server is rebooted spontaneously, new parts can be lost or damaged.
To protect data parts created by merges source parts are not deleted immediately. After merging several parts into a new part, ClickHouse marks the original parts as inactive and deletes them only after `old_parts_lifetime` seconds.
Inactive parts are removed if they are not used by current queries, i.e. if the `refcount` of the part is zero.
During startup ClickHouse checks the integrity of the parts.
If the merged part is damaged ClickHouse returns the inactive parts to the active list, and later merges them again. Then the damaged part is renamed (the `broken_` prefix is added) and moved to the `detached` folder.
If the merged part is not damaged, then the original inactive parts are renamed (the `ignored_` prefix is added) and moved to the `detached` folder.
The default `dirty_expire_centisecs` value (a Linux kernel setting) is 30 seconds (the maximum time that written data is stored only in RAM), but under heavy loads on the disk system data can be written much later. Experimentally, a value of 480 seconds was chosen for `old_parts_lifetime`, during which a new part is guaranteed to be written to disk.
## max_bytes_to_merge_at_max_space_in_pool {#max-bytes-to-merge-at-max-space-in-pool}
The maximum total parts size (in bytes) to be merged into one part, if there are enough resources available.
`max_bytes_to_merge_at_max_space_in_pool` -- roughly corresponds to the maximum possible part size created by an automatic background merge.
Possible values:
- Any positive integer.
Default value: 161061273600 (150 GB).
The merge scheduler periodically analyzes the sizes and number of parts in partitions, and if there is enough free resources in the pool, it starts background merges. Merges occur until the total size of the source parts is less than `max_bytes_to_merge_at_max_space_in_pool`.
Merges initiated by `optimize final` ignore `max_bytes_to_merge_at_max_space_in_pool` and merge parts only taking into account available resources (free disk's space) until one part remains in the partition.
## max_bytes_to_merge_at_min_space_in_pool {#max-bytes-to-merge-at-min-space-in-pool}
The maximum total part size (in bytes) to be merged into one part, with the minimum available resources in the background pool.
Possible values:
- Any positive integer.
Default value: 1048576 (1 MB)
`max_bytes_to_merge_at_min_space_in_pool` defines the maximum total size of parts which can be merged despite the lack of available disk space (in pool). This is necessary to reduce the number of small parts and the chance of `Too many parts` errors.
Merges book disk space by doubling the total merged parts sizes. Thus, with a small amount of free disk space, a situation may happen that there is free space, but this space is already booked by ongoing large merges, so other merges unable to start, and the number of small parts grows with every insert.
## merge_max_block_size {#merge-max-block-size}
The number of rows that are read from the merged parts into memory.
Possible values:
- Any positive integer.
Default value: 8192
Merge reads rows from parts in blocks of `merge_max_block_size` rows, then merges and writes the result into a new part. The read block is placed in RAM, so `merge_max_block_size` affects the size of the RAM required for the merge. Thus, merges can consume a large amount of RAM for tables with very wide rows (if the average row size is 100kb, then when merging 10 parts, (100kb * 10 * 8192) = ~ 8GB of RAM). By decreasing `merge_max_block_size`, you can reduce the amount of RAM required for a merge but slow down a merge.
## max_part_loading_threads {#max-part-loading-threads}
The maximum number of threads that read parts when ClickHouse starts.
Possible values:
- Any positive integer.
Default value: auto (number of CPU cores).
During startup ClickHouse reads all parts of all tables (reads files with metadata of parts) to build a list of all parts in memory. In some systems with a large number of parts this process can take a long time, and this time might be shortened by increasing `max_part_loading_threads` (if this process is not CPU and disk I/O bound).
[Original article](https://clickhouse.tech/docs/en/operations/settings/merge_tree_settings/) <!--hide-->

View File

@ -9,11 +9,18 @@ toc_title: Float32, Float64
Types are equivalent to types of C:
- `Float32` - `float`
- `Float64` - `double`
- `Float32` `float`.
- `Float64` `double`.
We recommend that you store data in integer form whenever possible. For example, convert fixed precision numbers to integer values, such as monetary amounts or page load times in milliseconds.
Aliases:
- `Float32``FLOAT`.
- `Float64``DOUBLE`.
When creating tables, numeric parameters for floating point numbers can be set (e.g. `FLOAT(12)`, `FLOAT(15, 22)`, `DOUBLE(12)`, `DOUBLE(4, 18)`), but ClickHouse ignores them.
## Using Floating-point Numbers {#using-floating-point-numbers}
- Computations with floating-point numbers might produce a rounding error.
@ -52,7 +59,7 @@ SELECT 0.5 / 0
└────────────────┘
```
- `-Inf` Negative infinity.
- `-Inf` Negative infinity.
<!-- -->
@ -66,7 +73,7 @@ SELECT -0.5 / 0
└─────────────────┘
```
- `NaN` Not a number.
- `NaN` Not a number.
<!-- -->
@ -80,6 +87,6 @@ SELECT 0 / 0
└──────────────┘
```
See the rules for `NaN` sorting in the section [ORDER BY clause](../sql_reference/statements/select/order-by.md).
See the rules for `NaN` sorting in the section [ORDER BY clause](../../sql-reference/statements/select/order-by.md).
[Original article](https://clickhouse.tech/docs/en/data_types/float/) <!--hide-->

View File

@ -7,23 +7,32 @@ toc_title: UInt8, UInt16, UInt32, UInt64, UInt256, Int8, Int16, Int32, Int64, In
Fixed-length integers, with or without a sign.
When creating tables, numeric parameters for integer numbers can be set (e.g. `TINYINT(8)`, `SMALLINT(16)`, `INT(32)`, `BIGINT(64)`), but ClickHouse ignores them.
## Int Ranges {#int-ranges}
- Int8 - \[-128 : 127\]
- Int16 - \[-32768 : 32767\]
- Int32 - \[-2147483648 : 2147483647\]
- Int64 - \[-9223372036854775808 : 9223372036854775807\]
- Int128 - \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
- Int256 - \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
- `Int8` — \[-128 : 127\]
- `Int16` — \[-32768 : 32767\]
- `Int32` — \[-2147483648 : 2147483647\]
- `Int64` — \[-9223372036854775808 : 9223372036854775807\]
- `Int128` — \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
- `Int256` — \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
Aliases:
- `Int8``TINYINT`, `BOOL`, `BOOLEAN`, `INT1`.
- `Int16``SMALLINT`, `INT2`.
- `Int32``INT`, `INT4`, `INTEGER`.
- `Int64``BIGINT`.
## Uint Ranges {#uint-ranges}
- UInt8 - \[0 : 255\]
- UInt16 - \[0 : 65535\]
- UInt32 - \[0 : 4294967295\]
- UInt64 - \[0 : 18446744073709551615\]
- UInt256 - \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
- `UInt8` \[0 : 255\]
- `UInt16` \[0 : 65535\]
- `UInt32` \[0 : 4294967295\]
- `UInt64` \[0 : 18446744073709551615\]
- `UInt256` \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
UInt128 is not supported yet.
`UInt128` is not supported yet.
[Original article](https://clickhouse.tech/docs/en/data_types/int_uint/) <!--hide-->

View File

@ -8,6 +8,8 @@ toc_title: String
Strings of an arbitrary length. The length is not limited. The value can contain an arbitrary set of bytes, including null bytes.
The String type replaces the types VARCHAR, BLOB, CLOB, and others from other DBMSs.
When creating tables, numeric parameters for string fields can be set (e.g. `VARCHAR(255)`), but ClickHouse ignores them.
## Encodings {#encodings}
ClickHouse doesnt have the concept of encodings. Strings can contain an arbitrary set of bytes, which are stored and output as-is.

View File

@ -93,7 +93,7 @@ Setting fields:
- `path` The absolute path to the file.
- `format` The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported.
When dictionary with FILE source is created via DDL command (`CREATE DICTIONARY ...`), source of the dictionary have to be located in `user_files` directory, to prevent DB users accessing arbitrary file on clickhouse node.
When dictionary with source `FILE` is created via DDL command (`CREATE DICTIONARY ...`), the source file needs to be located in `user_files` directory, to prevent DB users accessing arbitrary file on ClickHouse node.
## Executable File {#dicts-external_dicts_dict_sources-executable}
@ -115,7 +115,7 @@ Setting fields:
- `command` The absolute path to the executable file, or the file name (if the program directory is written to `PATH`).
- `format` The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported.
That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on clickhouse node.
That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on ClickHouse node.
## Http(s) {#dicts-external_dicts_dict_sources-http}
@ -160,14 +160,14 @@ Setting fields:
- `url` The source URL.
- `format` The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported.
- `credentials` Basic HTTP authentication. Optional parameter.
- `user` Username required for the authentication.
- `password` Password required for the authentication.
- `user` Username required for the authentication.
- `password` Password required for the authentication.
- `headers` All custom HTTP headers entries used for the HTTP request. Optional parameter.
- `header` Single HTTP header entry.
- `name` Identifiant name used for the header send on the request.
- `value` Value set for a specific identifiant name.
- `header` Single HTTP header entry.
- `name` Identifiant name used for the header send on the request.
- `value` Value set for a specific identifiant name.
When creating a dictionary using the DDL command (`CREATE DICTIONARY ...`) remote hosts for HTTP dictionaries checked with the `remote_url_allow_hosts` section from config to prevent database users to access arbitrary HTTP server.
When creating a dictionary using the DDL command (`CREATE DICTIONARY ...`) remote hosts for HTTP dictionaries are checked against the contents of `remote_url_allow_hosts` section from config to prevent database users to access arbitrary HTTP server.
## ODBC {#dicts-external_dicts_dict_sources-odbc}

View File

@ -622,7 +622,7 @@ Using replacement fields, you can define a pattern for the resulting string. “
| %C | year divided by 100 and truncated to integer (00-99) | 20 |
| %d | day of the month, zero-padded (01-31) | 02 |
| %D | Short MM/DD/YY date, equivalent to %m/%d/%y | 01/02/18 |
| %e | day of the month, space-padded ( 1-31) | 2 |
| %e | day of the month, space-padded ( 1-31) | 2 |
| %F | short YYYY-MM-DD date, equivalent to %Y-%m-%d | 2018-01-02 |
| %G | four-digit year format for ISO week number, calculated from the week-based year [defined by the ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Week_dates) standard, normally useful only with %V | 2018 |
| %g | two-digit year format, aligned to ISO 8601, abbreviated from four-digit notation | 18 |
@ -633,6 +633,7 @@ Using replacement fields, you can define a pattern for the resulting string. “
| %M | minute (00-59) | 33 |
| %n | new-line character () | |
| %p | AM or PM designation | PM |
| %Q | Quarter (1-4) | 1 |
| %R | 24-hour HH:MM time, equivalent to %H:%M | 22:33 |
| %S | second (00-59) | 44 |
| %t | horizontal-tab character () | |

View File

@ -25,6 +25,7 @@ SELECT [DISTINCT] expr_list
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
[LIMIT [offset_value, ]n BY columns]
[LIMIT [n, ]m] [WITH TIES]
[SETTINGS ...]
[UNION ...]
[INTO OUTFILE filename]
[FORMAT format]
@ -265,5 +266,17 @@ SELECT * REPLACE(i + 1 AS i) EXCEPT (j) APPLY(sum) from columns_transformers;
└─────────────────┴────────┘
```
## SETTINGS in SELECT Query {#settings-in-select}
You can specify the necessary settings right in the `SELECT` query. The setting value is applied only to this query and is reset to default or previous value after the query is executed.
Other ways to make settings see [here](../../../operations/settings/index.md).
**Example**
``` sql
SELECT * FROM some_table SETTINGS optimize_read_in_order=1, cast_keep_nullable=1;
```
[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/)
<!--hide-->

File diff suppressed because it is too large Load Diff

View File

@ -1,7 +1,7 @@
---
toc_folder_title: Changelog
toc_priority: 74
toc_title: '2020'
toc_title: '2021'
---
{% include "content/changelog.md" %}

View File

@ -24,6 +24,7 @@ toc_title: Introduction
- При запуске консольного клиента ClickHouse в не интерактивном режиме установите параметр запуска `--setting=value`.
- При использовании HTTP API передавайте cgi-параметры (`URL?setting_1=value&setting_2=value...`).
- Укажите необходимые настройки в секции [SETTINGS](../../sql-reference/statements/select/index.md#settings-in-select) запроса SELECT. Эти настройки действуют только в рамках данного запроса, а после его выполнения сбрасываются до предыдущего значения или значения по умолчанию.
Настройки, которые можно задать только в конфигурационном файле сервера, в разделе не рассматриваются.

View File

@ -9,8 +9,15 @@ toc_title: Float32, Float64
Типы эквивалентны типам языка С:
- `Float32` - `float`;
- `Float64` - `double`.
- `Float32``float`.
- `Float64``double`.
Синонимы:
- `Float32``FLOAT`.
- `Float64``DOUBLE`.
При создании таблиц для чисел с плавающей запятой можно указывать числовые параметры (например, `FLOAT(12)`, `FLOAT(15, 22)`, `DOUBLE(12)`, `DOUBLE(4, 18)`), но ClickHouse их проигнорирует.
Рекомендуется хранить данные в целочисленном виде всегда, когда это возможно. Например, переводите в целочисленные значения числа с фиксированной точностью, такие как денежные суммы или времена загрузки страниц в миллисекундах.
@ -38,7 +45,7 @@ SELECT 1 - 0.9
В отличие от стандартного SQL, ClickHouse поддерживает следующие категории чисел с плавающей запятой:
- `Inf` - бесконечность.
- `Inf` бесконечность.
<!-- -->
@ -52,7 +59,7 @@ SELECT 0.5 / 0
└────────────────┘
```
- `-Inf` - отрицательная бесконечность;
- `-Inf` — отрицательная бесконечность.
<!-- -->
@ -66,7 +73,7 @@ SELECT -0.5 / 0
└─────────────────┘
```
- `NaN` - не число.
- `NaN` не число.
<!-- -->
@ -80,6 +87,6 @@ SELECT 0 / 0
└──────────────┘
```
Смотрите правила сортировки `NaN` в разделе [Секция ORDER BY](../sql_reference/data_types/float.md).
Смотрите правила сортировки `NaN` в разделе [Секция ORDER BY ](../../sql-reference/statements/select/order-by.md).
[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/float/) <!--hide-->

View File

@ -7,23 +7,32 @@ toc_title: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64
Целые числа фиксированной длины, без знака или со знаком.
При создании таблиц для целых чисел можно указывать числовые параметры (например `TINYINT(8)`, `SMALLINT(16)`, `INT(32)`, `BIGINT(64)`), но ClickHouse их проигнорирует.
## Диапазоны Int {#int-ranges}
- Int8 - \[-128 : 127\]
- Int16 - \[-32768 : 32767\]
- Int32 - \[-2147483648 : 2147483647\]
- Int64 - \[-9223372036854775808 : 9223372036854775807\]
- Int128 - \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
- Int256 - \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
- `Int8` — \[-128 : 127\]
- `Int16` — \[-32768 : 32767\]
- `Int32` — \[-2147483648 : 2147483647\]
- `Int64` — \[-9223372036854775808 : 9223372036854775807\]
- `Int128` — \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
- `Int256` — \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
Синонимы:
- `Int8``TINYINT`, `BOOL`, `BOOLEAN`, `INT1`.
- `Int16``SMALLINT`, `INT2`.
- `Int32``INT`, `INT4`, `INTEGER`.
- `Int64``BIGINT`.
## Диапазоны Uint {#uint-ranges}
- UInt8 - \[0 : 255\]
- UInt16 - \[0 : 65535\]
- UInt32 - \[0 : 4294967295\]
- UInt64 - \[0 : 18446744073709551615\]
- UInt256 - \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
- `UInt8` \[0 : 255\]
- `UInt16` \[0 : 65535\]
- `UInt32` \[0 : 4294967295\]
- `UInt64` \[0 : 18446744073709551615\]
- `UInt256` \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
UInt128 пока не реализован.
`UInt128` пока не реализован.
[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/int_uint/) <!--hide-->

View File

@ -8,6 +8,8 @@ toc_title: String
Строки произвольной длины. Длина не ограничена. Значение может содержать произвольный набор байт, включая нулевые байты.
Таким образом, тип String заменяет типы VARCHAR, BLOB, CLOB и т. п. из других СУБД.
При создании таблиц для строк можно указывать числовые параметры (например `VARCHAR(255)`), но СlickHouse их проигнорирует.
## Кодировки {#kodirovki}
В ClickHouse нет понятия кодировок. Строки могут содержать произвольный набор байт, который хранится и выводится, как есть.

View File

@ -90,8 +90,10 @@ SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated'))
Поля настройки:
- `path` — Абсолютный путь к файлу.
- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
- `path` — абсолютный путь к файлу.
- `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
Если словарь с источником `FILE` создается с помощью DDL-команды (`CREATE DICTIONARY ...`), источник словаря должен быть расположен в каталоге `user_files`. Иначе пользователи базы данных будут иметь доступ к произвольному файлу на узле ClickHouse.
## Исполняемый файл {#dicts-external_dicts_dict_sources-executable}
@ -108,16 +110,12 @@ SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated'))
</source>
```
или
``` sql
SOURCE(EXECUTABLE(command 'cat /opt/dictionaries/os.tsv' format 'TabSeparated'))
```
Поля настройки:
- `command` — Абсолютный путь к исполняемому файлу или имя файла (если каталог программы прописан в `PATH`).
- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
- `command` — абсолютный путь к исполняемому файлу или имя файла (если каталог программы прописан в `PATH`).
- `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
Этот источник словаря может быть настроен только с помощью XML-конфигурации. Создание словарей с исполняемым источником с помощью DDL отключено. Иначе пользователь базы данных сможет выполнить произвольный бинарный файл на узле ClickHouse.
## HTTP(s) {#dicts-external_dicts_dict_sources-http}
@ -160,7 +158,16 @@ SOURCE(HTTP(
Поля настройки:
- `url` — URL источника.
- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
- `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
- `credentials` базовая HTTP-аутентификация. Необязательный параметр.
- `user` имя пользователя, необходимое для аутентификации.
- `password` пароль, необходимый для аутентификации.
- `headers` все пользовательские записи HTTP-заголовков, используемые для HTTP-запроса. Необязательный параметр.
- `header` одна запись HTTP-заголовка.
- `name` идентифицирующее имя, используемое для отправки заголовка запроса.
- `value` значение, заданное для конкретного идентифицирующего имени.
При создании словаря с помощью DDL-команды (`CREATE DICTIONARY ...`) удаленные хосты для HTTP-словарей проверяются в разделе `remote_url_allow_hosts` из конфигурации сервера. Иначе пользователи базы данных будут иметь доступ к произвольному HTTP-серверу.
## ODBC {#dicts-external_dicts_dict_sources-odbc}

View File

@ -23,6 +23,7 @@ SELECT [DISTINCT] expr_list
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
[LIMIT [offset_value, ]n BY columns]
[LIMIT [n, ]m] [WITH TIES]
[SETTINGS ...]
[UNION ALL ...]
[INTO OUTFILE filename]
[FORMAT format]
@ -161,4 +162,17 @@ Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of
Подробнее смотрите в разделе «Настройки». Присутствует возможность использовать внешнюю сортировку (с сохранением временных данных на диск) и внешнюю агрегацию.
{## [Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/statements/select/) ##}
## SETTINGS в запросе SELECT {#settings-in-select}
Вы можете задать значения необходимых настроек непосредственно в запросе `SELECT` в секции `SETTINGS`. Эти настройки действуют только в рамках данного запроса, а после его выполнения сбрасываются до предыдущего значения или значения по умолчанию.
Другие способы задания настроек описаны [здесь](../../../operations/settings/index.md).
**Пример**
``` sql
SELECT * FROM some_table SETTINGS optimize_read_in_order=1, cast_keep_nullable=1;
```
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/)
<!--hide-->

View File

@ -613,24 +613,25 @@ formatDateTime(Time, Format\[, Timezone\])
| %C | 年除以100并截断为整数(00-99) | 20 |
| %d | 月中的一天零填充01-31) | 02 |
| %D | 短MM/DD/YY日期相当于%m/%d/%y | 01/02/2018 |
| %e | 月中的一天空格填充1-31) | 2 |
| %e | 月中的一天,空格填充( 1-31) | 2 |
| %F | 短YYYY-MM-DD日期相当于%Y-%m-%d | 2018-01-02 |
| %G | ISO周号的四位数年份格式 从基于周的年份[由ISO 8601定义](https://en.wikipedia.org/wiki/ISO_8601#Week_dates) 标准计算得出通常仅对V有用 | 2018 |
| %g | 两位数的年份格式与ISO 8601一致四位数表示法的缩写 | 18 |
| %H | 24小时格式00-23) | 22 |
| %I | 小时12h格式01-12) | 10 |
| %j | 一年(001-366) | 002 |
| %I | 12小时格式01-12) | 10 |
| %j | 一年中的一天 (001-366) | 002 |
| %m | 月份为十进制数01-12) | 01 |
| %M | 分钟(00-59) | 33 |
| %n | 换行符(") | |
| %p | AM或PM指定 | PM |
| %Q | 季度1-4) | 1 |
| %R | 24小时HH:MM时间相当于%H:%M | 22:33 |
| %S | 第二(00-59) | 44 |
| %S | (00-59) | 44 |
| %t | 水平制表符() | |
| %T | ISO8601时间格式(HH:MM:SS),相当于%H:%M:%S | 22:33:44 |
| %u | ISO8601平日as编号星期一为1(1-7) | 2 |
| %u | ISO8601工作日为数字星期一为1(1-7) | 2 |
| %V | ISO8601周编号(01-53) | 01 |
| %w | 日为十进制数周日为0(0-6) | 2 |
| %w | 工作日为十进制数周日为0(0-6) | 2 |
| %y | 年份最后两位数字00-99) | 18 |
| %Y | 年 | 2018 |
| %% | %符号 | % |

View File

@ -141,6 +141,8 @@ private:
/// If not empty, queries will be read from these files
std::vector<std::string> queries_files;
/// If not empty, run queries from these files before processing every file from 'queries_files'.
std::vector<std::string> interleave_queries_files;
std::unique_ptr<Connection> connection; /// Connection to DB.
String full_query; /// Current query as it was given to the client.
@ -796,13 +798,22 @@ private:
if (!queries_files.empty())
{
for (const auto & queries_file : queries_files)
auto process_file = [&](const std::string & file)
{
connection->setDefaultDatabase(connection_parameters.default_database);
ReadBufferFromFile in(queries_file);
ReadBufferFromFile in(file);
readStringUntilEOF(text, in);
if (!processMultiQuery(text))
break;
return processMultiQuery(text);
};
for (const auto & queries_file : queries_files)
{
for (const auto & interleave_file : interleave_queries_files)
if (!process_file(interleave_file))
return;
if (!process_file(queries_file))
return;
}
return;
}
@ -945,6 +956,7 @@ private:
continue;
}
return true;
}
@ -2271,7 +2283,9 @@ public:
("highlight", po::value<bool>()->default_value(true), "enable or disable basic syntax highlight in interactive command line")
("log-level", po::value<std::string>(), "client log level")
("server_logs_file", po::value<std::string>(), "put server logs into specified file")
("query-fuzzer-runs", po::value<int>()->default_value(0), "query fuzzer runs")
("query-fuzzer-runs", po::value<int>()->default_value(0), "After executing every SELECT query, do random mutations in it and run again specified number of times. This is used for testing to discover unexpected corner cases.")
("interleave-queries-file", po::value<std::vector<std::string>>()->multitoken(),
"file path with queries to execute before every file from 'queries-file'; multiple files can be specified (--queries-file file1 file2...); this is needed to enable more aggressive fuzzing of newly added tests (see 'query-fuzzer-runs' option)")
("opentelemetry-traceparent", po::value<std::string>(), "OpenTelemetry traceparent header as described by W3C Trace Context recommendation")
("opentelemetry-tracestate", po::value<std::string>(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation")
("history_file", po::value<std::string>(), "path to history file")
@ -2386,6 +2400,8 @@ public:
config().setString("query", options["query"].as<std::string>());
if (options.count("queries-file"))
queries_files = options["queries-file"].as<std::vector<std::string>>();
if (options.count("interleave-queries-file"))
interleave_queries_files = options["interleave-queries-file"].as<std::vector<std::string>>();
if (options.count("database"))
config().setString("database", options["database"].as<std::string>());
if (options.count("pager"))

View File

@ -692,6 +692,37 @@ int Server::main(const std::vector<std::string> & /*args*/)
{
Settings::checkNoSettingNamesAtTopLevel(*config, config_path);
/// Limit on total memory usage
size_t max_server_memory_usage = config->getUInt64("max_server_memory_usage", 0);
double max_server_memory_usage_to_ram_ratio = config->getDouble("max_server_memory_usage_to_ram_ratio", 0.9);
size_t default_max_server_memory_usage = memory_amount * max_server_memory_usage_to_ram_ratio;
if (max_server_memory_usage == 0)
{
max_server_memory_usage = default_max_server_memory_usage;
LOG_INFO(log, "Setting max_server_memory_usage was set to {}"
" ({} available * {:.2f} max_server_memory_usage_to_ram_ratio)",
formatReadableSizeWithBinarySuffix(max_server_memory_usage),
formatReadableSizeWithBinarySuffix(memory_amount),
max_server_memory_usage_to_ram_ratio);
}
else if (max_server_memory_usage > default_max_server_memory_usage)
{
max_server_memory_usage = default_max_server_memory_usage;
LOG_INFO(log, "Setting max_server_memory_usage was lowered to {}"
" because the system has low amount of memory. The amount was"
" calculated as {} available"
" * {:.2f} max_server_memory_usage_to_ram_ratio",
formatReadableSizeWithBinarySuffix(max_server_memory_usage),
formatReadableSizeWithBinarySuffix(memory_amount),
max_server_memory_usage_to_ram_ratio);
}
total_memory_tracker.setHardLimit(max_server_memory_usage);
total_memory_tracker.setDescription("(total)");
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
// FIXME logging-related things need synchronization -- see the 'Logger * log' saved
// in a lot of places. For now, disable updating log configuration without server restart.
//setTextLog(global_context->getTextLog());
@ -780,37 +811,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->getMergeTreeSettings().sanityCheck(settings);
global_context->getReplicatedMergeTreeSettings().sanityCheck(settings);
/// Limit on total memory usage
size_t max_server_memory_usage = config().getUInt64("max_server_memory_usage", 0);
double max_server_memory_usage_to_ram_ratio = config().getDouble("max_server_memory_usage_to_ram_ratio", 0.9);
size_t default_max_server_memory_usage = memory_amount * max_server_memory_usage_to_ram_ratio;
if (max_server_memory_usage == 0)
{
max_server_memory_usage = default_max_server_memory_usage;
LOG_INFO(log, "Setting max_server_memory_usage was set to {}"
" ({} available * {:.2f} max_server_memory_usage_to_ram_ratio)",
formatReadableSizeWithBinarySuffix(max_server_memory_usage),
formatReadableSizeWithBinarySuffix(memory_amount),
max_server_memory_usage_to_ram_ratio);
}
else if (max_server_memory_usage > default_max_server_memory_usage)
{
max_server_memory_usage = default_max_server_memory_usage;
LOG_INFO(log, "Setting max_server_memory_usage was lowered to {}"
" because the system has low amount of memory. The amount was"
" calculated as {} available"
" * {:.2f} max_server_memory_usage_to_ram_ratio",
formatReadableSizeWithBinarySuffix(max_server_memory_usage),
formatReadableSizeWithBinarySuffix(memory_amount),
max_server_memory_usage_to_ram_ratio);
}
total_memory_tracker.setOrRaiseHardLimit(max_server_memory_usage);
total_memory_tracker.setDescription("(total)");
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0);
Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024));

View File

@ -29,61 +29,74 @@ struct AggregateFunctionSumData
template <typename Value>
void NO_INLINE addMany(const Value * __restrict ptr, size_t count)
{
/// Compiler cannot unroll this loop, do it manually.
/// (at least for floats, most likely due to the lack of -fassociative-math)
/// Something around the number of SSE registers * the number of elements fit in register.
constexpr size_t unroll_count = 128 / sizeof(T);
T partial_sums[unroll_count]{};
const auto * end = ptr + count;
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
while (ptr < unrolled_end)
if constexpr (std::is_floating_point_v<T>)
{
/// Compiler cannot unroll this loop, do it manually.
/// (at least for floats, most likely due to the lack of -fassociative-math)
/// Something around the number of SSE registers * the number of elements fit in register.
constexpr size_t unroll_count = 128 / sizeof(T);
T partial_sums[unroll_count]{};
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
while (ptr < unrolled_end)
{
for (size_t i = 0; i < unroll_count; ++i)
partial_sums[i] += ptr[i];
ptr += unroll_count;
}
for (size_t i = 0; i < unroll_count; ++i)
partial_sums[i] += ptr[i];
ptr += unroll_count;
sum += partial_sums[i];
}
for (size_t i = 0; i < unroll_count; ++i)
sum += partial_sums[i];
/// clang cannot vectorize the loop if accumulator is class member instead of local variable.
T local_sum{};
while (ptr < end)
{
sum += *ptr;
local_sum += *ptr;
++ptr;
}
sum += local_sum;
}
template <typename Value>
void NO_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count)
{
constexpr size_t unroll_count = 128 / sizeof(T);
T partial_sums[unroll_count]{};
const auto * end = ptr + count;
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
while (ptr < unrolled_end)
if constexpr (std::is_floating_point_v<T>)
{
constexpr size_t unroll_count = 128 / sizeof(T);
T partial_sums[unroll_count]{};
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
while (ptr < unrolled_end)
{
for (size_t i = 0; i < unroll_count; ++i)
if (!null_map[i])
partial_sums[i] += ptr[i];
ptr += unroll_count;
null_map += unroll_count;
}
for (size_t i = 0; i < unroll_count; ++i)
if (!null_map[i])
partial_sums[i] += ptr[i];
ptr += unroll_count;
null_map += unroll_count;
sum += partial_sums[i];
}
for (size_t i = 0; i < unroll_count; ++i)
sum += partial_sums[i];
T local_sum{};
while (ptr < end)
{
if (!*null_map)
sum += *ptr;
local_sum += *ptr;
++ptr;
++null_map;
}
sum += local_sum;
}
void merge(const AggregateFunctionSumData & rhs)

View File

@ -145,8 +145,9 @@ void MemoryTracker::alloc(Int64 size)
*/
Int64 will_be = size + amount.fetch_add(size, std::memory_order_relaxed);
if (metric != CurrentMetrics::end())
CurrentMetrics::add(metric, size);
auto metric_loaded = metric.load(std::memory_order_relaxed);
if (metric_loaded != CurrentMetrics::end())
CurrentMetrics::add(metric_loaded, size);
Int64 current_hard_limit = hard_limit.load(std::memory_order_relaxed);
Int64 current_profiler_limit = profiler_limit.load(std::memory_order_relaxed);
@ -286,8 +287,9 @@ void MemoryTracker::free(Int64 size)
if (auto * loaded_next = parent.load(std::memory_order_relaxed))
loaded_next->free(size);
if (metric != CurrentMetrics::end())
CurrentMetrics::sub(metric, accounted_size);
auto metric_loaded = metric.load(std::memory_order_relaxed);
if (metric_loaded != CurrentMetrics::end())
CurrentMetrics::sub(metric_loaded, accounted_size);
}
@ -302,8 +304,9 @@ void MemoryTracker::resetCounters()
void MemoryTracker::reset()
{
if (metric != CurrentMetrics::end())
CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed));
auto metric_loaded = metric.load(std::memory_order_relaxed);
if (metric_loaded != CurrentMetrics::end())
CurrentMetrics::sub(metric_loaded, amount.load(std::memory_order_relaxed));
resetCounters();
}
@ -316,6 +319,12 @@ void MemoryTracker::set(Int64 to)
}
void MemoryTracker::setHardLimit(Int64 value)
{
hard_limit.store(value, std::memory_order_relaxed);
}
void MemoryTracker::setOrRaiseHardLimit(Int64 value)
{
/// This is just atomic set to maximum.

View File

@ -53,7 +53,7 @@ private:
std::atomic<MemoryTracker *> parent {};
/// You could specify custom metric to track memory usage.
CurrentMetrics::Metric metric = CurrentMetrics::end();
std::atomic<CurrentMetrics::Metric> metric = CurrentMetrics::end();
/// This description will be used as prefix into log messages (if isn't nullptr)
std::atomic<const char *> description_ptr = nullptr;
@ -96,6 +96,8 @@ public:
return peak.load(std::memory_order_relaxed);
}
void setHardLimit(Int64 value);
/** Set limit if it was not set.
* Otherwise, set limit to new value, if new value is greater than previous limit.
*/
@ -132,7 +134,7 @@ public:
/// The memory consumption could be shown in realtime via CurrentMetrics counter
void setMetric(CurrentMetrics::Metric metric_)
{
metric = metric_;
metric.store(metric_, std::memory_order_relaxed);
}
void setDescription(const char * description)

View File

@ -67,6 +67,7 @@ class IColumn;
M(UInt64, s3_min_upload_part_size, 512*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \
M(UInt64, s3_max_single_part_upload_size, 64*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \
M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \
M(UInt64, s3_max_connections, 1024, "The maximum number of connections per server.", 0) \
M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \
M(Bool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.", 0) \
M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \

View File

@ -120,6 +120,7 @@ void registerDiskS3(DiskFactory & factory)
cfg.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 10000);
cfg.httpRequestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 5000);
cfg.maxConnections = config.getUInt(config_prefix + ".max_connections", 100);
cfg.endpointOverride = uri.endpoint;
auto proxy_config = getProxyConfiguration(config_prefix, config);

View File

@ -272,6 +272,11 @@ private:
writeNumber2(target + 3, ToMinuteImpl::execute(source, timezone));
writeNumber2(target + 6, ToSecondImpl::execute(source, timezone));
}
static void quarter(char * target, Time source, const DateLUTImpl & timezone)
{
*target += ToQuarterImpl::execute(source, timezone);
}
};
public:
@ -621,6 +626,12 @@ public:
result.append("0000");
break;
// Quarter (1-4)
case 'Q':
instructions.template emplace_back(&Action<T>::quarter, 1);
result.append("0");
break;
/// Time components. If the argument is Date, not a DateTime, then this components will have default value.
// Minute (00-59)

View File

@ -106,23 +106,72 @@ namespace
const std::string host;
const UInt16 port;
bool https;
const String proxy_host;
const UInt16 proxy_port;
bool proxy_https;
bool resolve_host;
using Base = PoolBase<Poco::Net::HTTPClientSession>;
ObjectPtr allocObject() override
{
return makeHTTPSessionImpl(host, port, https, true);
auto session = makeHTTPSessionImpl(host, port, https, true, resolve_host);
if (!proxy_host.empty())
{
const String proxy_scheme = proxy_https ? "https" : "http";
session->setProxyHost(proxy_host);
session->setProxyPort(proxy_port);
#if !defined(ARCADIA_BUILD)
session->setProxyProtocol(proxy_scheme);
/// Turn on tunnel mode if proxy scheme is HTTP while endpoint scheme is HTTPS.
session->setProxyTunnel(!proxy_https && https);
#endif
}
return session;
}
public:
SingleEndpointHTTPSessionPool(const std::string & host_, UInt16 port_, bool https_, size_t max_pool_size_)
: Base(max_pool_size_, &Poco::Logger::get("HTTPSessionPool")), host(host_), port(port_), https(https_)
SingleEndpointHTTPSessionPool(
const std::string & host_,
UInt16 port_,
bool https_,
const std::string & proxy_host_,
UInt16 proxy_port_,
bool proxy_https_,
size_t max_pool_size_,
bool resolve_host_ = true)
: Base(max_pool_size_, &Poco::Logger::get("HTTPSessionPool"))
, host(host_)
, port(port_)
, https(https_)
, proxy_host(proxy_host_)
, proxy_port(proxy_port_)
, proxy_https(proxy_https_)
, resolve_host(resolve_host_)
{
}
};
class HTTPSessionPool : private boost::noncopyable
{
public:
struct Key
{
String target_host;
UInt16 target_port;
bool is_target_https;
String proxy_host;
UInt16 proxy_port;
bool is_proxy_https;
bool operator ==(const Key & rhs) const
{
return std::tie(target_host, target_port, is_target_https, proxy_host, proxy_port, is_proxy_https)
== std::tie(rhs.target_host, rhs.target_port, rhs.is_target_https, rhs.proxy_host, rhs.proxy_port, rhs.is_proxy_https);
}
};
private:
using Key = std::tuple<std::string, UInt16, bool>;
using PoolPtr = std::shared_ptr<SingleEndpointHTTPSessionPool>;
using Entry = SingleEndpointHTTPSessionPool::Entry;
@ -131,9 +180,12 @@ namespace
size_t operator()(const Key & k) const
{
SipHash s;
s.update(std::get<0>(k));
s.update(std::get<1>(k));
s.update(std::get<2>(k));
s.update(k.target_host);
s.update(k.target_port);
s.update(k.is_target_https);
s.update(k.proxy_host);
s.update(k.proxy_port);
s.update(k.is_proxy_https);
return s.get64();
}
};
@ -153,18 +205,32 @@ namespace
Entry getSession(
const Poco::URI & uri,
const Poco::URI & proxy_uri,
const ConnectionTimeouts & timeouts,
size_t max_connections_per_endpoint)
size_t max_connections_per_endpoint,
bool resolve_host = true)
{
std::unique_lock lock(mutex);
const std::string & host = uri.getHost();
UInt16 port = uri.getPort();
bool https = isHTTPS(uri);
auto key = std::make_tuple(host, port, https);
String proxy_host;
UInt16 proxy_port = 0;
bool proxy_https = false;
if (!proxy_uri.empty())
{
proxy_host = proxy_uri.getHost();
proxy_port = proxy_uri.getPort();
proxy_https = isHTTPS(proxy_uri);
}
HTTPSessionPool::Key key{host, port, https, proxy_host, proxy_port, proxy_https};
auto pool_ptr = endpoints_pool.find(key);
if (pool_ptr == endpoints_pool.end())
std::tie(pool_ptr, std::ignore) = endpoints_pool.emplace(
key, std::make_shared<SingleEndpointHTTPSessionPool>(host, port, https, max_connections_per_endpoint));
key, std::make_shared<SingleEndpointHTTPSessionPool>(host, port, https, proxy_host, proxy_port, proxy_https, max_connections_per_endpoint, resolve_host));
auto retry_timeout = timeouts.connection_timeout.totalMicroseconds();
auto session = pool_ptr->second->get(retry_timeout);
@ -178,13 +244,17 @@ namespace
if (!msg.empty())
{
LOG_TRACE((&Poco::Logger::get("HTTPCommon")), "Failed communicating with {} with error '{}' will try to reconnect session", host, msg);
/// Host can change IP
const auto ip = DNSResolver::instance().resolveHost(host).toString();
if (ip != session->getHost())
if (resolve_host)
{
session->reset();
session->setHost(ip);
session->attachSessionData({});
/// Host can change IP
const auto ip = DNSResolver::instance().resolveHost(host).toString();
if (ip != session->getHost())
{
session->reset();
session->setHost(ip);
session->attachSessionData({});
}
}
}
}
@ -218,9 +288,14 @@ HTTPSessionPtr makeHTTPSession(const Poco::URI & uri, const ConnectionTimeouts &
}
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size)
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size, bool resolve_host)
{
return HTTPSessionPool::instance().getSession(uri, timeouts, per_endpoint_pool_size);
return makePooledHTTPSession(uri, {}, timeouts, per_endpoint_pool_size, resolve_host);
}
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const Poco::URI & proxy_uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size, bool resolve_host)
{
return HTTPSessionPool::instance().getSession(uri, proxy_uri, timeouts, per_endpoint_pool_size, resolve_host);
}
bool isRedirect(const Poco::Net::HTTPResponse::HTTPStatus status) { return status == Poco::Net::HTTPResponse::HTTP_MOVED_PERMANENTLY || status == Poco::Net::HTTPResponse::HTTP_FOUND || status == Poco::Net::HTTPResponse::HTTP_SEE_OTHER || status == Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT; }

View File

@ -50,8 +50,9 @@ void setResponseDefaultHeaders(Poco::Net::HTTPServerResponse & response, unsigne
/// Create session object to perform requests and set required parameters.
HTTPSessionPtr makeHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, bool resolve_host = true);
/// As previous method creates session, but tooks it from pool
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size);
/// As previous method creates session, but tooks it from pool, without and with proxy uri.
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size, bool resolve_host = true);
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const Poco::URI & proxy_uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size, bool resolve_host = true);
bool isRedirect(const Poco::Net::HTTPResponse::HTTPStatus status);

View File

@ -6,7 +6,7 @@
#include <utility>
#include <IO/HTTPCommon.h>
#include <IO/S3/PocoHTTPResponseStream.h>
#include <IO/S3/SessionAwareAwsStream.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#include <Common/Stopwatch.h>
@ -86,6 +86,7 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & clientConfigu
))
, remote_host_filter(clientConfiguration.remote_host_filter)
, s3_max_redirects(clientConfiguration.s3_max_redirects)
, max_connections(clientConfiguration.maxConnections)
{
}
@ -164,28 +165,24 @@ void PocoHTTPClient::makeRequestInternal(
{
for (unsigned int attempt = 0; attempt <= s3_max_redirects; ++attempt)
{
Poco::URI poco_uri(uri);
/// Reverse proxy can replace host header with resolved ip address instead of host name.
/// This can lead to request signature difference on S3 side.
auto session = makeHTTPSession(poco_uri, timeouts, false);
Poco::URI target_uri(uri);
Poco::URI proxy_uri;
auto request_configuration = per_request_configuration(request);
if (!request_configuration.proxyHost.empty())
{
/// Turn on tunnel mode if proxy scheme is HTTP while endpoint scheme is HTTPS.
bool use_tunnel = request_configuration.proxyScheme == Aws::Http::Scheme::HTTP && poco_uri.getScheme() == "https";
session->setProxy(
request_configuration.proxyHost,
request_configuration.proxyPort,
Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme),
use_tunnel
);
proxy_uri.setScheme(Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme));
proxy_uri.setHost(request_configuration.proxyHost);
proxy_uri.setPort(request_configuration.proxyPort);
}
/// Reverse proxy can replace host header with resolved ip address instead of host name.
/// This can lead to request signature difference on S3 side.
auto session = makePooledHTTPSession(target_uri, proxy_uri, timeouts, max_connections, false);
Poco::Net::HTTPRequest poco_request(Poco::Net::HTTPRequest::HTTP_1_1);
poco_request.setURI(poco_uri.getPathAndQuery());
poco_request.setURI(target_uri.getPathAndQuery());
switch (request.GetMethod())
{
@ -281,7 +278,7 @@ void PocoHTTPClient::makeRequestInternal(
}
}
else
response->GetResponseStream().SetUnderlyingStream(std::make_shared<PocoHTTPResponseStream>(session, response_body_stream));
response->GetResponseStream().SetUnderlyingStream(std::make_shared<SessionAwareAwsStream<decltype(session)>>(session, response_body_stream));
return;
}
@ -297,6 +294,7 @@ void PocoHTTPClient::makeRequestInternal(
ProfileEvents::increment(select_metric(S3MetricType::Errors));
}
}
}
#endif

View File

@ -56,6 +56,7 @@ private:
ConnectionTimeouts timeouts;
const RemoteHostFilter & remote_host_filter;
unsigned int s3_max_redirects;
unsigned int max_connections;
};
}

View File

@ -1,19 +0,0 @@
#include <Common/config.h>
#if USE_AWS_S3
#include "PocoHTTPResponseStream.h"
#include <utility>
namespace DB::S3
{
PocoHTTPResponseStream::PocoHTTPResponseStream(std::shared_ptr<Poco::Net::HTTPClientSession> session_, std::istream & response_stream_)
: Aws::IOStream(response_stream_.rdbuf()), session(std::move(session_))
{
}
}
#endif

View File

@ -1,21 +0,0 @@
#pragma once
#include <aws/core/utils/stream/ResponseStream.h>
#include <Poco/Net/HTTPClientSession.h>
namespace DB::S3
{
/**
* Wrapper of IStream to store response stream and corresponding HTTP session.
*/
class PocoHTTPResponseStream : public Aws::IOStream
{
public:
PocoHTTPResponseStream(std::shared_ptr<Poco::Net::HTTPClientSession> session_, std::istream & response_stream_);
private:
/// Poco HTTP session is holder of response stream.
std::shared_ptr<Poco::Net::HTTPClientSession> session;
};
}

View File

@ -0,0 +1,27 @@
#pragma once
#include <IO/HTTPCommon.h>
#include <aws/core/utils/stream/ResponseStream.h>
namespace DB::S3
{
/**
* Wrapper of IOStream to store response stream and corresponding HTTP session.
*/
template <typename Session>
class SessionAwareAwsStream : public Aws::IStream
{
public:
SessionAwareAwsStream(Session session_, std::istream & response_stream_)
: Aws::IStream(response_stream_.rdbuf()), session(std::move(session_))
{
}
private:
/// Poco HTTP session is holder of response stream.
Session session;
};
}

View File

@ -280,7 +280,7 @@ namespace S3
}
std::shared_ptr<Aws::S3::S3Client> ClientFactory::create( // NOLINT
Aws::Client::ClientConfiguration & cfg,
const Aws::Client::ClientConfiguration & cfg,
bool is_virtual_hosted_style,
const String & access_key_id,
const String & secret_access_key,
@ -306,7 +306,7 @@ namespace S3
}
std::shared_ptr<Aws::S3::S3Client> ClientFactory::create( // NOLINT
const String & endpoint,
const Aws::Client::ClientConfiguration & cfg,
bool is_virtual_hosted_style,
const String & access_key_id,
const String & secret_access_key,
@ -315,10 +315,7 @@ namespace S3
const RemoteHostFilter & remote_host_filter,
unsigned int s3_max_redirects)
{
PocoHTTPClientConfiguration client_configuration({}, remote_host_filter, s3_max_redirects);
if (!endpoint.empty())
client_configuration.endpointOverride = endpoint;
PocoHTTPClientConfiguration client_configuration(cfg, remote_host_filter, s3_max_redirects);
client_configuration.updateSchemeAndRegion();

View File

@ -41,7 +41,7 @@ public:
unsigned int s3_max_redirects);
std::shared_ptr<Aws::S3::S3Client> create(
Aws::Client::ClientConfiguration & cfg,
const Aws::Client::ClientConfiguration & cfg,
bool is_virtual_hosted_style,
const String & access_key_id,
const String & secret_access_key,
@ -50,7 +50,7 @@ public:
unsigned int s3_max_redirects);
std::shared_ptr<Aws::S3::S3Client> create(
const String & endpoint,
const Aws::Client::ClientConfiguration & cfg,
bool is_virtual_hosted_style,
const String & access_key_id,
const String & secret_access_key,

View File

@ -10,7 +10,6 @@
#include <Access/AccessFlags.h>
#include <Common/typeid_cast.h>
namespace DB
{
@ -40,6 +39,7 @@ BlockInputStreamPtr InterpreterExistsQuery::executeImpl()
{
ASTQueryWithTableAndOutput * exists_query;
bool result = false;
if ((exists_query = query_ptr->as<ASTExistsTableQuery>()))
{
if (exists_query->temporary)
@ -57,8 +57,8 @@ BlockInputStreamPtr InterpreterExistsQuery::executeImpl()
{
String database = context.resolveDatabase(exists_query->database);
context.checkAccess(AccessType::SHOW_TABLES, database, exists_query->table);
auto tbl = DatabaseCatalog::instance().tryGetTable({database, exists_query->table}, context);
result = tbl != nullptr && tbl->isView();
auto table = DatabaseCatalog::instance().tryGetTable({database, exists_query->table}, context);
result = table && table->isView();
}
else if ((exists_query = query_ptr->as<ASTExistsDatabaseQuery>()))
{

View File

@ -41,6 +41,16 @@ DropQuery::createDropTable(bool detach, bool if_exists, bool temporary, PtrTo<Ta
return query;
}
// static
PtrTo<DropQuery>
DropQuery::createDropView(bool detach, bool if_exists, PtrTo<TableIdentifier> identifier, PtrTo<ClusterClause> cluster)
{
auto query = PtrTo<DropQuery>(new DropQuery(cluster, QueryType::VIEW, {identifier}));
query->detach = detach;
query->if_exists = if_exists;
return query;
}
DropQuery::DropQuery(PtrTo<ClusterClause> cluster, QueryType type, PtrList exprs) : DDLQuery(cluster, exprs), query_type(type)
{
}
@ -73,6 +83,14 @@ ASTPtr DropQuery::convertToOld() const
query->database = database->getName();
break;
}
case QueryType::VIEW:
{
query->is_view = true;
query->table = get<TableIdentifier>(NAME)->getName();
if (auto database = get<TableIdentifier>(NAME)->getDatabase())
query->database = database->getName();
break;
}
}
convertToOldPartially(query);
@ -100,6 +118,8 @@ antlrcpp::Any ParseTreeVisitor::visitDropTableStmt(ClickHouseParser::DropTableSt
return DropQuery::createDropTable(!!ctx->DETACH(), !!ctx->EXISTS(), !!ctx->TEMPORARY(), visit(ctx->tableIdentifier()), cluster);
if (ctx->DICTIONARY())
return DropQuery::createDropDictionary(!!ctx->DETACH(), !!ctx->EXISTS(), visit(ctx->tableIdentifier()), cluster);
if (ctx->VIEW())
return DropQuery::createDropView(!!ctx->DETACH(), !!ctx->EXISTS(), visit(ctx->tableIdentifier()), cluster);
__builtin_unreachable();
}

View File

@ -15,6 +15,8 @@ class DropQuery : public DDLQuery
createDropTable(bool detach, bool if_exists, bool temporary, PtrTo<TableIdentifier> identifier, PtrTo<ClusterClause> cluster);
static PtrTo<DropQuery>
createDropDictionary(bool detach, bool if_exists, PtrTo<TableIdentifier> identifier, PtrTo<ClusterClause> cluster);
static PtrTo<DropQuery>
createDropView(bool detach, bool if_exists, PtrTo<TableIdentifier> identifier, PtrTo<ClusterClause> cluster);
ASTPtr convertToOld() const override;
@ -29,6 +31,7 @@ class DropQuery : public DDLQuery
DATABASE,
DICTIONARY,
TABLE,
VIEW,
};
const QueryType query_type;

View File

@ -10,31 +10,51 @@
namespace DB::AST
{
ExistsQuery::ExistsQuery(QueryType type, bool temporary_, PtrTo<TableIdentifier> identifier)
: Query{identifier}, query_type(type), temporary(temporary_)
ExistsQuery::ExistsQuery(QueryType type, bool temporary_, PtrList exprs)
: Query(exprs), query_type(type), temporary(temporary_)
{
}
// static
PtrTo<ExistsQuery> ExistsQuery::createTable(QueryType type, bool temporary, PtrTo<TableIdentifier> identifier)
{
return PtrTo<ExistsQuery>(new ExistsQuery(type, temporary, {identifier}));
}
// static
PtrTo<ExistsQuery> ExistsQuery::createDatabase(PtrTo<DatabaseIdentifier> identifier)
{
return PtrTo<ExistsQuery>(new ExistsQuery(QueryType::DATABASE, false, {identifier}));
}
ASTPtr ExistsQuery::convertToOld() const
{
std::shared_ptr<ASTQueryWithTableAndOutput> query;
switch(query_type)
{
case QueryType::DATABASE:
query = std::make_shared<ASTExistsDatabaseQuery>();
tryGetIdentifierNameInto(get<DatabaseIdentifier>(IDENTIFIER)->convertToOld(), query->database);
return query;
case QueryType::DICTIONARY:
query = std::make_shared<ASTExistsDictionaryQuery>();
break;
case QueryType::TABLE:
query = std::make_shared<ASTExistsTableQuery>();
query->temporary = temporary;
break;
case QueryType::VIEW:
query = std::make_shared<ASTExistsViewQuery>();
break;
}
// FIXME: this won't work if table doesn't exist
auto table_id = getTableIdentifier(get(TABLE)->convertToOld());
auto table_id = getTableIdentifier(get<TableIdentifier>(IDENTIFIER)->convertToOld());
query->database = table_id.database_name;
query->table = table_id.table_name;
query->uuid = table_id.uuid;
query->temporary = temporary;
return query;
}
@ -46,10 +66,22 @@ namespace DB
using namespace AST;
antlrcpp::Any ParseTreeVisitor::visitExistsStmt(ClickHouseParser::ExistsStmtContext *ctx)
antlrcpp::Any ParseTreeVisitor::visitExistsTableStmt(ClickHouseParser::ExistsTableStmtContext *ctx)
{
auto type = ctx->TABLE() ? ExistsQuery::QueryType::TABLE : ExistsQuery::QueryType::DICTIONARY;
return std::make_shared<ExistsQuery>(type, !!ctx->TEMPORARY(), visit(ctx->tableIdentifier()));
ExistsQuery::QueryType type;
if (ctx->DICTIONARY())
type = ExistsQuery::QueryType::DICTIONARY;
else if (ctx->VIEW())
type = ExistsQuery::QueryType::VIEW;
else // Query 'EXISTS <table_name>' is interptered as 'EXISTS TABLE <table_name>'
type = ExistsQuery::QueryType::TABLE;
return ExistsQuery::createTable(type, !!ctx->TEMPORARY(), visit(ctx->tableIdentifier()));
}
antlrcpp::Any ParseTreeVisitor::visitExistsDatabaseStmt(ClickHouseParser::ExistsDatabaseStmtContext *ctx)
{
return ExistsQuery::createDatabase(visit(ctx->databaseIdentifier()));
}
}

View File

@ -13,16 +13,21 @@ class ExistsQuery : public Query
{
DICTIONARY,
TABLE,
VIEW,
DATABASE,
};
ExistsQuery(QueryType type, bool temporary, PtrTo<TableIdentifier> identifier);
static PtrTo<ExistsQuery> createTable(QueryType type, bool temporary, PtrTo<TableIdentifier> identifier);
static PtrTo<ExistsQuery> createDatabase(PtrTo<DatabaseIdentifier> identifier);
ExistsQuery(QueryType type, bool temporary, PtrList exprs);
ASTPtr convertToOld() const override;
private:
enum ChildIndex : UInt8
{
TABLE = 0, // TableIdentifier
IDENTIFIER = 0, // DatabaseIdentifier or TableIdentifier
};
const QueryType query_type;

File diff suppressed because it is too large Load Diff

View File

@ -173,12 +173,15 @@ describeStmt: (DESCRIBE | DESC) TABLE? tableExpr;
dropStmt
: (DETACH | DROP) DATABASE (IF EXISTS)? databaseIdentifier clusterClause? # DropDatabaseStmt
| (DETACH | DROP) (DICTIONARY | TEMPORARY? TABLE) (IF EXISTS)? tableIdentifier clusterClause? (NO DELAY)? # DropTableStmt
| (DETACH | DROP) (DICTIONARY | TEMPORARY? TABLE | VIEW) (IF EXISTS)? tableIdentifier clusterClause? (NO DELAY)? # DropTableStmt
;
// EXISTS statement
existsStmt: EXISTS (DICTIONARY | TEMPORARY? TABLE)? tableIdentifier;
existsStmt
: EXISTS DATABASE databaseIdentifier # ExistsDatabaseStmt
| EXISTS (DICTIONARY | TEMPORARY? TABLE | VIEW)? tableIdentifier # ExistsTableStmt
;
// EXPLAIN statement

File diff suppressed because it is too large Load Diff

View File

@ -164,7 +164,9 @@ public:
virtual antlrcpp::Any visitDropTableStmt(ClickHouseParser::DropTableStmtContext *context) = 0;
virtual antlrcpp::Any visitExistsStmt(ClickHouseParser::ExistsStmtContext *context) = 0;
virtual antlrcpp::Any visitExistsDatabaseStmt(ClickHouseParser::ExistsDatabaseStmtContext *context) = 0;
virtual antlrcpp::Any visitExistsTableStmt(ClickHouseParser::ExistsTableStmtContext *context) = 0;
virtual antlrcpp::Any visitExplainStmt(ClickHouseParser::ExplainStmtContext *context) = 0;

View File

@ -136,7 +136,8 @@ public:
antlrcpp::Any visitTtlExpr(ClickHouseParser::TtlExprContext * ctx) override;
// ExistsQuery
antlrcpp::Any visitExistsStmt(ClickHouseParser::ExistsStmtContext * ctx) override;
antlrcpp::Any visitExistsTableStmt(ClickHouseParser::ExistsTableStmtContext * ctx) override;
antlrcpp::Any visitExistsDatabaseStmt(ClickHouseParser::ExistsDatabaseStmtContext * ctx) override;
// ExplainQuery
antlrcpp::Any visitExplainStmt(ClickHouseParser::ExplainStmtContext * ctx) override;

View File

@ -76,7 +76,6 @@ struct ASTDescribeQueryExistsQueryIDAndQueryNames
static constexpr auto QueryTemporary = "DESCRIBE TEMPORARY TABLE";
};
using ASTExistsDatabaseQuery = ASTQueryWithTableAndOutputImpl<ASTExistsDatabaseQueryIDAndQueryNames>;
using ASTExistsTableQuery = ASTQueryWithTableAndOutputImpl<ASTExistsTableQueryIDAndQueryNames>;
using ASTExistsViewQuery = ASTQueryWithTableAndOutputImpl<ASTExistsViewQueryIDAndQueryNames>;
using ASTExistsDictionaryQuery = ASTQueryWithTableAndOutputImpl<ASTExistsDictionaryQueryIDAndQueryNames>;
@ -84,6 +83,16 @@ using ASTShowCreateTableQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateTabl
using ASTShowCreateViewQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateViewQueryIDAndQueryNames>;
using ASTShowCreateDictionaryQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateDictionaryQueryIDAndQueryNames>;
class ASTExistsDatabaseQuery : public ASTQueryWithTableAndOutputImpl<ASTExistsDatabaseQueryIDAndQueryNames>
{
protected:
void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << ASTExistsDatabaseQueryIDAndQueryNames::Query
<< " " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database);
}
};
class ASTShowCreateDatabaseQuery : public ASTQueryWithTableAndOutputImpl<ASTShowCreateDatabaseQueryIDAndQueryNames>
{
protected:

View File

@ -196,6 +196,7 @@ StorageS3::StorageS3(
const String & format_name_,
UInt64 min_upload_part_size_,
UInt64 max_single_part_upload_size_,
UInt64 max_connections_,
const ColumnsDescription & columns_,
const ConstraintsDescription & constraints_,
const Context & context_,
@ -220,8 +221,12 @@ StorageS3::StorageS3(
if (access_key_id_.empty())
credentials = Aws::Auth::AWSCredentials(std::move(settings.access_key_id), std::move(settings.secret_access_key));
Aws::Client::ClientConfiguration client_configuration;
client_configuration.endpointOverride = uri_.endpoint;
client_configuration.maxConnections = max_connections_;
client = S3::ClientFactory::instance().create(
uri_.endpoint,
client_configuration,
uri_.is_virtual_hosted_style,
credentials.GetAWSAccessKeyId(),
credentials.GetAWSSecretKey(),
@ -374,6 +379,7 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory)
UInt64 min_upload_part_size = args.local_context.getSettingsRef().s3_min_upload_part_size;
UInt64 max_single_part_upload_size = args.local_context.getSettingsRef().s3_max_single_part_upload_size;
UInt64 max_connections = args.local_context.getSettingsRef().s3_max_connections;
String compression_method;
String format_name;
@ -396,6 +402,7 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory)
format_name,
min_upload_part_size,
max_single_part_upload_size,
max_connections,
args.columns,
args.constraints,
args.context,

View File

@ -32,6 +32,7 @@ public:
const String & format_name_,
UInt64 min_upload_part_size_,
UInt64 max_single_part_upload_size_,
UInt64 max_connections_,
const ColumnsDescription & columns_,
const ConstraintsDescription & constraints_,
const Context & context_,

View File

@ -68,6 +68,7 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, const C
S3::URI s3_uri (uri);
UInt64 min_upload_part_size = context.getSettingsRef().s3_min_upload_part_size;
UInt64 max_single_part_upload_size = context.getSettingsRef().s3_max_single_part_upload_size;
UInt64 max_connections = context.getSettingsRef().s3_max_connections;
StoragePtr storage = StorageS3::create(
s3_uri,
@ -77,6 +78,7 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, const C
format,
min_upload_part_size,
max_single_part_upload_size,
max_connections,
getActualTableStructure(context),
ConstraintsDescription{},
const_cast<Context &>(context),

View File

@ -293,6 +293,8 @@ def run_tests_array(all_tests_with_params):
try:
sys.stdout.flush()
sys.stdout.write("{0:72}".format(name + ": "))
# This flush is needed so you can see the test name of the long running test before it will finish.
sys.stdout.flush()
if args.skip and any(s in name for s in args.skip):
print(MSG_SKIPPED + " - skip")

View File

@ -28,8 +28,8 @@
number,
number
FROM
system.numbers
LIMIT 100000000
system.numbers_mt
LIMIT 200000000
</fill_query>
<query>
SELECT
@ -41,7 +41,7 @@
SETTINGS
compile_expressions = 0;
</query>
<query short="1">
<query>
SELECT
COUNT()
FROM

View File

@ -28,7 +28,7 @@ EXISTS t_01048; -- Does not work for temporary tables. Maybe have to fix.
EXISTS TABLE t_01048;
EXISTS DICTIONARY t_01048;
CREATE DICTIONARY db_01048.t_01048 (k UInt64, v String) PRIMARY KEY k LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://example.test/' FORMAT TSV)) LIFETIME(1000);
CREATE DICTIONARY db_01048.t_01048 (k UInt64, v String) PRIMARY KEY k LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://example.test/' FORMAT 'TSV')) LIFETIME(1000);
EXISTS db_01048.t_01048;
EXISTS TABLE db_01048.t_01048; -- Dictionaries are tables as well. But not all tables are dictionaries.
EXISTS DICTIONARY db_01048.t_01048;

View File

@ -0,0 +1,2 @@
202101
202101_0_0_0 1 2021-01-01 some

View File

@ -0,0 +1,32 @@
DROP TABLE IF EXISTS test_01640;
DROP TABLE IF EXISTS restore_01640;
CREATE TABLE test_01640(i Int64, d Date, s String)
ENGINE = ReplicatedMergeTree('/clickhouse/{shard}/tables/test_01640','{replica}')
PARTITION BY toYYYYMM(d) ORDER BY i;
insert into test_01640 values (1, '2021-01-01','some');
CREATE TABLE restore_01640(i Int64, d Date, s String)
ENGINE = ReplicatedMergeTree('/clickhouse/{shard}/tables/restore_01640','{replica}')
PARTITION BY toYYYYMM(d) ORDER BY i;
ALTER TABLE restore_01640 FETCH PARTITION tuple(toYYYYMM(toDate('2021-01-01')))
FROM '/clickhouse/{shard}/tables/test_01640';
SELECT partition_id
FROM system.detached_parts
WHERE (table = 'restore_01640') AND (database = currentDatabase());
ALTER TABLE restore_01640 ATTACH PARTITION tuple(toYYYYMM(toDate('2021-01-01')));
SELECT partition_id
FROM system.detached_parts
WHERE (table = 'restore_01640') AND (database = currentDatabase());
SELECT _part, * FROM restore_01640;
DROP TABLE test_01640;
DROP TABLE restore_01640;

View File

@ -0,0 +1,4 @@
SELECT formatDateTime(toDate('2010-01-04'), '%Q');
SELECT formatDateTime(toDate('2010-04-30'), '%Q');
SELECT formatDateTime(toDate('2010-07-31'), '%Q');
SELECT formatDateTime(toDate('2010-10-07'), '%Q');

View File

@ -185,4 +185,5 @@
01639_distributed_sync_insert_zero_rows
01644_distributed_async_insert_fsync_smoke
01552_impl_aggfunc_cloneresize
01650_fetch_patition_with_macro_in_zk_path
01651_bugs_from_15889

View File

@ -162,7 +162,6 @@
"01039_test_setting_parse",
"01042_system_reload_dictionary_reloads_completely",
"01045_dictionaries_restrictions",
"01048_exists_query",
"01055_compact_parts_1",
"01056_create_table_as",
"01066_bit_count",
@ -197,7 +196,6 @@
"01190_full_attach_syntax",
"01191_rename_dictionary",
"01192_rename_database_zookeeper",
"01210_drop_view",
"01213_alter_rename_column",
"01232_untuple",
"01244_optimize_distributed_group_by_sharding_key",
@ -209,7 +207,6 @@
"01269_create_with_null",
"01271_show_privileges",
"01272_offset_without_limit",
"01275_parallel_mv",
"01277_alter_rename_column_constraint_zookeeper",
"01278_min_insert_block_size_rows_for_materialized_views",
"01280_min_map_max_map",

View File

@ -1,3 +1,4 @@
v21.1.2.15-stable 2021-01-18
v20.12.5.14-stable 2020-12-28
v20.12.4.5-stable 2020-12-24
v20.12.3.3-stable 2020-12-09

1 v20.12.5.14-stable v21.1.2.15-stable 2020-12-28 2021-01-18
1 v21.1.2.15-stable 2021-01-18
2 v20.12.5.14-stable v20.12.5.14-stable 2020-12-28 2020-12-28
3 v20.12.4.5-stable v20.12.4.5-stable 2020-12-24 2020-12-24
4 v20.12.3.3-stable v20.12.3.3-stable 2020-12-09 2020-12-09

View File

@ -78,7 +78,7 @@ def parse_one_pull_request(item):
# This array gives the preferred category order, and is also used to
# normalize category names.
categories_preferred_order = ['Backward Incompatible Change',
'New Feature', 'Bug Fix', 'Improvement', 'Performance Improvement',
'New Feature', 'Performance Improvement', 'Improvement', 'Bug Fix',
'Build/Testing/Packaging Improvement', 'Other']
category_to_pr = collections.defaultdict(lambda: [])