mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 21:20:49 +00:00
Merge branch 'master' into in_memory_raft
This commit is contained in:
alesapin
commit
6883143994
3
.github/ISSUE_TEMPLATE/40_bug-report.md
vendored
3
.github/ISSUE_TEMPLATE/40_bug-report.md
vendored
@ -12,6 +12,9 @@ assignees: ''
|
||||
**Describe the bug**
|
||||
A clear and concise description of what works not as it is supposed to.
|
||||
|
||||
**Does it reproduce on recent release?**
|
||||
[The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv)
|
||||
|
||||
**How to reproduce**
|
||||
* Which ClickHouse server version to use
|
||||
* Which interface to use, if matters
|
||||
|
||||
3613
CHANGELOG.md
3613
CHANGELOG.md
File diff suppressed because it is too large
Load Diff
@ -17,3 +17,4 @@ ClickHouse® is an open-source column-oriented database management system that a
|
||||
|
||||
## Upcoming Events
|
||||
* [SF Bay Area ClickHouse Virtual Office Hours (online)](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/274273549/) on 20 January 2020.
|
||||
* [Chinese ClickHouse Meetup (online)](http://hdxu.cn/8KxZE) on 6 February 2020.
|
||||
|
||||
@ -4,6 +4,12 @@
|
||||
#include "syscall.h"
|
||||
#include "atomic.h"
|
||||
|
||||
#if defined(__has_feature)
|
||||
#if __has_feature(memory_sanitizer)
|
||||
#include <sanitizer/msan_interface.h>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef VDSO_GETCPU_SYM
|
||||
|
||||
static void *volatile vdso_func;
|
||||
@ -37,6 +43,13 @@ int sched_getcpu(void)
|
||||
#endif
|
||||
|
||||
r = __syscall(SYS_getcpu, &cpu, 0, 0);
|
||||
if (!r) return cpu;
|
||||
if (!r) {
|
||||
#if defined(__has_feature)
|
||||
#if __has_feature(memory_sanitizer)
|
||||
__msan_unpoison(&cpu, sizeof(cpu));
|
||||
#endif
|
||||
#endif
|
||||
return cpu;
|
||||
}
|
||||
return __syscall_ret(r);
|
||||
}
|
||||
|
||||
11
contrib/CMakeLists.txt
vendored
11
contrib/CMakeLists.txt
vendored
@ -119,12 +119,6 @@ if (USE_INTERNAL_LDAP_LIBRARY)
|
||||
add_subdirectory (openldap-cmake)
|
||||
endif ()
|
||||
|
||||
# Should go before:
|
||||
# - mariadb-connector-c
|
||||
# - aws-s3-cmake
|
||||
# - sentry-native
|
||||
add_subdirectory (curl-cmake)
|
||||
|
||||
function(mysql_support)
|
||||
set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC)
|
||||
set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC)
|
||||
@ -142,6 +136,7 @@ function(mysql_support)
|
||||
set(ZLIB_LIBRARY ${ZLIB_LIBRARIES})
|
||||
set(WITH_EXTERNAL_ZLIB ON)
|
||||
endif()
|
||||
set(WITH_CURL OFF)
|
||||
add_subdirectory (mariadb-connector-c)
|
||||
endfunction()
|
||||
if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
|
||||
@ -288,6 +283,10 @@ if (USE_CASSANDRA)
|
||||
add_subdirectory (cassandra)
|
||||
endif()
|
||||
|
||||
# Should go before:
|
||||
# - sentry-native
|
||||
add_subdirectory (curl-cmake)
|
||||
|
||||
if (USE_SENTRY)
|
||||
add_subdirectory (sentry-native)
|
||||
endif()
|
||||
|
||||
2
contrib/dragonbox
vendored
2
contrib/dragonbox
vendored
@ -1 +1 @@
|
||||
Subproject commit b2751c65c0592c0239aec3becd53d0ea2fde9329
|
||||
Subproject commit 923705af6fd953aa948fc175f6020b15f7359838
|
||||
2
contrib/krb5
vendored
2
contrib/krb5
vendored
@ -1 +1 @@
|
||||
Subproject commit 90ff6f4f8c695d6bf1aaba78a9b8942be92141c2
|
||||
Subproject commit 5149dea4e2be0f67707383d2682b897c14631374
|
||||
@ -1,4 +1,6 @@
|
||||
#!/bin/bash
|
||||
# shellcheck disable=SC2086
|
||||
|
||||
set -eux
|
||||
set -o pipefail
|
||||
trap "exit" INT TERM
|
||||
@ -20,6 +22,7 @@ function clone
|
||||
git init
|
||||
git remote add origin https://github.com/ClickHouse/ClickHouse
|
||||
git fetch --depth=1 origin "$SHA_TO_TEST"
|
||||
git fetch --depth=1 origin master # Used to obtain the list of modified or added tests
|
||||
|
||||
# If not master, try to fetch pull/.../{head,merge}
|
||||
if [ "$PR_TO_TEST" != "0" ]
|
||||
@ -33,9 +36,6 @@ function clone
|
||||
|
||||
function download
|
||||
{
|
||||
# wget -O- -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/performance/performance.tgz" \
|
||||
# | tar --strip-components=1 -zxv
|
||||
|
||||
wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"
|
||||
chmod +x clickhouse
|
||||
ln -s ./clickhouse ./clickhouse-server
|
||||
@ -73,6 +73,17 @@ function watchdog
|
||||
|
||||
function fuzz
|
||||
{
|
||||
# Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests.
|
||||
cd ch
|
||||
NEW_TESTS=$(git diff --name-only master | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R)
|
||||
cd ..
|
||||
if [[ -n "$NEW_TESTS" ]]
|
||||
then
|
||||
NEW_TESTS_OPT="--interleave-queries-file ${NEW_TESTS}"
|
||||
else
|
||||
NEW_TESTS_OPT=""
|
||||
fi
|
||||
|
||||
./clickhouse-server --config-file db/config.xml -- --path db 2>&1 | tail -100000 > server.log &
|
||||
server_pid=$!
|
||||
kill -0 $server_pid
|
||||
@ -85,7 +96,7 @@ function fuzz
|
||||
# SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric.
|
||||
# SC2046: Quote this to prevent word splitting. Actually I need word splitting.
|
||||
# shellcheck disable=SC2012,SC2046
|
||||
./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \
|
||||
./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) $NEW_TESTS_OPT \
|
||||
> >(tail -n 100000 > fuzzer.log) \
|
||||
2>&1 \
|
||||
|| fuzzer_exit_code=$?
|
||||
@ -107,7 +118,7 @@ function fuzz
|
||||
|
||||
case "$stage" in
|
||||
"")
|
||||
;&
|
||||
;& # Did you know? This is "fallthrough" in bash. https://stackoverflow.com/questions/12010686/case-statement-fallthrough
|
||||
"clone")
|
||||
time clone
|
||||
if [ -v FUZZ_LOCAL_SCRIPT ]
|
||||
|
||||
@ -25,6 +25,7 @@ Ways to configure settings, in order of priority:
|
||||
|
||||
- When starting the ClickHouse console client in non-interactive mode, set the startup parameter `--setting=value`.
|
||||
- When using the HTTP API, pass CGI parameters (`URL?setting_1=value&setting_2=value...`).
|
||||
- Make settings in the [SETTINGS](../../sql-reference/statements/select/index.md#settings-in-select) clause of the SELECT query. The setting value is applied only to that query and is reset to default or previous value after the query is executed.
|
||||
|
||||
Settings that can only be made in the server config file are not covered in this section.
|
||||
|
||||
|
||||
190
docs/en/operations/settings/merge-tree-settings.md
Normal file
190
docs/en/operations/settings/merge-tree-settings.md
Normal file
@ -0,0 +1,190 @@
|
||||
# MergeTree tables settings {#merge-tree-settings}
|
||||
|
||||
The values of `merge_tree` settings (for all MergeTree tables) can be viewed in the table `system.merge_tree_settings`, they can be overridden in `config.xml` in the `merge_tree` section, or set in the `SETTINGS` section of each table.
|
||||
|
||||
Override example in `config.xml`:
|
||||
|
||||
``` text
|
||||
<merge_tree>
|
||||
<max_suspicious_broken_parts>5</max_suspicious_broken_parts>
|
||||
</merge_tree>
|
||||
```
|
||||
|
||||
An example to set in `SETTINGS` for a particular table:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE foo
|
||||
(
|
||||
`A` Int64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY tuple()
|
||||
SETTINGS max_suspicious_broken_parts = 500;
|
||||
```
|
||||
|
||||
An example of changing the settings for a specific table with the `ALTER TABLE ... MODIFY SETTING` command:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE foo
|
||||
MODIFY SETTING max_suspicious_broken_parts = 100;
|
||||
```
|
||||
|
||||
## parts_to_throw_insert {#parts-to-throw-insert}
|
||||
|
||||
If the number of active parts in a single partition exceeds the `parts_to_throw_insert` value, `INSERT` is interrupted with the `Too many parts (N). Merges are processing significantly slower than inserts` exception.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 300.
|
||||
|
||||
To achieve maximum performance of `SELECT` queries, it is necessary to minimize the number of parts processed, see [Merge Tree](../../development/architecture.md#merge-tree).
|
||||
|
||||
You can set a larger value to 600 (1200), this will reduce the probability of the `Too many parts` error, but at the same time `SELECT` performance might degrade. Also in case of a merge issue (for example, due to insufficient disk space) you will notice it later than it could be with the original 300.
|
||||
|
||||
|
||||
## parts_to_delay_insert {#parts-to-delay-insert}
|
||||
|
||||
If the number of active parts in a single partition exceeds the `parts_to_delay_insert` value, an `INSERT` artificially slows down.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 150.
|
||||
|
||||
ClickHouse artificially executes `INSERT` longer (adds ‘sleep’) so that the background merge process can merge parts faster than they are added.
|
||||
|
||||
## max_delay_to_insert {#max-delay-to-insert}
|
||||
|
||||
The value in seconds, which is used to calculate the `INSERT` delay, if the number of active parts in a single partition exceeds the [parts_to_delay_insert](#parts-to-delay-insert) value.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 1.
|
||||
|
||||
The delay (in milliseconds) for `INSERT` is calculated by the formula:
|
||||
|
||||
```code
|
||||
max_k = parts_to_throw_insert - parts_to_delay_insert
|
||||
k = 1 + parts_count_in_partition - parts_to_delay_insert
|
||||
delay_milliseconds = pow(max_delay_to_insert * 1000, k / max_k)
|
||||
```
|
||||
|
||||
For example if a partition has 299 active parts and parts_to_throw_insert = 300, parts_to_delay_insert = 150, max_delay_to_insert = 1, `INSERT` is delayed for `pow( 1 * 1000, (1 + 299 - 150) / (300 - 150) ) = 1000` milliseconds.
|
||||
|
||||
## max_parts_in_total {#max-parts-in-total}
|
||||
|
||||
If the total number of active parts in all partitions of a table exceeds the `max_parts_in_total` value `INSERT` is interrupted with the `Too many parts (N)` exception.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 100000.
|
||||
|
||||
A large number of parts in a table reduces performance of ClickHouse queries and increases ClickHouse boot time. Most often this is a consequence of an incorrect design (mistakes when choosing a partitioning strategy - too small partitions).
|
||||
|
||||
## replicated_deduplication_window {#replicated-deduplication-window}
|
||||
|
||||
The number of most recently inserted blocks for which Zookeeper stores hash sums to check for duplicates.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
- 0 (disable deduplication)
|
||||
|
||||
Default value: 100.
|
||||
|
||||
The `Insert` command creates one or more blocks (parts). When inserting into Replicated tables, ClickHouse for [insert deduplication](../../engines/table-engines/mergetree-family/replication/) writes the hash sums of the created parts into Zookeeper. Hash sums are stored only for the most recent `replicated_deduplication_window` blocks. The oldest hash sums are removed from Zookeeper.
|
||||
A large number of `replicated_deduplication_window` slows down `Inserts` because it needs to compare more entries.
|
||||
The hash sum is calculated from the composition of the field names and types and the data of the inserted part (stream of bytes).
|
||||
|
||||
## replicated_deduplication_window_seconds {#replicated-deduplication-window-seconds}
|
||||
|
||||
The number of seconds after which the hash sums of the inserted blocks are removed from Zookeeper.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 604800 (1 week).
|
||||
|
||||
Similar to [replicated_deduplication_window](#replicated-deduplication-window), `replicated_deduplication_window_seconds` specifies how long to store hash sums of blocks for insert deduplication. Hash sums older than `replicated_deduplication_window_seconds` are removed from Zookeeper, even if they are less than ` replicated_deduplication_window`.
|
||||
|
||||
## old_parts_lifetime {#old-parts-lifetime}
|
||||
|
||||
The time (in seconds) of storing inactive parts to protect against data loss during spontaneous server reboots.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 480.
|
||||
|
||||
`fsync` is not called for new parts, so for some time new parts exist only in the server's RAM (OS cache). If the server is rebooted spontaneously, new parts can be lost or damaged.
|
||||
To protect data parts created by merges source parts are not deleted immediately. After merging several parts into a new part, ClickHouse marks the original parts as inactive and deletes them only after `old_parts_lifetime` seconds.
|
||||
Inactive parts are removed if they are not used by current queries, i.e. if the `refcount` of the part is zero.
|
||||
|
||||
During startup ClickHouse checks the integrity of the parts.
|
||||
If the merged part is damaged ClickHouse returns the inactive parts to the active list, and later merges them again. Then the damaged part is renamed (the `broken_` prefix is added) and moved to the `detached` folder.
|
||||
If the merged part is not damaged, then the original inactive parts are renamed (the `ignored_` prefix is added) and moved to the `detached` folder.
|
||||
|
||||
The default `dirty_expire_centisecs` value (a Linux kernel setting) is 30 seconds (the maximum time that written data is stored only in RAM), but under heavy loads on the disk system data can be written much later. Experimentally, a value of 480 seconds was chosen for `old_parts_lifetime`, during which a new part is guaranteed to be written to disk.
|
||||
|
||||
## max_bytes_to_merge_at_max_space_in_pool {#max-bytes-to-merge-at-max-space-in-pool}
|
||||
|
||||
The maximum total parts size (in bytes) to be merged into one part, if there are enough resources available.
|
||||
`max_bytes_to_merge_at_max_space_in_pool` -- roughly corresponds to the maximum possible part size created by an automatic background merge.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 161061273600 (150 GB).
|
||||
|
||||
The merge scheduler periodically analyzes the sizes and number of parts in partitions, and if there is enough free resources in the pool, it starts background merges. Merges occur until the total size of the source parts is less than `max_bytes_to_merge_at_max_space_in_pool`.
|
||||
|
||||
Merges initiated by `optimize final` ignore `max_bytes_to_merge_at_max_space_in_pool` and merge parts only taking into account available resources (free disk's space) until one part remains in the partition.
|
||||
|
||||
## max_bytes_to_merge_at_min_space_in_pool {#max-bytes-to-merge-at-min-space-in-pool}
|
||||
|
||||
The maximum total part size (in bytes) to be merged into one part, with the minimum available resources in the background pool.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 1048576 (1 MB)
|
||||
|
||||
`max_bytes_to_merge_at_min_space_in_pool` defines the maximum total size of parts which can be merged despite the lack of available disk space (in pool). This is necessary to reduce the number of small parts and the chance of `Too many parts` errors.
|
||||
Merges book disk space by doubling the total merged parts sizes. Thus, with a small amount of free disk space, a situation may happen that there is free space, but this space is already booked by ongoing large merges, so other merges unable to start, and the number of small parts grows with every insert.
|
||||
|
||||
## merge_max_block_size {#merge-max-block-size}
|
||||
|
||||
The number of rows that are read from the merged parts into memory.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 8192
|
||||
|
||||
Merge reads rows from parts in blocks of `merge_max_block_size` rows, then merges and writes the result into a new part. The read block is placed in RAM, so `merge_max_block_size` affects the size of the RAM required for the merge. Thus, merges can consume a large amount of RAM for tables with very wide rows (if the average row size is 100kb, then when merging 10 parts, (100kb * 10 * 8192) = ~ 8GB of RAM). By decreasing `merge_max_block_size`, you can reduce the amount of RAM required for a merge but slow down a merge.
|
||||
|
||||
## max_part_loading_threads {#max-part-loading-threads}
|
||||
|
||||
The maximum number of threads that read parts when ClickHouse starts.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: auto (number of CPU cores).
|
||||
|
||||
During startup ClickHouse reads all parts of all tables (reads files with metadata of parts) to build a list of all parts in memory. In some systems with a large number of parts this process can take a long time, and this time might be shortened by increasing `max_part_loading_threads` (if this process is not CPU and disk I/O bound).
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/settings/merge_tree_settings/) <!--hide-->
|
||||
@ -9,11 +9,18 @@ toc_title: Float32, Float64
|
||||
|
||||
Types are equivalent to types of C:
|
||||
|
||||
- `Float32` - `float`
|
||||
- `Float64` - `double`
|
||||
- `Float32` — `float`.
|
||||
- `Float64` — `double`.
|
||||
|
||||
We recommend that you store data in integer form whenever possible. For example, convert fixed precision numbers to integer values, such as monetary amounts or page load times in milliseconds.
|
||||
|
||||
Aliases:
|
||||
|
||||
- `Float32` — `FLOAT`.
|
||||
- `Float64` — `DOUBLE`.
|
||||
|
||||
When creating tables, numeric parameters for floating point numbers can be set (e.g. `FLOAT(12)`, `FLOAT(15, 22)`, `DOUBLE(12)`, `DOUBLE(4, 18)`), but ClickHouse ignores them.
|
||||
|
||||
## Using Floating-point Numbers {#using-floating-point-numbers}
|
||||
|
||||
- Computations with floating-point numbers might produce a rounding error.
|
||||
@ -52,7 +59,7 @@ SELECT 0.5 / 0
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
- `-Inf` – Negative infinity.
|
||||
- `-Inf` — Negative infinity.
|
||||
|
||||
<!-- -->
|
||||
|
||||
@ -66,7 +73,7 @@ SELECT -0.5 / 0
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
- `NaN` – Not a number.
|
||||
- `NaN` — Not a number.
|
||||
|
||||
<!-- -->
|
||||
|
||||
@ -80,6 +87,6 @@ SELECT 0 / 0
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
See the rules for `NaN` sorting in the section [ORDER BY clause](../sql_reference/statements/select/order-by.md).
|
||||
See the rules for `NaN` sorting in the section [ORDER BY clause](../../sql-reference/statements/select/order-by.md).
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/data_types/float/) <!--hide-->
|
||||
|
||||
@ -7,23 +7,32 @@ toc_title: UInt8, UInt16, UInt32, UInt64, UInt256, Int8, Int16, Int32, Int64, In
|
||||
|
||||
Fixed-length integers, with or without a sign.
|
||||
|
||||
When creating tables, numeric parameters for integer numbers can be set (e.g. `TINYINT(8)`, `SMALLINT(16)`, `INT(32)`, `BIGINT(64)`), but ClickHouse ignores them.
|
||||
|
||||
## Int Ranges {#int-ranges}
|
||||
|
||||
- Int8 - \[-128 : 127\]
|
||||
- Int16 - \[-32768 : 32767\]
|
||||
- Int32 - \[-2147483648 : 2147483647\]
|
||||
- Int64 - \[-9223372036854775808 : 9223372036854775807\]
|
||||
- Int128 - \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
|
||||
- Int256 - \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
|
||||
- `Int8` — \[-128 : 127\]
|
||||
- `Int16` — \[-32768 : 32767\]
|
||||
- `Int32` — \[-2147483648 : 2147483647\]
|
||||
- `Int64` — \[-9223372036854775808 : 9223372036854775807\]
|
||||
- `Int128` — \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
|
||||
- `Int256` — \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
|
||||
|
||||
Aliases:
|
||||
|
||||
- `Int8` — `TINYINT`, `BOOL`, `BOOLEAN`, `INT1`.
|
||||
- `Int16` — `SMALLINT`, `INT2`.
|
||||
- `Int32` — `INT`, `INT4`, `INTEGER`.
|
||||
- `Int64` — `BIGINT`.
|
||||
|
||||
## Uint Ranges {#uint-ranges}
|
||||
|
||||
- UInt8 - \[0 : 255\]
|
||||
- UInt16 - \[0 : 65535\]
|
||||
- UInt32 - \[0 : 4294967295\]
|
||||
- UInt64 - \[0 : 18446744073709551615\]
|
||||
- UInt256 - \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
|
||||
- `UInt8` — \[0 : 255\]
|
||||
- `UInt16` — \[0 : 65535\]
|
||||
- `UInt32` — \[0 : 4294967295\]
|
||||
- `UInt64` — \[0 : 18446744073709551615\]
|
||||
- `UInt256` — \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
|
||||
|
||||
UInt128 is not supported yet.
|
||||
`UInt128` is not supported yet.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/data_types/int_uint/) <!--hide-->
|
||||
|
||||
@ -8,6 +8,8 @@ toc_title: String
|
||||
Strings of an arbitrary length. The length is not limited. The value can contain an arbitrary set of bytes, including null bytes.
|
||||
The String type replaces the types VARCHAR, BLOB, CLOB, and others from other DBMSs.
|
||||
|
||||
When creating tables, numeric parameters for string fields can be set (e.g. `VARCHAR(255)`), but ClickHouse ignores them.
|
||||
|
||||
## Encodings {#encodings}
|
||||
|
||||
ClickHouse doesn’t have the concept of encodings. Strings can contain an arbitrary set of bytes, which are stored and output as-is.
|
||||
|
||||
@ -93,7 +93,7 @@ Setting fields:
|
||||
- `path` – The absolute path to the file.
|
||||
- `format` – The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported.
|
||||
|
||||
When dictionary with FILE source is created via DDL command (`CREATE DICTIONARY ...`), source of the dictionary have to be located in `user_files` directory, to prevent DB users accessing arbitrary file on clickhouse node.
|
||||
When dictionary with source `FILE` is created via DDL command (`CREATE DICTIONARY ...`), the source file needs to be located in `user_files` directory, to prevent DB users accessing arbitrary file on ClickHouse node.
|
||||
|
||||
## Executable File {#dicts-external_dicts_dict_sources-executable}
|
||||
|
||||
@ -115,7 +115,7 @@ Setting fields:
|
||||
- `command` – The absolute path to the executable file, or the file name (if the program directory is written to `PATH`).
|
||||
- `format` – The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported.
|
||||
|
||||
That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on clickhouse node.
|
||||
That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on ClickHouse node.
|
||||
|
||||
## Http(s) {#dicts-external_dicts_dict_sources-http}
|
||||
|
||||
@ -160,14 +160,14 @@ Setting fields:
|
||||
- `url` – The source URL.
|
||||
- `format` – The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported.
|
||||
- `credentials` – Basic HTTP authentication. Optional parameter.
|
||||
- `user` – Username required for the authentication.
|
||||
- `password` – Password required for the authentication.
|
||||
- `user` – Username required for the authentication.
|
||||
- `password` – Password required for the authentication.
|
||||
- `headers` – All custom HTTP headers entries used for the HTTP request. Optional parameter.
|
||||
- `header` – Single HTTP header entry.
|
||||
- `name` – Identifiant name used for the header send on the request.
|
||||
- `value` – Value set for a specific identifiant name.
|
||||
- `header` – Single HTTP header entry.
|
||||
- `name` – Identifiant name used for the header send on the request.
|
||||
- `value` – Value set for a specific identifiant name.
|
||||
|
||||
When creating a dictionary using the DDL command (`CREATE DICTIONARY ...`) remote hosts for HTTP dictionaries checked with the `remote_url_allow_hosts` section from config to prevent database users to access arbitrary HTTP server.
|
||||
When creating a dictionary using the DDL command (`CREATE DICTIONARY ...`) remote hosts for HTTP dictionaries are checked against the contents of `remote_url_allow_hosts` section from config to prevent database users to access arbitrary HTTP server.
|
||||
|
||||
## ODBC {#dicts-external_dicts_dict_sources-odbc}
|
||||
|
||||
|
||||
@ -633,6 +633,7 @@ Using replacement fields, you can define a pattern for the resulting string. “
|
||||
| %M | minute (00-59) | 33 |
|
||||
| %n | new-line character (‘’) | |
|
||||
| %p | AM or PM designation | PM |
|
||||
| %Q | Quarter (1-4) | 1 |
|
||||
| %R | 24-hour HH:MM time, equivalent to %H:%M | 22:33 |
|
||||
| %S | second (00-59) | 44 |
|
||||
| %t | horizontal-tab character (’) | |
|
||||
|
||||
@ -25,6 +25,7 @@ SELECT [DISTINCT] expr_list
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[SETTINGS ...]
|
||||
[UNION ...]
|
||||
[INTO OUTFILE filename]
|
||||
[FORMAT format]
|
||||
@ -265,5 +266,17 @@ SELECT * REPLACE(i + 1 AS i) EXCEPT (j) APPLY(sum) from columns_transformers;
|
||||
└─────────────────┴────────┘
|
||||
```
|
||||
|
||||
## SETTINGS in SELECT Query {#settings-in-select}
|
||||
|
||||
You can specify the necessary settings right in the `SELECT` query. The setting value is applied only to this query and is reset to default or previous value after the query is executed.
|
||||
|
||||
Other ways to make settings see [here](../../../operations/settings/index.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT * FROM some_table SETTINGS optimize_read_in_order=1, cast_keep_nullable=1;
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/)
|
||||
<!--hide-->
|
||||
|
||||
3376
docs/en/whats-new/changelog/2020.md
Normal file
3376
docs/en/whats-new/changelog/2020.md
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,7 +1,7 @@
|
||||
---
|
||||
toc_folder_title: Changelog
|
||||
toc_priority: 74
|
||||
toc_title: '2020'
|
||||
toc_title: '2021'
|
||||
---
|
||||
|
||||
{% include "content/changelog.md" %}
|
||||
|
||||
@ -24,6 +24,7 @@ toc_title: Introduction
|
||||
|
||||
- При запуске консольного клиента ClickHouse в не интерактивном режиме установите параметр запуска `--setting=value`.
|
||||
- При использовании HTTP API передавайте cgi-параметры (`URL?setting_1=value&setting_2=value...`).
|
||||
- Укажите необходимые настройки в секции [SETTINGS](../../sql-reference/statements/select/index.md#settings-in-select) запроса SELECT. Эти настройки действуют только в рамках данного запроса, а после его выполнения сбрасываются до предыдущего значения или значения по умолчанию.
|
||||
|
||||
Настройки, которые можно задать только в конфигурационном файле сервера, в разделе не рассматриваются.
|
||||
|
||||
|
||||
@ -9,8 +9,15 @@ toc_title: Float32, Float64
|
||||
|
||||
Типы эквивалентны типам языка С:
|
||||
|
||||
- `Float32` - `float`;
|
||||
- `Float64` - `double`.
|
||||
- `Float32` — `float`.
|
||||
- `Float64` — `double`.
|
||||
|
||||
Синонимы:
|
||||
|
||||
- `Float32` — `FLOAT`.
|
||||
- `Float64` — `DOUBLE`.
|
||||
|
||||
При создании таблиц для чисел с плавающей запятой можно указывать числовые параметры (например, `FLOAT(12)`, `FLOAT(15, 22)`, `DOUBLE(12)`, `DOUBLE(4, 18)`), но ClickHouse их проигнорирует.
|
||||
|
||||
Рекомендуется хранить данные в целочисленном виде всегда, когда это возможно. Например, переводите в целочисленные значения числа с фиксированной точностью, такие как денежные суммы или времена загрузки страниц в миллисекундах.
|
||||
|
||||
@ -38,7 +45,7 @@ SELECT 1 - 0.9
|
||||
|
||||
В отличие от стандартного SQL, ClickHouse поддерживает следующие категории чисел с плавающей запятой:
|
||||
|
||||
- `Inf` - бесконечность.
|
||||
- `Inf` — бесконечность.
|
||||
|
||||
<!-- -->
|
||||
|
||||
@ -52,7 +59,7 @@ SELECT 0.5 / 0
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
- `-Inf` - отрицательная бесконечность;
|
||||
- `-Inf` — отрицательная бесконечность.
|
||||
|
||||
<!-- -->
|
||||
|
||||
@ -66,7 +73,7 @@ SELECT -0.5 / 0
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
- `NaN` - не число.
|
||||
- `NaN` — не число.
|
||||
|
||||
<!-- -->
|
||||
|
||||
@ -80,6 +87,6 @@ SELECT 0 / 0
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
Смотрите правила сортировки `NaN` в разделе [Секция ORDER BY](../sql_reference/data_types/float.md).
|
||||
Смотрите правила сортировки `NaN` в разделе [Секция ORDER BY ](../../sql-reference/statements/select/order-by.md).
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/float/) <!--hide-->
|
||||
|
||||
@ -7,23 +7,32 @@ toc_title: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64
|
||||
|
||||
Целые числа фиксированной длины, без знака или со знаком.
|
||||
|
||||
При создании таблиц для целых чисел можно указывать числовые параметры (например `TINYINT(8)`, `SMALLINT(16)`, `INT(32)`, `BIGINT(64)`), но ClickHouse их проигнорирует.
|
||||
|
||||
## Диапазоны Int {#int-ranges}
|
||||
|
||||
- Int8 - \[-128 : 127\]
|
||||
- Int16 - \[-32768 : 32767\]
|
||||
- Int32 - \[-2147483648 : 2147483647\]
|
||||
- Int64 - \[-9223372036854775808 : 9223372036854775807\]
|
||||
- Int128 - \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
|
||||
- Int256 - \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
|
||||
- `Int8` — \[-128 : 127\]
|
||||
- `Int16` — \[-32768 : 32767\]
|
||||
- `Int32` — \[-2147483648 : 2147483647\]
|
||||
- `Int64` — \[-9223372036854775808 : 9223372036854775807\]
|
||||
- `Int128` — \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
|
||||
- `Int256` — \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
|
||||
|
||||
Синонимы:
|
||||
|
||||
- `Int8` — `TINYINT`, `BOOL`, `BOOLEAN`, `INT1`.
|
||||
- `Int16` — `SMALLINT`, `INT2`.
|
||||
- `Int32` — `INT`, `INT4`, `INTEGER`.
|
||||
- `Int64` — `BIGINT`.
|
||||
|
||||
## Диапазоны Uint {#uint-ranges}
|
||||
|
||||
- UInt8 - \[0 : 255\]
|
||||
- UInt16 - \[0 : 65535\]
|
||||
- UInt32 - \[0 : 4294967295\]
|
||||
- UInt64 - \[0 : 18446744073709551615\]
|
||||
- UInt256 - \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
|
||||
- `UInt8` — \[0 : 255\]
|
||||
- `UInt16` — \[0 : 65535\]
|
||||
- `UInt32` — \[0 : 4294967295\]
|
||||
- `UInt64` — \[0 : 18446744073709551615\]
|
||||
- `UInt256` — \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
|
||||
|
||||
UInt128 пока не реализован.
|
||||
`UInt128` пока не реализован.
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/int_uint/) <!--hide-->
|
||||
|
||||
@ -8,6 +8,8 @@ toc_title: String
|
||||
Строки произвольной длины. Длина не ограничена. Значение может содержать произвольный набор байт, включая нулевые байты.
|
||||
Таким образом, тип String заменяет типы VARCHAR, BLOB, CLOB и т. п. из других СУБД.
|
||||
|
||||
При создании таблиц для строк можно указывать числовые параметры (например `VARCHAR(255)`), но СlickHouse их проигнорирует.
|
||||
|
||||
## Кодировки {#kodirovki}
|
||||
|
||||
В ClickHouse нет понятия кодировок. Строки могут содержать произвольный набор байт, который хранится и выводится, как есть.
|
||||
|
||||
@ -90,8 +90,10 @@ SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated'))
|
||||
|
||||
Поля настройки:
|
||||
|
||||
- `path` — Абсолютный путь к файлу.
|
||||
- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
|
||||
- `path` — абсолютный путь к файлу.
|
||||
- `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
|
||||
|
||||
Если словарь с источником `FILE` создается с помощью DDL-команды (`CREATE DICTIONARY ...`), источник словаря должен быть расположен в каталоге `user_files`. Иначе пользователи базы данных будут иметь доступ к произвольному файлу на узле ClickHouse.
|
||||
|
||||
## Исполняемый файл {#dicts-external_dicts_dict_sources-executable}
|
||||
|
||||
@ -108,16 +110,12 @@ SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated'))
|
||||
</source>
|
||||
```
|
||||
|
||||
или
|
||||
|
||||
``` sql
|
||||
SOURCE(EXECUTABLE(command 'cat /opt/dictionaries/os.tsv' format 'TabSeparated'))
|
||||
```
|
||||
|
||||
Поля настройки:
|
||||
|
||||
- `command` — Абсолютный путь к исполняемому файлу или имя файла (если каталог программы прописан в `PATH`).
|
||||
- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
|
||||
- `command` — абсолютный путь к исполняемому файлу или имя файла (если каталог программы прописан в `PATH`).
|
||||
- `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
|
||||
|
||||
Этот источник словаря может быть настроен только с помощью XML-конфигурации. Создание словарей с исполняемым источником с помощью DDL отключено. Иначе пользователь базы данных сможет выполнить произвольный бинарный файл на узле ClickHouse.
|
||||
|
||||
## HTTP(s) {#dicts-external_dicts_dict_sources-http}
|
||||
|
||||
@ -160,7 +158,16 @@ SOURCE(HTTP(
|
||||
Поля настройки:
|
||||
|
||||
- `url` — URL источника.
|
||||
- `format` — Формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
|
||||
- `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)».
|
||||
- `credentials` – базовая HTTP-аутентификация. Необязательный параметр.
|
||||
- `user` – имя пользователя, необходимое для аутентификации.
|
||||
- `password` – пароль, необходимый для аутентификации.
|
||||
- `headers` – все пользовательские записи HTTP-заголовков, используемые для HTTP-запроса. Необязательный параметр.
|
||||
- `header` – одна запись HTTP-заголовка.
|
||||
- `name` – идентифицирующее имя, используемое для отправки заголовка запроса.
|
||||
- `value` – значение, заданное для конкретного идентифицирующего имени.
|
||||
|
||||
При создании словаря с помощью DDL-команды (`CREATE DICTIONARY ...`) удаленные хосты для HTTP-словарей проверяются в разделе `remote_url_allow_hosts` из конфигурации сервера. Иначе пользователи базы данных будут иметь доступ к произвольному HTTP-серверу.
|
||||
|
||||
## ODBC {#dicts-external_dicts_dict_sources-odbc}
|
||||
|
||||
|
||||
@ -23,6 +23,7 @@ SELECT [DISTINCT] expr_list
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[SETTINGS ...]
|
||||
[UNION ALL ...]
|
||||
[INTO OUTFILE filename]
|
||||
[FORMAT format]
|
||||
@ -161,4 +162,17 @@ Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of
|
||||
|
||||
Подробнее смотрите в разделе «Настройки». Присутствует возможность использовать внешнюю сортировку (с сохранением временных данных на диск) и внешнюю агрегацию.
|
||||
|
||||
{## [Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/statements/select/) ##}
|
||||
## SETTINGS в запросе SELECT {#settings-in-select}
|
||||
|
||||
Вы можете задать значения необходимых настроек непосредственно в запросе `SELECT` в секции `SETTINGS`. Эти настройки действуют только в рамках данного запроса, а после его выполнения сбрасываются до предыдущего значения или значения по умолчанию.
|
||||
|
||||
Другие способы задания настроек описаны [здесь](../../../operations/settings/index.md).
|
||||
|
||||
**Пример**
|
||||
|
||||
``` sql
|
||||
SELECT * FROM some_table SETTINGS optimize_read_in_order=1, cast_keep_nullable=1;
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/)
|
||||
<!--hide-->
|
||||
|
||||
@ -613,24 +613,25 @@ formatDateTime(Time, Format\[, Timezone\])
|
||||
| %C | 年除以100并截断为整数(00-99) | 20 |
|
||||
| %d | 月中的一天，零填充（01-31) | 02 |
|
||||
| %D | 短MM/DD/YY日期，相当于%m/%d/%y | 01/02/2018 |
|
||||
| %e | 月中的一天，空格填充（1-31) | 2 |
|
||||
| %e | 月中的一天，空格填充（ 1-31) | 2 |
|
||||
| %F | 短YYYY-MM-DD日期，相当于%Y-%m-%d | 2018-01-02 |
|
||||
| %G | ISO周号的四位数年份格式， 从基于周的年份[由ISO 8601定义](https://en.wikipedia.org/wiki/ISO_8601#Week_dates) 标准计算得出，通常仅对％V有用 | 2018 |
|
||||
| %g | 两位数的年份格式，与ISO 8601一致，四位数表示法的缩写 | 18 |
|
||||
| %H | 24小时格式（00-23) | 22 |
|
||||
| %I | 小时12h格式（01-12) | 10 |
|
||||
| %j | 一年(001-366) | 002 |
|
||||
| %I | 12小时格式（01-12) | 10 |
|
||||
| %j | 一年中的一天 (001-366) | 002 |
|
||||
| %m | 月份为十进制数（01-12) | 01 |
|
||||
| %M | 分钟(00-59) | 33 |
|
||||
| %n | 换行符(") | |
|
||||
| %p | AM或PM指定 | PM |
|
||||
| %Q | 季度（1-4) | 1 |
|
||||
| %R | 24小时HH:MM时间，相当于%H:%M | 22:33 |
|
||||
| %S | 第二(00-59) | 44 |
|
||||
| %S | 秒 (00-59) | 44 |
|
||||
| %t | 水平制表符(’) | |
|
||||
| %T | ISO8601时间格式(HH:MM:SS)，相当于%H:%M:%S | 22:33:44 |
|
||||
| %u | ISO8601平日as编号，星期一为1(1-7) | 2 |
|
||||
| %u | ISO8601工作日为数字，星期一为1(1-7) | 2 |
|
||||
| %V | ISO8601周编号(01-53) | 01 |
|
||||
| %w | 周日为十进制数，周日为0(0-6) | 2 |
|
||||
| %w | 工作日为十进制数，周日为0(0-6) | 2 |
|
||||
| %y | 年份，最后两位数字（00-99) | 18 |
|
||||
| %Y | 年 | 2018 |
|
||||
| %% | %符号 | % |
|
||||
|
||||
@ -141,6 +141,8 @@ private:
|
||||
|
||||
/// If not empty, queries will be read from these files
|
||||
std::vector<std::string> queries_files;
|
||||
/// If not empty, run queries from these files before processing every file from 'queries_files'.
|
||||
std::vector<std::string> interleave_queries_files;
|
||||
|
||||
std::unique_ptr<Connection> connection; /// Connection to DB.
|
||||
String full_query; /// Current query as it was given to the client.
|
||||
@ -796,13 +798,22 @@ private:
|
||||
|
||||
if (!queries_files.empty())
|
||||
{
|
||||
for (const auto & queries_file : queries_files)
|
||||
auto process_file = [&](const std::string & file)
|
||||
{
|
||||
connection->setDefaultDatabase(connection_parameters.default_database);
|
||||
ReadBufferFromFile in(queries_file);
|
||||
ReadBufferFromFile in(file);
|
||||
readStringUntilEOF(text, in);
|
||||
if (!processMultiQuery(text))
|
||||
break;
|
||||
return processMultiQuery(text);
|
||||
};
|
||||
|
||||
for (const auto & queries_file : queries_files)
|
||||
{
|
||||
for (const auto & interleave_file : interleave_queries_files)
|
||||
if (!process_file(interleave_file))
|
||||
return;
|
||||
|
||||
if (!process_file(queries_file))
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
@ -945,6 +956,7 @@ private:
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -2271,7 +2283,9 @@ public:
|
||||
("highlight", po::value<bool>()->default_value(true), "enable or disable basic syntax highlight in interactive command line")
|
||||
("log-level", po::value<std::string>(), "client log level")
|
||||
("server_logs_file", po::value<std::string>(), "put server logs into specified file")
|
||||
("query-fuzzer-runs", po::value<int>()->default_value(0), "query fuzzer runs")
|
||||
("query-fuzzer-runs", po::value<int>()->default_value(0), "After executing every SELECT query, do random mutations in it and run again specified number of times. This is used for testing to discover unexpected corner cases.")
|
||||
("interleave-queries-file", po::value<std::vector<std::string>>()->multitoken(),
|
||||
"file path with queries to execute before every file from 'queries-file'; multiple files can be specified (--queries-file file1 file2...); this is needed to enable more aggressive fuzzing of newly added tests (see 'query-fuzzer-runs' option)")
|
||||
("opentelemetry-traceparent", po::value<std::string>(), "OpenTelemetry traceparent header as described by W3C Trace Context recommendation")
|
||||
("opentelemetry-tracestate", po::value<std::string>(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation")
|
||||
("history_file", po::value<std::string>(), "path to history file")
|
||||
@ -2386,6 +2400,8 @@ public:
|
||||
config().setString("query", options["query"].as<std::string>());
|
||||
if (options.count("queries-file"))
|
||||
queries_files = options["queries-file"].as<std::vector<std::string>>();
|
||||
if (options.count("interleave-queries-file"))
|
||||
interleave_queries_files = options["interleave-queries-file"].as<std::vector<std::string>>();
|
||||
if (options.count("database"))
|
||||
config().setString("database", options["database"].as<std::string>());
|
||||
if (options.count("pager"))
|
||||
|
||||
@ -692,6 +692,37 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
{
|
||||
Settings::checkNoSettingNamesAtTopLevel(*config, config_path);
|
||||
|
||||
/// Limit on total memory usage
|
||||
size_t max_server_memory_usage = config->getUInt64("max_server_memory_usage", 0);
|
||||
|
||||
double max_server_memory_usage_to_ram_ratio = config->getDouble("max_server_memory_usage_to_ram_ratio", 0.9);
|
||||
size_t default_max_server_memory_usage = memory_amount * max_server_memory_usage_to_ram_ratio;
|
||||
|
||||
if (max_server_memory_usage == 0)
|
||||
{
|
||||
max_server_memory_usage = default_max_server_memory_usage;
|
||||
LOG_INFO(log, "Setting max_server_memory_usage was set to {}"
|
||||
" ({} available * {:.2f} max_server_memory_usage_to_ram_ratio)",
|
||||
formatReadableSizeWithBinarySuffix(max_server_memory_usage),
|
||||
formatReadableSizeWithBinarySuffix(memory_amount),
|
||||
max_server_memory_usage_to_ram_ratio);
|
||||
}
|
||||
else if (max_server_memory_usage > default_max_server_memory_usage)
|
||||
{
|
||||
max_server_memory_usage = default_max_server_memory_usage;
|
||||
LOG_INFO(log, "Setting max_server_memory_usage was lowered to {}"
|
||||
" because the system has low amount of memory. The amount was"
|
||||
" calculated as {} available"
|
||||
" * {:.2f} max_server_memory_usage_to_ram_ratio",
|
||||
formatReadableSizeWithBinarySuffix(max_server_memory_usage),
|
||||
formatReadableSizeWithBinarySuffix(memory_amount),
|
||||
max_server_memory_usage_to_ram_ratio);
|
||||
}
|
||||
|
||||
total_memory_tracker.setHardLimit(max_server_memory_usage);
|
||||
total_memory_tracker.setDescription("(total)");
|
||||
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
|
||||
|
||||
// FIXME logging-related things need synchronization -- see the 'Logger * log' saved
|
||||
// in a lot of places. For now, disable updating log configuration without server restart.
|
||||
//setTextLog(global_context->getTextLog());
|
||||
@ -780,37 +811,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
global_context->getMergeTreeSettings().sanityCheck(settings);
|
||||
global_context->getReplicatedMergeTreeSettings().sanityCheck(settings);
|
||||
|
||||
/// Limit on total memory usage
|
||||
size_t max_server_memory_usage = config().getUInt64("max_server_memory_usage", 0);
|
||||
|
||||
double max_server_memory_usage_to_ram_ratio = config().getDouble("max_server_memory_usage_to_ram_ratio", 0.9);
|
||||
size_t default_max_server_memory_usage = memory_amount * max_server_memory_usage_to_ram_ratio;
|
||||
|
||||
if (max_server_memory_usage == 0)
|
||||
{
|
||||
max_server_memory_usage = default_max_server_memory_usage;
|
||||
LOG_INFO(log, "Setting max_server_memory_usage was set to {}"
|
||||
" ({} available * {:.2f} max_server_memory_usage_to_ram_ratio)",
|
||||
formatReadableSizeWithBinarySuffix(max_server_memory_usage),
|
||||
formatReadableSizeWithBinarySuffix(memory_amount),
|
||||
max_server_memory_usage_to_ram_ratio);
|
||||
}
|
||||
else if (max_server_memory_usage > default_max_server_memory_usage)
|
||||
{
|
||||
max_server_memory_usage = default_max_server_memory_usage;
|
||||
LOG_INFO(log, "Setting max_server_memory_usage was lowered to {}"
|
||||
" because the system has low amount of memory. The amount was"
|
||||
" calculated as {} available"
|
||||
" * {:.2f} max_server_memory_usage_to_ram_ratio",
|
||||
formatReadableSizeWithBinarySuffix(max_server_memory_usage),
|
||||
formatReadableSizeWithBinarySuffix(memory_amount),
|
||||
max_server_memory_usage_to_ram_ratio);
|
||||
}
|
||||
|
||||
total_memory_tracker.setOrRaiseHardLimit(max_server_memory_usage);
|
||||
total_memory_tracker.setDescription("(total)");
|
||||
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
|
||||
|
||||
Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0);
|
||||
|
||||
Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024));
|
||||
|
||||
@ -28,6 +28,10 @@ struct AggregateFunctionSumData
|
||||
/// Vectorized version
|
||||
template <typename Value>
|
||||
void NO_INLINE addMany(const Value * __restrict ptr, size_t count)
|
||||
{
|
||||
const auto * end = ptr + count;
|
||||
|
||||
if constexpr (std::is_floating_point_v<T>)
|
||||
{
|
||||
/// Compiler cannot unroll this loop, do it manually.
|
||||
/// (at least for floats, most likely due to the lack of -fassociative-math)
|
||||
@ -36,7 +40,6 @@ struct AggregateFunctionSumData
|
||||
constexpr size_t unroll_count = 128 / sizeof(T);
|
||||
T partial_sums[unroll_count]{};
|
||||
|
||||
const auto * end = ptr + count;
|
||||
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
|
||||
|
||||
while (ptr < unrolled_end)
|
||||
@ -48,21 +51,28 @@ struct AggregateFunctionSumData
|
||||
|
||||
for (size_t i = 0; i < unroll_count; ++i)
|
||||
sum += partial_sums[i];
|
||||
}
|
||||
|
||||
/// clang cannot vectorize the loop if accumulator is class member instead of local variable.
|
||||
T local_sum{};
|
||||
while (ptr < end)
|
||||
{
|
||||
sum += *ptr;
|
||||
local_sum += *ptr;
|
||||
++ptr;
|
||||
}
|
||||
sum += local_sum;
|
||||
}
|
||||
|
||||
template <typename Value>
|
||||
void NO_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count)
|
||||
{
|
||||
const auto * end = ptr + count;
|
||||
|
||||
if constexpr (std::is_floating_point_v<T>)
|
||||
{
|
||||
constexpr size_t unroll_count = 128 / sizeof(T);
|
||||
T partial_sums[unroll_count]{};
|
||||
|
||||
const auto * end = ptr + count;
|
||||
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
|
||||
|
||||
while (ptr < unrolled_end)
|
||||
@ -76,14 +86,17 @@ struct AggregateFunctionSumData
|
||||
|
||||
for (size_t i = 0; i < unroll_count; ++i)
|
||||
sum += partial_sums[i];
|
||||
}
|
||||
|
||||
T local_sum{};
|
||||
while (ptr < end)
|
||||
{
|
||||
if (!*null_map)
|
||||
sum += *ptr;
|
||||
local_sum += *ptr;
|
||||
++ptr;
|
||||
++null_map;
|
||||
}
|
||||
sum += local_sum;
|
||||
}
|
||||
|
||||
void merge(const AggregateFunctionSumData & rhs)
|
||||
|
||||
@ -145,8 +145,9 @@ void MemoryTracker::alloc(Int64 size)
|
||||
*/
|
||||
Int64 will_be = size + amount.fetch_add(size, std::memory_order_relaxed);
|
||||
|
||||
if (metric != CurrentMetrics::end())
|
||||
CurrentMetrics::add(metric, size);
|
||||
auto metric_loaded = metric.load(std::memory_order_relaxed);
|
||||
if (metric_loaded != CurrentMetrics::end())
|
||||
CurrentMetrics::add(metric_loaded, size);
|
||||
|
||||
Int64 current_hard_limit = hard_limit.load(std::memory_order_relaxed);
|
||||
Int64 current_profiler_limit = profiler_limit.load(std::memory_order_relaxed);
|
||||
@ -286,8 +287,9 @@ void MemoryTracker::free(Int64 size)
|
||||
if (auto * loaded_next = parent.load(std::memory_order_relaxed))
|
||||
loaded_next->free(size);
|
||||
|
||||
if (metric != CurrentMetrics::end())
|
||||
CurrentMetrics::sub(metric, accounted_size);
|
||||
auto metric_loaded = metric.load(std::memory_order_relaxed);
|
||||
if (metric_loaded != CurrentMetrics::end())
|
||||
CurrentMetrics::sub(metric_loaded, accounted_size);
|
||||
}
|
||||
|
||||
|
||||
@ -302,8 +304,9 @@ void MemoryTracker::resetCounters()
|
||||
|
||||
void MemoryTracker::reset()
|
||||
{
|
||||
if (metric != CurrentMetrics::end())
|
||||
CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed));
|
||||
auto metric_loaded = metric.load(std::memory_order_relaxed);
|
||||
if (metric_loaded != CurrentMetrics::end())
|
||||
CurrentMetrics::sub(metric_loaded, amount.load(std::memory_order_relaxed));
|
||||
|
||||
resetCounters();
|
||||
}
|
||||
@ -316,6 +319,12 @@ void MemoryTracker::set(Int64 to)
|
||||
}
|
||||
|
||||
|
||||
void MemoryTracker::setHardLimit(Int64 value)
|
||||
{
|
||||
hard_limit.store(value, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
|
||||
void MemoryTracker::setOrRaiseHardLimit(Int64 value)
|
||||
{
|
||||
/// This is just atomic set to maximum.
|
||||
|
||||
@ -53,7 +53,7 @@ private:
|
||||
std::atomic<MemoryTracker *> parent {};
|
||||
|
||||
/// You could specify custom metric to track memory usage.
|
||||
CurrentMetrics::Metric metric = CurrentMetrics::end();
|
||||
std::atomic<CurrentMetrics::Metric> metric = CurrentMetrics::end();
|
||||
|
||||
/// This description will be used as prefix into log messages (if isn't nullptr)
|
||||
std::atomic<const char *> description_ptr = nullptr;
|
||||
@ -96,6 +96,8 @@ public:
|
||||
return peak.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
void setHardLimit(Int64 value);
|
||||
|
||||
/** Set limit if it was not set.
|
||||
* Otherwise, set limit to new value, if new value is greater than previous limit.
|
||||
*/
|
||||
@ -132,7 +134,7 @@ public:
|
||||
/// The memory consumption could be shown in realtime via CurrentMetrics counter
|
||||
void setMetric(CurrentMetrics::Metric metric_)
|
||||
{
|
||||
metric = metric_;
|
||||
metric.store(metric_, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
void setDescription(const char * description)
|
||||
|
||||
@ -67,6 +67,7 @@ class IColumn;
|
||||
M(UInt64, s3_min_upload_part_size, 512*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \
|
||||
M(UInt64, s3_max_single_part_upload_size, 64*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \
|
||||
M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \
|
||||
M(UInt64, s3_max_connections, 1024, "The maximum number of connections per server.", 0) \
|
||||
M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \
|
||||
M(Bool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.", 0) \
|
||||
M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \
|
||||
|
||||
@ -120,6 +120,7 @@ void registerDiskS3(DiskFactory & factory)
|
||||
|
||||
cfg.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 10000);
|
||||
cfg.httpRequestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 5000);
|
||||
cfg.maxConnections = config.getUInt(config_prefix + ".max_connections", 100);
|
||||
cfg.endpointOverride = uri.endpoint;
|
||||
|
||||
auto proxy_config = getProxyConfiguration(config_prefix, config);
|
||||
|
||||
@ -272,6 +272,11 @@ private:
|
||||
writeNumber2(target + 3, ToMinuteImpl::execute(source, timezone));
|
||||
writeNumber2(target + 6, ToSecondImpl::execute(source, timezone));
|
||||
}
|
||||
|
||||
static void quarter(char * target, Time source, const DateLUTImpl & timezone)
|
||||
{
|
||||
*target += ToQuarterImpl::execute(source, timezone);
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
@ -621,6 +626,12 @@ public:
|
||||
result.append("0000");
|
||||
break;
|
||||
|
||||
// Quarter (1-4)
|
||||
case 'Q':
|
||||
instructions.template emplace_back(&Action<T>::quarter, 1);
|
||||
result.append("0");
|
||||
break;
|
||||
|
||||
/// Time components. If the argument is Date, not a DateTime, then this components will have default value.
|
||||
|
||||
// Minute (00-59)
|
||||
|
||||
@ -106,23 +106,72 @@ namespace
|
||||
const std::string host;
|
||||
const UInt16 port;
|
||||
bool https;
|
||||
const String proxy_host;
|
||||
const UInt16 proxy_port;
|
||||
bool proxy_https;
|
||||
bool resolve_host;
|
||||
using Base = PoolBase<Poco::Net::HTTPClientSession>;
|
||||
ObjectPtr allocObject() override
|
||||
{
|
||||
return makeHTTPSessionImpl(host, port, https, true);
|
||||
auto session = makeHTTPSessionImpl(host, port, https, true, resolve_host);
|
||||
if (!proxy_host.empty())
|
||||
{
|
||||
const String proxy_scheme = proxy_https ? "https" : "http";
|
||||
session->setProxyHost(proxy_host);
|
||||
session->setProxyPort(proxy_port);
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
session->setProxyProtocol(proxy_scheme);
|
||||
|
||||
/// Turn on tunnel mode if proxy scheme is HTTP while endpoint scheme is HTTPS.
|
||||
session->setProxyTunnel(!proxy_https && https);
|
||||
#endif
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
public:
|
||||
SingleEndpointHTTPSessionPool(const std::string & host_, UInt16 port_, bool https_, size_t max_pool_size_)
|
||||
: Base(max_pool_size_, &Poco::Logger::get("HTTPSessionPool")), host(host_), port(port_), https(https_)
|
||||
SingleEndpointHTTPSessionPool(
|
||||
const std::string & host_,
|
||||
UInt16 port_,
|
||||
bool https_,
|
||||
const std::string & proxy_host_,
|
||||
UInt16 proxy_port_,
|
||||
bool proxy_https_,
|
||||
size_t max_pool_size_,
|
||||
bool resolve_host_ = true)
|
||||
: Base(max_pool_size_, &Poco::Logger::get("HTTPSessionPool"))
|
||||
, host(host_)
|
||||
, port(port_)
|
||||
, https(https_)
|
||||
, proxy_host(proxy_host_)
|
||||
, proxy_port(proxy_port_)
|
||||
, proxy_https(proxy_https_)
|
||||
, resolve_host(resolve_host_)
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
class HTTPSessionPool : private boost::noncopyable
|
||||
{
|
||||
public:
|
||||
struct Key
|
||||
{
|
||||
String target_host;
|
||||
UInt16 target_port;
|
||||
bool is_target_https;
|
||||
String proxy_host;
|
||||
UInt16 proxy_port;
|
||||
bool is_proxy_https;
|
||||
|
||||
bool operator ==(const Key & rhs) const
|
||||
{
|
||||
return std::tie(target_host, target_port, is_target_https, proxy_host, proxy_port, is_proxy_https)
|
||||
== std::tie(rhs.target_host, rhs.target_port, rhs.is_target_https, rhs.proxy_host, rhs.proxy_port, rhs.is_proxy_https);
|
||||
}
|
||||
};
|
||||
|
||||
private:
|
||||
using Key = std::tuple<std::string, UInt16, bool>;
|
||||
using PoolPtr = std::shared_ptr<SingleEndpointHTTPSessionPool>;
|
||||
using Entry = SingleEndpointHTTPSessionPool::Entry;
|
||||
|
||||
@ -131,9 +180,12 @@ namespace
|
||||
size_t operator()(const Key & k) const
|
||||
{
|
||||
SipHash s;
|
||||
s.update(std::get<0>(k));
|
||||
s.update(std::get<1>(k));
|
||||
s.update(std::get<2>(k));
|
||||
s.update(k.target_host);
|
||||
s.update(k.target_port);
|
||||
s.update(k.is_target_https);
|
||||
s.update(k.proxy_host);
|
||||
s.update(k.proxy_port);
|
||||
s.update(k.is_proxy_https);
|
||||
return s.get64();
|
||||
}
|
||||
};
|
||||
@ -153,18 +205,32 @@ namespace
|
||||
|
||||
Entry getSession(
|
||||
const Poco::URI & uri,
|
||||
const Poco::URI & proxy_uri,
|
||||
const ConnectionTimeouts & timeouts,
|
||||
size_t max_connections_per_endpoint)
|
||||
size_t max_connections_per_endpoint,
|
||||
bool resolve_host = true)
|
||||
{
|
||||
std::unique_lock lock(mutex);
|
||||
const std::string & host = uri.getHost();
|
||||
UInt16 port = uri.getPort();
|
||||
bool https = isHTTPS(uri);
|
||||
auto key = std::make_tuple(host, port, https);
|
||||
|
||||
|
||||
String proxy_host;
|
||||
UInt16 proxy_port = 0;
|
||||
bool proxy_https = false;
|
||||
if (!proxy_uri.empty())
|
||||
{
|
||||
proxy_host = proxy_uri.getHost();
|
||||
proxy_port = proxy_uri.getPort();
|
||||
proxy_https = isHTTPS(proxy_uri);
|
||||
}
|
||||
|
||||
HTTPSessionPool::Key key{host, port, https, proxy_host, proxy_port, proxy_https};
|
||||
auto pool_ptr = endpoints_pool.find(key);
|
||||
if (pool_ptr == endpoints_pool.end())
|
||||
std::tie(pool_ptr, std::ignore) = endpoints_pool.emplace(
|
||||
key, std::make_shared<SingleEndpointHTTPSessionPool>(host, port, https, max_connections_per_endpoint));
|
||||
key, std::make_shared<SingleEndpointHTTPSessionPool>(host, port, https, proxy_host, proxy_port, proxy_https, max_connections_per_endpoint, resolve_host));
|
||||
|
||||
auto retry_timeout = timeouts.connection_timeout.totalMicroseconds();
|
||||
auto session = pool_ptr->second->get(retry_timeout);
|
||||
@ -178,6 +244,9 @@ namespace
|
||||
if (!msg.empty())
|
||||
{
|
||||
LOG_TRACE((&Poco::Logger::get("HTTPCommon")), "Failed communicating with {} with error '{}' will try to reconnect session", host, msg);
|
||||
|
||||
if (resolve_host)
|
||||
{
|
||||
/// Host can change IP
|
||||
const auto ip = DNSResolver::instance().resolveHost(host).toString();
|
||||
if (ip != session->getHost())
|
||||
@ -188,6 +257,7 @@ namespace
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setTimeouts(*session, timeouts);
|
||||
|
||||
@ -218,9 +288,14 @@ HTTPSessionPtr makeHTTPSession(const Poco::URI & uri, const ConnectionTimeouts &
|
||||
}
|
||||
|
||||
|
||||
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size)
|
||||
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size, bool resolve_host)
|
||||
{
|
||||
return HTTPSessionPool::instance().getSession(uri, timeouts, per_endpoint_pool_size);
|
||||
return makePooledHTTPSession(uri, {}, timeouts, per_endpoint_pool_size, resolve_host);
|
||||
}
|
||||
|
||||
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const Poco::URI & proxy_uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size, bool resolve_host)
|
||||
{
|
||||
return HTTPSessionPool::instance().getSession(uri, proxy_uri, timeouts, per_endpoint_pool_size, resolve_host);
|
||||
}
|
||||
|
||||
bool isRedirect(const Poco::Net::HTTPResponse::HTTPStatus status) { return status == Poco::Net::HTTPResponse::HTTP_MOVED_PERMANENTLY || status == Poco::Net::HTTPResponse::HTTP_FOUND || status == Poco::Net::HTTPResponse::HTTP_SEE_OTHER || status == Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT; }
|
||||
|
||||
@ -50,8 +50,9 @@ void setResponseDefaultHeaders(Poco::Net::HTTPServerResponse & response, unsigne
|
||||
/// Create session object to perform requests and set required parameters.
|
||||
HTTPSessionPtr makeHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, bool resolve_host = true);
|
||||
|
||||
/// As previous method creates session, but tooks it from pool
|
||||
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size);
|
||||
/// As previous method creates session, but tooks it from pool, without and with proxy uri.
|
||||
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size, bool resolve_host = true);
|
||||
PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const Poco::URI & proxy_uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size, bool resolve_host = true);
|
||||
|
||||
bool isRedirect(const Poco::Net::HTTPResponse::HTTPStatus status);
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
|
||||
#include <utility>
|
||||
#include <IO/HTTPCommon.h>
|
||||
#include <IO/S3/PocoHTTPResponseStream.h>
|
||||
#include <IO/S3/SessionAwareAwsStream.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
@ -86,6 +86,7 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & clientConfigu
|
||||
))
|
||||
, remote_host_filter(clientConfiguration.remote_host_filter)
|
||||
, s3_max_redirects(clientConfiguration.s3_max_redirects)
|
||||
, max_connections(clientConfiguration.maxConnections)
|
||||
{
|
||||
}
|
||||
|
||||
@ -164,28 +165,24 @@ void PocoHTTPClient::makeRequestInternal(
|
||||
{
|
||||
for (unsigned int attempt = 0; attempt <= s3_max_redirects; ++attempt)
|
||||
{
|
||||
Poco::URI poco_uri(uri);
|
||||
|
||||
/// Reverse proxy can replace host header with resolved ip address instead of host name.
|
||||
/// This can lead to request signature difference on S3 side.
|
||||
auto session = makeHTTPSession(poco_uri, timeouts, false);
|
||||
Poco::URI target_uri(uri);
|
||||
Poco::URI proxy_uri;
|
||||
|
||||
auto request_configuration = per_request_configuration(request);
|
||||
if (!request_configuration.proxyHost.empty())
|
||||
{
|
||||
/// Turn on tunnel mode if proxy scheme is HTTP while endpoint scheme is HTTPS.
|
||||
bool use_tunnel = request_configuration.proxyScheme == Aws::Http::Scheme::HTTP && poco_uri.getScheme() == "https";
|
||||
session->setProxy(
|
||||
request_configuration.proxyHost,
|
||||
request_configuration.proxyPort,
|
||||
Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme),
|
||||
use_tunnel
|
||||
);
|
||||
proxy_uri.setScheme(Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme));
|
||||
proxy_uri.setHost(request_configuration.proxyHost);
|
||||
proxy_uri.setPort(request_configuration.proxyPort);
|
||||
}
|
||||
|
||||
/// Reverse proxy can replace host header with resolved ip address instead of host name.
|
||||
/// This can lead to request signature difference on S3 side.
|
||||
auto session = makePooledHTTPSession(target_uri, proxy_uri, timeouts, max_connections, false);
|
||||
|
||||
Poco::Net::HTTPRequest poco_request(Poco::Net::HTTPRequest::HTTP_1_1);
|
||||
|
||||
poco_request.setURI(poco_uri.getPathAndQuery());
|
||||
poco_request.setURI(target_uri.getPathAndQuery());
|
||||
|
||||
switch (request.GetMethod())
|
||||
{
|
||||
@ -281,7 +278,7 @@ void PocoHTTPClient::makeRequestInternal(
|
||||
}
|
||||
}
|
||||
else
|
||||
response->GetResponseStream().SetUnderlyingStream(std::make_shared<PocoHTTPResponseStream>(session, response_body_stream));
|
||||
response->GetResponseStream().SetUnderlyingStream(std::make_shared<SessionAwareAwsStream<decltype(session)>>(session, response_body_stream));
|
||||
|
||||
return;
|
||||
}
|
||||
@ -297,6 +294,7 @@ void PocoHTTPClient::makeRequestInternal(
|
||||
ProfileEvents::increment(select_metric(S3MetricType::Errors));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -56,6 +56,7 @@ private:
|
||||
ConnectionTimeouts timeouts;
|
||||
const RemoteHostFilter & remote_host_filter;
|
||||
unsigned int s3_max_redirects;
|
||||
unsigned int max_connections;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
#include <Common/config.h>
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
|
||||
#include "PocoHTTPResponseStream.h"
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace DB::S3
|
||||
{
|
||||
PocoHTTPResponseStream::PocoHTTPResponseStream(std::shared_ptr<Poco::Net::HTTPClientSession> session_, std::istream & response_stream_)
|
||||
: Aws::IOStream(response_stream_.rdbuf()), session(std::move(session_))
|
||||
{
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -1,21 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <aws/core/utils/stream/ResponseStream.h>
|
||||
#include <Poco/Net/HTTPClientSession.h>
|
||||
|
||||
namespace DB::S3
|
||||
{
|
||||
/**
|
||||
* Wrapper of IStream to store response stream and corresponding HTTP session.
|
||||
*/
|
||||
class PocoHTTPResponseStream : public Aws::IOStream
|
||||
{
|
||||
public:
|
||||
PocoHTTPResponseStream(std::shared_ptr<Poco::Net::HTTPClientSession> session_, std::istream & response_stream_);
|
||||
|
||||
private:
|
||||
/// Poco HTTP session is holder of response stream.
|
||||
std::shared_ptr<Poco::Net::HTTPClientSession> session;
|
||||
};
|
||||
|
||||
}
|
||||
27
src/IO/S3/SessionAwareAwsStream.h
Normal file
27
src/IO/S3/SessionAwareAwsStream.h
Normal file
@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include <IO/HTTPCommon.h>
|
||||
|
||||
#include <aws/core/utils/stream/ResponseStream.h>
|
||||
|
||||
|
||||
namespace DB::S3
|
||||
{
|
||||
/**
|
||||
* Wrapper of IOStream to store response stream and corresponding HTTP session.
|
||||
*/
|
||||
template <typename Session>
|
||||
class SessionAwareAwsStream : public Aws::IStream
|
||||
{
|
||||
public:
|
||||
SessionAwareAwsStream(Session session_, std::istream & response_stream_)
|
||||
: Aws::IStream(response_stream_.rdbuf()), session(std::move(session_))
|
||||
{
|
||||
}
|
||||
|
||||
private:
|
||||
/// Poco HTTP session is holder of response stream.
|
||||
Session session;
|
||||
};
|
||||
|
||||
}
|
||||
@ -280,7 +280,7 @@ namespace S3
|
||||
}
|
||||
|
||||
std::shared_ptr<Aws::S3::S3Client> ClientFactory::create( // NOLINT
|
||||
Aws::Client::ClientConfiguration & cfg,
|
||||
const Aws::Client::ClientConfiguration & cfg,
|
||||
bool is_virtual_hosted_style,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key,
|
||||
@ -306,7 +306,7 @@ namespace S3
|
||||
}
|
||||
|
||||
std::shared_ptr<Aws::S3::S3Client> ClientFactory::create( // NOLINT
|
||||
const String & endpoint,
|
||||
const Aws::Client::ClientConfiguration & cfg,
|
||||
bool is_virtual_hosted_style,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key,
|
||||
@ -315,10 +315,7 @@ namespace S3
|
||||
const RemoteHostFilter & remote_host_filter,
|
||||
unsigned int s3_max_redirects)
|
||||
{
|
||||
PocoHTTPClientConfiguration client_configuration({}, remote_host_filter, s3_max_redirects);
|
||||
|
||||
if (!endpoint.empty())
|
||||
client_configuration.endpointOverride = endpoint;
|
||||
PocoHTTPClientConfiguration client_configuration(cfg, remote_host_filter, s3_max_redirects);
|
||||
|
||||
client_configuration.updateSchemeAndRegion();
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@ public:
|
||||
unsigned int s3_max_redirects);
|
||||
|
||||
std::shared_ptr<Aws::S3::S3Client> create(
|
||||
Aws::Client::ClientConfiguration & cfg,
|
||||
const Aws::Client::ClientConfiguration & cfg,
|
||||
bool is_virtual_hosted_style,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key,
|
||||
@ -50,7 +50,7 @@ public:
|
||||
unsigned int s3_max_redirects);
|
||||
|
||||
std::shared_ptr<Aws::S3::S3Client> create(
|
||||
const String & endpoint,
|
||||
const Aws::Client::ClientConfiguration & cfg,
|
||||
bool is_virtual_hosted_style,
|
||||
const String & access_key_id,
|
||||
const String & secret_access_key,
|
||||
|
||||
@ -10,7 +10,6 @@
|
||||
#include <Access/AccessFlags.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -40,6 +39,7 @@ BlockInputStreamPtr InterpreterExistsQuery::executeImpl()
|
||||
{
|
||||
ASTQueryWithTableAndOutput * exists_query;
|
||||
bool result = false;
|
||||
|
||||
if ((exists_query = query_ptr->as<ASTExistsTableQuery>()))
|
||||
{
|
||||
if (exists_query->temporary)
|
||||
@ -57,8 +57,8 @@ BlockInputStreamPtr InterpreterExistsQuery::executeImpl()
|
||||
{
|
||||
String database = context.resolveDatabase(exists_query->database);
|
||||
context.checkAccess(AccessType::SHOW_TABLES, database, exists_query->table);
|
||||
auto tbl = DatabaseCatalog::instance().tryGetTable({database, exists_query->table}, context);
|
||||
result = tbl != nullptr && tbl->isView();
|
||||
auto table = DatabaseCatalog::instance().tryGetTable({database, exists_query->table}, context);
|
||||
result = table && table->isView();
|
||||
}
|
||||
else if ((exists_query = query_ptr->as<ASTExistsDatabaseQuery>()))
|
||||
{
|
||||
|
||||
@ -41,6 +41,16 @@ DropQuery::createDropTable(bool detach, bool if_exists, bool temporary, PtrTo<Ta
|
||||
return query;
|
||||
}
|
||||
|
||||
// static
|
||||
PtrTo<DropQuery>
|
||||
DropQuery::createDropView(bool detach, bool if_exists, PtrTo<TableIdentifier> identifier, PtrTo<ClusterClause> cluster)
|
||||
{
|
||||
auto query = PtrTo<DropQuery>(new DropQuery(cluster, QueryType::VIEW, {identifier}));
|
||||
query->detach = detach;
|
||||
query->if_exists = if_exists;
|
||||
return query;
|
||||
}
|
||||
|
||||
DropQuery::DropQuery(PtrTo<ClusterClause> cluster, QueryType type, PtrList exprs) : DDLQuery(cluster, exprs), query_type(type)
|
||||
{
|
||||
}
|
||||
@ -73,6 +83,14 @@ ASTPtr DropQuery::convertToOld() const
|
||||
query->database = database->getName();
|
||||
break;
|
||||
}
|
||||
case QueryType::VIEW:
|
||||
{
|
||||
query->is_view = true;
|
||||
query->table = get<TableIdentifier>(NAME)->getName();
|
||||
if (auto database = get<TableIdentifier>(NAME)->getDatabase())
|
||||
query->database = database->getName();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
convertToOldPartially(query);
|
||||
@ -100,6 +118,8 @@ antlrcpp::Any ParseTreeVisitor::visitDropTableStmt(ClickHouseParser::DropTableSt
|
||||
return DropQuery::createDropTable(!!ctx->DETACH(), !!ctx->EXISTS(), !!ctx->TEMPORARY(), visit(ctx->tableIdentifier()), cluster);
|
||||
if (ctx->DICTIONARY())
|
||||
return DropQuery::createDropDictionary(!!ctx->DETACH(), !!ctx->EXISTS(), visit(ctx->tableIdentifier()), cluster);
|
||||
if (ctx->VIEW())
|
||||
return DropQuery::createDropView(!!ctx->DETACH(), !!ctx->EXISTS(), visit(ctx->tableIdentifier()), cluster);
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
|
||||
@ -15,6 +15,8 @@ class DropQuery : public DDLQuery
|
||||
createDropTable(bool detach, bool if_exists, bool temporary, PtrTo<TableIdentifier> identifier, PtrTo<ClusterClause> cluster);
|
||||
static PtrTo<DropQuery>
|
||||
createDropDictionary(bool detach, bool if_exists, PtrTo<TableIdentifier> identifier, PtrTo<ClusterClause> cluster);
|
||||
static PtrTo<DropQuery>
|
||||
createDropView(bool detach, bool if_exists, PtrTo<TableIdentifier> identifier, PtrTo<ClusterClause> cluster);
|
||||
|
||||
ASTPtr convertToOld() const override;
|
||||
|
||||
@ -29,6 +31,7 @@ class DropQuery : public DDLQuery
|
||||
DATABASE,
|
||||
DICTIONARY,
|
||||
TABLE,
|
||||
VIEW,
|
||||
};
|
||||
|
||||
const QueryType query_type;
|
||||
|
||||
@ -10,31 +10,51 @@
|
||||
namespace DB::AST
|
||||
{
|
||||
|
||||
ExistsQuery::ExistsQuery(QueryType type, bool temporary_, PtrTo<TableIdentifier> identifier)
|
||||
: Query{identifier}, query_type(type), temporary(temporary_)
|
||||
ExistsQuery::ExistsQuery(QueryType type, bool temporary_, PtrList exprs)
|
||||
: Query(exprs), query_type(type), temporary(temporary_)
|
||||
{
|
||||
}
|
||||
|
||||
// static
|
||||
PtrTo<ExistsQuery> ExistsQuery::createTable(QueryType type, bool temporary, PtrTo<TableIdentifier> identifier)
|
||||
{
|
||||
return PtrTo<ExistsQuery>(new ExistsQuery(type, temporary, {identifier}));
|
||||
}
|
||||
|
||||
// static
|
||||
PtrTo<ExistsQuery> ExistsQuery::createDatabase(PtrTo<DatabaseIdentifier> identifier)
|
||||
{
|
||||
return PtrTo<ExistsQuery>(new ExistsQuery(QueryType::DATABASE, false, {identifier}));
|
||||
}
|
||||
|
||||
ASTPtr ExistsQuery::convertToOld() const
|
||||
{
|
||||
std::shared_ptr<ASTQueryWithTableAndOutput> query;
|
||||
|
||||
switch(query_type)
|
||||
{
|
||||
case QueryType::DATABASE:
|
||||
query = std::make_shared<ASTExistsDatabaseQuery>();
|
||||
tryGetIdentifierNameInto(get<DatabaseIdentifier>(IDENTIFIER)->convertToOld(), query->database);
|
||||
return query;
|
||||
|
||||
case QueryType::DICTIONARY:
|
||||
query = std::make_shared<ASTExistsDictionaryQuery>();
|
||||
break;
|
||||
case QueryType::TABLE:
|
||||
query = std::make_shared<ASTExistsTableQuery>();
|
||||
query->temporary = temporary;
|
||||
break;
|
||||
case QueryType::VIEW:
|
||||
query = std::make_shared<ASTExistsViewQuery>();
|
||||
break;
|
||||
}
|
||||
|
||||
// FIXME: this won't work if table doesn't exist
|
||||
auto table_id = getTableIdentifier(get(TABLE)->convertToOld());
|
||||
auto table_id = getTableIdentifier(get<TableIdentifier>(IDENTIFIER)->convertToOld());
|
||||
query->database = table_id.database_name;
|
||||
query->table = table_id.table_name;
|
||||
query->uuid = table_id.uuid;
|
||||
query->temporary = temporary;
|
||||
|
||||
return query;
|
||||
}
|
||||
@ -46,10 +66,22 @@ namespace DB
|
||||
|
||||
using namespace AST;
|
||||
|
||||
antlrcpp::Any ParseTreeVisitor::visitExistsStmt(ClickHouseParser::ExistsStmtContext *ctx)
|
||||
antlrcpp::Any ParseTreeVisitor::visitExistsTableStmt(ClickHouseParser::ExistsTableStmtContext *ctx)
|
||||
{
|
||||
auto type = ctx->TABLE() ? ExistsQuery::QueryType::TABLE : ExistsQuery::QueryType::DICTIONARY;
|
||||
return std::make_shared<ExistsQuery>(type, !!ctx->TEMPORARY(), visit(ctx->tableIdentifier()));
|
||||
ExistsQuery::QueryType type;
|
||||
if (ctx->DICTIONARY())
|
||||
type = ExistsQuery::QueryType::DICTIONARY;
|
||||
else if (ctx->VIEW())
|
||||
type = ExistsQuery::QueryType::VIEW;
|
||||
else // Query 'EXISTS <table_name>' is interptered as 'EXISTS TABLE <table_name>'
|
||||
type = ExistsQuery::QueryType::TABLE;
|
||||
|
||||
return ExistsQuery::createTable(type, !!ctx->TEMPORARY(), visit(ctx->tableIdentifier()));
|
||||
}
|
||||
|
||||
antlrcpp::Any ParseTreeVisitor::visitExistsDatabaseStmt(ClickHouseParser::ExistsDatabaseStmtContext *ctx)
|
||||
{
|
||||
return ExistsQuery::createDatabase(visit(ctx->databaseIdentifier()));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -13,16 +13,21 @@ class ExistsQuery : public Query
|
||||
{
|
||||
DICTIONARY,
|
||||
TABLE,
|
||||
VIEW,
|
||||
DATABASE,
|
||||
};
|
||||
|
||||
ExistsQuery(QueryType type, bool temporary, PtrTo<TableIdentifier> identifier);
|
||||
static PtrTo<ExistsQuery> createTable(QueryType type, bool temporary, PtrTo<TableIdentifier> identifier);
|
||||
static PtrTo<ExistsQuery> createDatabase(PtrTo<DatabaseIdentifier> identifier);
|
||||
|
||||
ExistsQuery(QueryType type, bool temporary, PtrList exprs);
|
||||
|
||||
ASTPtr convertToOld() const override;
|
||||
|
||||
private:
|
||||
enum ChildIndex : UInt8
|
||||
{
|
||||
TABLE = 0, // TableIdentifier
|
||||
IDENTIFIER = 0, // DatabaseIdentifier or TableIdentifier
|
||||
};
|
||||
|
||||
const QueryType query_type;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -173,12 +173,15 @@ describeStmt: (DESCRIBE | DESC) TABLE? tableExpr;
|
||||
|
||||
dropStmt
|
||||
: (DETACH | DROP) DATABASE (IF EXISTS)? databaseIdentifier clusterClause? # DropDatabaseStmt
|
||||
| (DETACH | DROP) (DICTIONARY | TEMPORARY? TABLE) (IF EXISTS)? tableIdentifier clusterClause? (NO DELAY)? # DropTableStmt
|
||||
| (DETACH | DROP) (DICTIONARY | TEMPORARY? TABLE | VIEW) (IF EXISTS)? tableIdentifier clusterClause? (NO DELAY)? # DropTableStmt
|
||||
;
|
||||
|
||||
// EXISTS statement
|
||||
|
||||
existsStmt: EXISTS (DICTIONARY | TEMPORARY? TABLE)? tableIdentifier;
|
||||
existsStmt
|
||||
: EXISTS DATABASE databaseIdentifier # ExistsDatabaseStmt
|
||||
| EXISTS (DICTIONARY | TEMPORARY? TABLE | VIEW)? tableIdentifier # ExistsTableStmt
|
||||
;
|
||||
|
||||
// EXPLAIN statement
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -164,7 +164,9 @@ public:
|
||||
|
||||
virtual antlrcpp::Any visitDropTableStmt(ClickHouseParser::DropTableStmtContext *context) = 0;
|
||||
|
||||
virtual antlrcpp::Any visitExistsStmt(ClickHouseParser::ExistsStmtContext *context) = 0;
|
||||
virtual antlrcpp::Any visitExistsDatabaseStmt(ClickHouseParser::ExistsDatabaseStmtContext *context) = 0;
|
||||
|
||||
virtual antlrcpp::Any visitExistsTableStmt(ClickHouseParser::ExistsTableStmtContext *context) = 0;
|
||||
|
||||
virtual antlrcpp::Any visitExplainStmt(ClickHouseParser::ExplainStmtContext *context) = 0;
|
||||
|
||||
|
||||
@ -136,7 +136,8 @@ public:
|
||||
antlrcpp::Any visitTtlExpr(ClickHouseParser::TtlExprContext * ctx) override;
|
||||
|
||||
// ExistsQuery
|
||||
antlrcpp::Any visitExistsStmt(ClickHouseParser::ExistsStmtContext * ctx) override;
|
||||
antlrcpp::Any visitExistsTableStmt(ClickHouseParser::ExistsTableStmtContext * ctx) override;
|
||||
antlrcpp::Any visitExistsDatabaseStmt(ClickHouseParser::ExistsDatabaseStmtContext * ctx) override;
|
||||
|
||||
// ExplainQuery
|
||||
antlrcpp::Any visitExplainStmt(ClickHouseParser::ExplainStmtContext * ctx) override;
|
||||
|
||||
@ -76,7 +76,6 @@ struct ASTDescribeQueryExistsQueryIDAndQueryNames
|
||||
static constexpr auto QueryTemporary = "DESCRIBE TEMPORARY TABLE";
|
||||
};
|
||||
|
||||
using ASTExistsDatabaseQuery = ASTQueryWithTableAndOutputImpl<ASTExistsDatabaseQueryIDAndQueryNames>;
|
||||
using ASTExistsTableQuery = ASTQueryWithTableAndOutputImpl<ASTExistsTableQueryIDAndQueryNames>;
|
||||
using ASTExistsViewQuery = ASTQueryWithTableAndOutputImpl<ASTExistsViewQueryIDAndQueryNames>;
|
||||
using ASTExistsDictionaryQuery = ASTQueryWithTableAndOutputImpl<ASTExistsDictionaryQueryIDAndQueryNames>;
|
||||
@ -84,6 +83,16 @@ using ASTShowCreateTableQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateTabl
|
||||
using ASTShowCreateViewQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateViewQueryIDAndQueryNames>;
|
||||
using ASTShowCreateDictionaryQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateDictionaryQueryIDAndQueryNames>;
|
||||
|
||||
class ASTExistsDatabaseQuery : public ASTQueryWithTableAndOutputImpl<ASTExistsDatabaseQueryIDAndQueryNames>
|
||||
{
|
||||
protected:
|
||||
void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << ASTExistsDatabaseQueryIDAndQueryNames::Query
|
||||
<< " " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database);
|
||||
}
|
||||
};
|
||||
|
||||
class ASTShowCreateDatabaseQuery : public ASTQueryWithTableAndOutputImpl<ASTShowCreateDatabaseQueryIDAndQueryNames>
|
||||
{
|
||||
protected:
|
||||
|
||||
@ -196,6 +196,7 @@ StorageS3::StorageS3(
|
||||
const String & format_name_,
|
||||
UInt64 min_upload_part_size_,
|
||||
UInt64 max_single_part_upload_size_,
|
||||
UInt64 max_connections_,
|
||||
const ColumnsDescription & columns_,
|
||||
const ConstraintsDescription & constraints_,
|
||||
const Context & context_,
|
||||
@ -220,8 +221,12 @@ StorageS3::StorageS3(
|
||||
if (access_key_id_.empty())
|
||||
credentials = Aws::Auth::AWSCredentials(std::move(settings.access_key_id), std::move(settings.secret_access_key));
|
||||
|
||||
Aws::Client::ClientConfiguration client_configuration;
|
||||
client_configuration.endpointOverride = uri_.endpoint;
|
||||
client_configuration.maxConnections = max_connections_;
|
||||
|
||||
client = S3::ClientFactory::instance().create(
|
||||
uri_.endpoint,
|
||||
client_configuration,
|
||||
uri_.is_virtual_hosted_style,
|
||||
credentials.GetAWSAccessKeyId(),
|
||||
credentials.GetAWSSecretKey(),
|
||||
@ -374,6 +379,7 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory)
|
||||
|
||||
UInt64 min_upload_part_size = args.local_context.getSettingsRef().s3_min_upload_part_size;
|
||||
UInt64 max_single_part_upload_size = args.local_context.getSettingsRef().s3_max_single_part_upload_size;
|
||||
UInt64 max_connections = args.local_context.getSettingsRef().s3_max_connections;
|
||||
|
||||
String compression_method;
|
||||
String format_name;
|
||||
@ -396,6 +402,7 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory)
|
||||
format_name,
|
||||
min_upload_part_size,
|
||||
max_single_part_upload_size,
|
||||
max_connections,
|
||||
args.columns,
|
||||
args.constraints,
|
||||
args.context,
|
||||
|
||||
@ -32,6 +32,7 @@ public:
|
||||
const String & format_name_,
|
||||
UInt64 min_upload_part_size_,
|
||||
UInt64 max_single_part_upload_size_,
|
||||
UInt64 max_connections_,
|
||||
const ColumnsDescription & columns_,
|
||||
const ConstraintsDescription & constraints_,
|
||||
const Context & context_,
|
||||
|
||||
@ -68,6 +68,7 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, const C
|
||||
S3::URI s3_uri (uri);
|
||||
UInt64 min_upload_part_size = context.getSettingsRef().s3_min_upload_part_size;
|
||||
UInt64 max_single_part_upload_size = context.getSettingsRef().s3_max_single_part_upload_size;
|
||||
UInt64 max_connections = context.getSettingsRef().s3_max_connections;
|
||||
|
||||
StoragePtr storage = StorageS3::create(
|
||||
s3_uri,
|
||||
@ -77,6 +78,7 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, const C
|
||||
format,
|
||||
min_upload_part_size,
|
||||
max_single_part_upload_size,
|
||||
max_connections,
|
||||
getActualTableStructure(context),
|
||||
ConstraintsDescription{},
|
||||
const_cast<Context &>(context),
|
||||
|
||||
@ -293,6 +293,8 @@ def run_tests_array(all_tests_with_params):
|
||||
try:
|
||||
sys.stdout.flush()
|
||||
sys.stdout.write("{0:72}".format(name + ": "))
|
||||
# This flush is needed so you can see the test name of the long running test before it will finish.
|
||||
sys.stdout.flush()
|
||||
|
||||
if args.skip and any(s in name for s in args.skip):
|
||||
print(MSG_SKIPPED + " - skip")
|
||||
|
||||
@ -28,8 +28,8 @@
|
||||
number,
|
||||
number
|
||||
FROM
|
||||
system.numbers
|
||||
LIMIT 100000000
|
||||
system.numbers_mt
|
||||
LIMIT 200000000
|
||||
</fill_query>
|
||||
<query>
|
||||
SELECT
|
||||
@ -41,7 +41,7 @@
|
||||
SETTINGS
|
||||
compile_expressions = 0;
|
||||
</query>
|
||||
<query short="1">
|
||||
<query>
|
||||
SELECT
|
||||
COUNT()
|
||||
FROM
|
||||
|
||||
@ -28,7 +28,7 @@ EXISTS t_01048; -- Does not work for temporary tables. Maybe have to fix.
|
||||
EXISTS TABLE t_01048;
|
||||
EXISTS DICTIONARY t_01048;
|
||||
|
||||
CREATE DICTIONARY db_01048.t_01048 (k UInt64, v String) PRIMARY KEY k LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://example.test/' FORMAT TSV)) LIFETIME(1000);
|
||||
CREATE DICTIONARY db_01048.t_01048 (k UInt64, v String) PRIMARY KEY k LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://example.test/' FORMAT 'TSV')) LIFETIME(1000);
|
||||
EXISTS db_01048.t_01048;
|
||||
EXISTS TABLE db_01048.t_01048; -- Dictionaries are tables as well. But not all tables are dictionaries.
|
||||
EXISTS DICTIONARY db_01048.t_01048;
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
202101
|
||||
202101_0_0_0 1 2021-01-01 some
|
||||
@ -0,0 +1,32 @@
|
||||
DROP TABLE IF EXISTS test_01640;
|
||||
DROP TABLE IF EXISTS restore_01640;
|
||||
|
||||
CREATE TABLE test_01640(i Int64, d Date, s String)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/{shard}/tables/test_01640','{replica}')
|
||||
PARTITION BY toYYYYMM(d) ORDER BY i;
|
||||
|
||||
insert into test_01640 values (1, '2021-01-01','some');
|
||||
|
||||
CREATE TABLE restore_01640(i Int64, d Date, s String)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/{shard}/tables/restore_01640','{replica}')
|
||||
PARTITION BY toYYYYMM(d) ORDER BY i;
|
||||
|
||||
ALTER TABLE restore_01640 FETCH PARTITION tuple(toYYYYMM(toDate('2021-01-01')))
|
||||
FROM '/clickhouse/{shard}/tables/test_01640';
|
||||
|
||||
SELECT partition_id
|
||||
FROM system.detached_parts
|
||||
WHERE (table = 'restore_01640') AND (database = currentDatabase());
|
||||
|
||||
ALTER TABLE restore_01640 ATTACH PARTITION tuple(toYYYYMM(toDate('2021-01-01')));
|
||||
|
||||
SELECT partition_id
|
||||
FROM system.detached_parts
|
||||
WHERE (table = 'restore_01640') AND (database = currentDatabase());
|
||||
|
||||
SELECT _part, * FROM restore_01640;
|
||||
|
||||
DROP TABLE test_01640;
|
||||
DROP TABLE restore_01640;
|
||||
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
@ -0,0 +1,4 @@
|
||||
SELECT formatDateTime(toDate('2010-01-04'), '%Q');
|
||||
SELECT formatDateTime(toDate('2010-04-30'), '%Q');
|
||||
SELECT formatDateTime(toDate('2010-07-31'), '%Q');
|
||||
SELECT formatDateTime(toDate('2010-10-07'), '%Q');
|
||||
@ -185,4 +185,5 @@
|
||||
01639_distributed_sync_insert_zero_rows
|
||||
01644_distributed_async_insert_fsync_smoke
|
||||
01552_impl_aggfunc_cloneresize
|
||||
01650_fetch_patition_with_macro_in_zk_path
|
||||
01651_bugs_from_15889
|
||||
|
||||
@ -162,7 +162,6 @@
|
||||
"01039_test_setting_parse",
|
||||
"01042_system_reload_dictionary_reloads_completely",
|
||||
"01045_dictionaries_restrictions",
|
||||
"01048_exists_query",
|
||||
"01055_compact_parts_1",
|
||||
"01056_create_table_as",
|
||||
"01066_bit_count",
|
||||
@ -197,7 +196,6 @@
|
||||
"01190_full_attach_syntax",
|
||||
"01191_rename_dictionary",
|
||||
"01192_rename_database_zookeeper",
|
||||
"01210_drop_view",
|
||||
"01213_alter_rename_column",
|
||||
"01232_untuple",
|
||||
"01244_optimize_distributed_group_by_sharding_key",
|
||||
@ -209,7 +207,6 @@
|
||||
"01269_create_with_null",
|
||||
"01271_show_privileges",
|
||||
"01272_offset_without_limit",
|
||||
"01275_parallel_mv",
|
||||
"01277_alter_rename_column_constraint_zookeeper",
|
||||
"01278_min_insert_block_size_rows_for_materialized_views",
|
||||
"01280_min_map_max_map",
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
v21.1.2.15-stable 2021-01-18
|
||||
v20.12.5.14-stable 2020-12-28
|
||||
v20.12.4.5-stable 2020-12-24
|
||||
v20.12.3.3-stable 2020-12-09
|
||||
|
||||
|
@ -78,7 +78,7 @@ def parse_one_pull_request(item):
|
||||
# This array gives the preferred category order, and is also used to
|
||||
# normalize category names.
|
||||
categories_preferred_order = ['Backward Incompatible Change',
|
||||
'New Feature', 'Bug Fix', 'Improvement', 'Performance Improvement',
|
||||
'New Feature', 'Performance Improvement', 'Improvement', 'Bug Fix',
|
||||
'Build/Testing/Packaging Improvement', 'Other']
|
||||
|
||||
category_to_pr = collections.defaultdict(lambda: [])
|
||||
|
||||
Loading…
Reference in New Issue
Block a user