Merge remote-tracking branch 'upstream/master' into improvement/multiply-decimals-and-floats

This commit is contained in:
myrrc 2020-12-18 15:42:01 +03:00
commit 05abfb04bd
320 changed files with 8094 additions and 1868 deletions

2
.gitmodules vendored
View File

@ -183,7 +183,7 @@
url = https://github.com/kthohr/stats.git
[submodule "contrib/krb5"]
path = contrib/krb5
url = https://github.com/krb5/krb5
url = https://github.com/ClickHouse-Extras/krb5
[submodule "contrib/cyrus-sasl"]
path = contrib/cyrus-sasl
url = https://github.com/cyrusimap/cyrus-sasl

View File

@ -6,6 +6,7 @@ set (SRCS
demangle.cpp
getFQDNOrHostName.cpp
getMemoryAmount.cpp
getPageSize.cpp
getThreadId.cpp
JSON.cpp
LineReader.cpp

View File

@ -1,4 +1,5 @@
#include <common/ReadlineLineReader.h>
#include <common/errnoToString.h>
#include <ext/scope_guard.h>
#include <errno.h>
@ -69,7 +70,7 @@ ReadlineLineReader::ReadlineLineReader(
{
int res = read_history(history_file_path.c_str());
if (res)
std::cerr << "Cannot read history from file " + history_file_path + ": "+ strerror(errno) << std::endl;
std::cerr << "Cannot read history from file " + history_file_path + ": "+ errnoToString(errno) << std::endl;
}
/// Added '.' to the default list. Because it is used to separate database and table.
@ -107,7 +108,7 @@ ReadlineLineReader::ReadlineLineReader(
};
if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + strerror(errno));
throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + errnoToString(errno));
rl_variable_bind("completion-ignore-case", "on");
// TODO: it doesn't work

View File

@ -47,7 +47,7 @@ ReplxxLineReader::ReplxxLineReader(
{
if (!rx.history_load(history_file_path))
{
rx.print("Loading history failed: %s\n", strerror(errno));
rx.print("Loading history failed: %s\n", errnoToString(errno).c_str());
}
if (flock(history_file_fd, LOCK_UN))
@ -88,7 +88,7 @@ ReplxxLineReader::ReplxxLineReader(
ReplxxLineReader::~ReplxxLineReader()
{
if (close(history_file_fd))
rx.print("Close of history file failed: %s\n", strerror(errno));
rx.print("Close of history file failed: %s\n", errnoToString(errno).c_str());
}
LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
@ -113,7 +113,7 @@ void ReplxxLineReader::addToHistory(const String & line)
// and that is why flock() is added here.
bool locked = false;
if (flock(history_file_fd, LOCK_EX))
rx.print("Lock of history file failed: %s\n", strerror(errno));
rx.print("Lock of history file failed: %s\n", errnoToString(errno).c_str());
else
locked = true;
@ -121,10 +121,10 @@ void ReplxxLineReader::addToHistory(const String & line)
// flush changes to the disk
if (!rx.history_save(history_file_path))
rx.print("Saving history failed: %s\n", strerror(errno));
rx.print("Saving history failed: %s\n", errnoToString(errno).c_str());
if (locked && 0 != flock(history_file_fd, LOCK_UN))
rx.print("Unlock of history file failed: %s\n", strerror(errno));
rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
}
void ReplxxLineReader::enableBracketedPaste()

View File

@ -1,5 +1,6 @@
#include <stdexcept>
#include "common/getMemoryAmount.h"
#include "common/getPageSize.h"
#include <unistd.h>
#include <sys/types.h>
@ -18,7 +19,7 @@ uint64_t getMemoryAmountOrZero()
if (num_pages <= 0)
return 0;
int64_t page_size = sysconf(_SC_PAGESIZE);
int64_t page_size = getPageSize();
if (page_size <= 0)
return 0;

View File

@ -0,0 +1,8 @@
#include "common/getPageSize.h"
#include <unistd.h>
Int64 getPageSize()
{
return sysconf(_SC_PAGESIZE);
}

View File

@ -0,0 +1,6 @@
#pragma once
#include "common/types.h"
/// Get memory page size
Int64 getPageSize();

View File

@ -1,6 +1,7 @@
// https://stackoverflow.com/questions/1413445/reading-a-password-from-stdcin
#include <common/setTerminalEcho.h>
#include <common/errnoToString.h>
#include <stdexcept>
#include <cstring>
#include <string>
@ -31,7 +32,7 @@ void setTerminalEcho(bool enable)
#else
struct termios tty;
if (tcgetattr(STDIN_FILENO, &tty))
throw std::runtime_error(std::string("setTerminalEcho failed get: ") + strerror(errno));
throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString(errno));
if (!enable)
tty.c_lflag &= ~ECHO;
else
@ -39,6 +40,6 @@ void setTerminalEcho(bool enable)
auto ret = tcsetattr(STDIN_FILENO, TCSANOW, &tty);
if (ret)
throw std::runtime_error(std::string("setTerminalEcho failed set: ") + strerror(errno));
throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString(errno));
#endif
}

View File

@ -47,6 +47,7 @@ SRCS(
errnoToString.cpp
getFQDNOrHostName.cpp
getMemoryAmount.cpp
getPageSize.cpp
getResource.cpp
getThreadId.cpp
mremap.cpp

2
contrib/boost vendored

@ -1 +1 @@
Subproject commit a7ceabe4747ecc3309dd3dcd9de4b29660dfd298
Subproject commit 0b98b443aa7bb77d65efd7b23b3b8c8a0ab5f1f3

2
contrib/krb5 vendored

@ -1 +1 @@
Subproject commit 99f7ad2831a01f264c07eed42a0a3a9336b86184
Subproject commit 90ff6f4f8c695d6bf1aaba78a9b8942be92141c2

@ -1 +1 @@
Subproject commit 92c74f938cf2c4dd529cae4f3d2923d153b029a7
Subproject commit a720b7105a610acbd7427eea475a5b6810c151eb

2
contrib/libgsasl vendored

@ -1 +1 @@
Subproject commit 140fb58250588c8323285b75fcf127c4adc33dfa
Subproject commit 383ee28e82f69fa16ed43b48bd9c8ee5b313ab84

2
contrib/libhdfs3 vendored

@ -1 +1 @@
Subproject commit 30552ac527f2c14070d834e171493b2e7f662375
Subproject commit 095b9d48b400abb72d967cb0539af13b1e3d90cf

View File

@ -17,7 +17,12 @@ if (NOT USE_INTERNAL_PROTOBUF_LIBRARY AND PROTOBUF_OLD_ABI_COMPAT)
endif ()
endif()
set(WITH_KERBEROS false)
if (${ENABLE_LIBRARIES} AND ${ENABLE_KRB5})
SET(WITH_KERBEROS 1)
else()
SET(WITH_KERBEROS 0)
endif()
# project and source dir
set(HDFS3_ROOT_DIR ${ClickHouse_SOURCE_DIR}/contrib/libhdfs3)
set(HDFS3_SOURCE_DIR ${HDFS3_ROOT_DIR}/src)
@ -28,11 +33,6 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
include(Platform)
include(Options)
# prefer shared libraries
if (WITH_KERBEROS)
find_package(KERBEROS REQUIRED)
endif()
# source
set(PROTO_FILES
#${HDFS3_SOURCE_DIR}/proto/encryption.proto
@ -207,14 +207,11 @@ target_include_directories(hdfs3 PRIVATE ${HDFS3_COMMON_DIR})
target_include_directories(hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR})
if (WITH_KERBEROS)
target_include_directories(hdfs3 PRIVATE ${KERBEROS_INCLUDE_DIRS})
endif()
target_include_directories(hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR})
target_link_libraries(hdfs3 PRIVATE ${LIBGSASL_LIBRARY})
if (WITH_KERBEROS)
target_link_libraries(hdfs3 PRIVATE ${KERBEROS_LIBRARIES})
target_link_libraries(hdfs3 PRIVATE ${KRB5_LIBRARY})
endif()
target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARIES})

2
debian/control vendored
View File

@ -40,7 +40,7 @@ Description: Common files for ClickHouse
Package: clickhouse-server
Architecture: all
Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-common-static (= ${binary:Version}), adduser
Recommends: libcap2-bin
Recommends: libcap2-bin, krb5-user
Replaces: clickhouse-server-common, clickhouse-server-base
Provides: clickhouse-server-common
Description: Server binary for ClickHouse

View File

@ -158,5 +158,9 @@
"name": "yandex/clickhouse-stateless-unbundled-test",
"dependent": [
]
},
"docker/test/integration/kerberized_hadoop": {
"name": "yandex/clickhouse-kerberized-hadoop",
"dependent": []
}
}

View File

@ -31,6 +31,7 @@ find . -name '*.so.*' -print -exec mv '{}' /output \;
if [ "performance" == "$COMBINED_OUTPUT" ]
then
cp -r ../tests/performance /output
cp -r ../tests/config/top_level_domains /
cp -r ../docker/test/performance-comparison/config /output ||:
rm /output/unit_tests_dbms ||:
rm /output/clickhouse-odbc-bridge ||:

View File

@ -0,0 +1,18 @@
# docker build -t yandex/clickhouse-kerberized-hadoop .
FROM sequenceiq/hadoop-docker:2.7.0
RUN sed -i -e 's/^\#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo
RUN sed -i -e 's/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Base.repo
RUN sed -i -e 's#http://mirror.centos.org/#http://vault.centos.org/#' /etc/yum.repos.d/CentOS-Base.repo
RUN yum clean all && \
rpm --rebuilddb && \
yum -y update && \
yum -y install yum-plugin-ovl && \
yum --quiet -y install krb5-workstation.x86_64
RUN cd /tmp && \
curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \
tar xzf commons-daemon-1.0.15-src.tar.gz && \
cd commons-daemon-1.0.15-src/src/native/unix && \
./configure && \
make && \
cp ./jsvc /usr/local/hadoop/sbin

View File

@ -29,6 +29,8 @@ RUN apt-get update \
libcurl4-openssl-dev \
gdb \
software-properties-common \
libkrb5-dev \
krb5-user \
&& rm -rf \
/var/lib/apt/lists/* \
/var/cache/debconf \
@ -75,7 +77,8 @@ RUN python3 -m pip install \
pytest-timeout \
redis \
tzlocal \
urllib3
urllib3 \
requests-kerberos
COPY modprobe.sh /usr/local/bin/modprobe
COPY dockerd-entrypoint.sh /usr/local/bin/

View File

@ -2,6 +2,7 @@ version: '2.3'
services:
hdfs1:
image: sequenceiq/hadoop-docker:2.7.0
hostname: hdfs1
restart: always
ports:
- 50075:50075

View File

@ -0,0 +1,29 @@
version: '2.3'
services:
kerberizedhdfs1:
cap_add:
- DAC_READ_SEARCH
image: yandex/clickhouse-kerberized-hadoop:16621
hostname: kerberizedhdfs1
restart: always
volumes:
- ${KERBERIZED_HDFS_DIR}/../../hdfs_configs/bootstrap.sh:/etc/bootstrap.sh:ro
- ${KERBERIZED_HDFS_DIR}/secrets:/usr/local/hadoop/etc/hadoop/conf
- ${KERBERIZED_HDFS_DIR}/secrets/krb_long.conf:/etc/krb5.conf:ro
ports:
- 1006:1006
- 50070:50070
- 9000:9000
depends_on:
- hdfskerberos
entrypoint: /etc/bootstrap.sh -d
hdfskerberos:
image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG}
hostname: hdfskerberos
volumes:
- ${KERBERIZED_HDFS_DIR}/secrets:/tmp/keytab
- ${KERBERIZED_HDFS_DIR}/../../kerberos_image_config.sh:/config.sh
- /dev/urandom:/dev/random
ports: [88, 749]

View File

@ -108,6 +108,95 @@ Create table with files named `file000`, `file001`, … , `file999`:
``` sql
CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV')
```
## Configuration {#configuration}
Similar to GraphiteMergeTree, the HDFS engine supports extended configuration using the ClickHouse config file. There are two configuration keys that you can use: global (`hdfs`) and user-level (`hdfs_*`). The global configuration is applied first, and then the user-level configuration is applied (if it exists).
``` xml
<!-- Global configuration options for HDFS engine type -->
<hdfs>
<hadoop_kerberos_keytab>/tmp/keytab/clickhouse.keytab</hadoop_kerberos_keytab>
<hadoop_kerberos_principal>clickuser@TEST.CLICKHOUSE.TECH</hadoop_kerberos_principal>
<hadoop_security_authentication>kerberos</hadoop_security_authentication>
</hdfs>
<!-- Configuration specific for user "root" -->
<hdfs_root>
<hadoop_kerberos_principal>root@TEST.CLICKHOUSE.TECH</hadoop_kerberos_principal>
</hdfs_root>
```
### List of possible configuration options with default values
#### Supported by libhdfs3
| **parameter** | **default value** |
| rpc\_client\_connect\_tcpnodelay | true |
| dfs\_client\_read\_shortcircuit | true |
| output\_replace-datanode-on-failure | true |
| input\_notretry-another-node | false |
| input\_localread\_mappedfile | true |
| dfs\_client\_use\_legacy\_blockreader\_local | false |
| rpc\_client\_ping\_interval | 10 * 1000 |
| rpc\_client\_connect\_timeout | 600 * 1000 |
| rpc\_client\_read\_timeout | 3600 * 1000 |
| rpc\_client\_write\_timeout | 3600 * 1000 |
| rpc\_client\_socekt\_linger\_timeout | -1 |
| rpc\_client\_connect\_retry | 10 |
| rpc\_client\_timeout | 3600 * 1000 |
| dfs\_default\_replica | 3 |
| input\_connect\_timeout | 600 * 1000 |
| input\_read\_timeout | 3600 * 1000 |
| input\_write\_timeout | 3600 * 1000 |
| input\_localread\_default\_buffersize | 1 * 1024 * 1024 |
| dfs\_prefetchsize | 10 |
| input\_read\_getblockinfo\_retry | 3 |
| input\_localread\_blockinfo\_cachesize | 1000 |
| input\_read\_max\_retry | 60 |
| output\_default\_chunksize | 512 |
| output\_default\_packetsize | 64 * 1024 |
| output\_default\_write\_retry | 10 |
| output\_connect\_timeout | 600 * 1000 |
| output\_read\_timeout | 3600 * 1000 |
| output\_write\_timeout | 3600 * 1000 |
| output\_close\_timeout | 3600 * 1000 |
| output\_packetpool\_size | 1024 |
| output\_heeartbeat\_interval | 10 * 1000 |
| dfs\_client\_failover\_max\_attempts | 15 |
| dfs\_client\_read\_shortcircuit\_streams\_cache\_size | 256 |
| dfs\_client\_socketcache\_expiryMsec | 3000 |
| dfs\_client\_socketcache\_capacity | 16 |
| dfs\_default\_blocksize | 64 * 1024 * 1024 |
| dfs\_default\_uri | "hdfs://localhost:9000" |
| hadoop\_security\_authentication | "simple" |
| hadoop\_security\_kerberos\_ticket\_cache\_path | "" |
| dfs\_client\_log\_severity | "INFO" |
| dfs\_domain\_socket\_path | "" |
[HDFS Configuration Reference ](https://hawq.apache.org/docs/userguide/2.3.0.0-incubating/reference/HDFSConfigurationParameterReference.html) might explain some parameters.
#### ClickHouse extras {#clickhouse-extras}
| **parameter** | **default value** |
|hadoop\_kerberos\_keytab | "" |
|hadoop\_kerberos\_principal | "" |
|hadoop\_kerberos\_kinit\_command | kinit |
#### Limitations {#limitations}
* hadoop\_security\_kerberos\_ticket\_cache\_path can be global only, not user specific
## Kerberos support {#kerberos-support}
If hadoop\_security\_authentication parameter has value 'kerberos', ClickHouse authentifies via Kerberos facility.
Parameters [here](#clickhouse-extras) and hadoop\_security\_kerberos\_ticket\_cache\_path may be of help.
Note that due to libhdfs3 limitations only old-fashioned approach is supported,
datanode communications are not secured by SASL (HADOOP\_SECURE\_DN\_USER is a reliable indicator of such
security approach). Use tests/integration/test\_storage\_kerberized\_hdfs/hdfs_configs/bootstrap.sh for reference.
If hadoop\_kerberos\_keytab, hadoop\_kerberos\_principal or hadoop\_kerberos\_kinit\_command is specified, kinit will be invoked. hadoop\_kerberos\_keytab and hadoop\_kerberos\_principal are mandatory in this case. kinit tool and krb5 configuration files are required.
## Virtual Columns {#virtual-columns}

View File

@ -57,7 +57,7 @@ The command line is based on replxx (similar to readline). In other
By default, the format used is PrettyCompact. You can change the format in the FORMAT clause of the query, or by specifying `\G` at the end of the query, using the `--format` or `--vertical` argument in the command line, or using the client configuration file.
To exit the client, press Ctrl+D (or Ctrl+C), or enter one of the following instead of a query: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q”
To exit the client, press Ctrl+D, or enter one of the following instead of a query: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q”
When processing a query, the client shows:

View File

@ -39,7 +39,7 @@ data_compressed_bytes: 499
last_exception:
```
**See also**
**See Also**
- [Distributed table engine](../../engines/table-engines/special/distributed.md)

View File

@ -0,0 +1,81 @@
# system.replication_queue {#system_tables-replication_queue}
Contains information about tasks from replication queues stored in ZooKeeper for tables in the `ReplicatedMergeTree` family.
Columns:
- `database` ([String](../../sql-reference/data-types/string.md)) — Name of the database.
- `table` ([String](../../sql-reference/data-types/string.md)) — Name of the table.
- `replica_name` ([String](../../sql-reference/data-types/string.md)) — Replica name in ZooKeeper. Different replicas of the same table have different names.
- `position` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Position of the task in the queue.
- `node_name` ([String](../../sql-reference/data-types/string.md)) — Node name in ZooKeeper.
- `type` ([String](../../sql-reference/data-types/string.md)) — Type of the task in the queue: `GET_PARTS`, `MERGE_PARTS`, `DETACH_PARTS`, `DROP_PARTS`, or `MUTATE_PARTS`.
- `create_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — Date and time when the task was submitted for execution.
- `required_quorum` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of replicas waiting for the task to complete with confirmation of completion. This column is only relevant for the `GET_PARTS` task.
- `source_replica` ([String](../../sql-reference/data-types/string.md)) — Name of the source replica.
- `new_part_name` ([String](../../sql-reference/data-types/string.md)) — Name of the new part.
- `parts_to_merge` ([Array](../../sql-reference/data-types/array.md) ([String](../../sql-reference/data-types/string.md))) — Names of parts to merge or update.
- `is_detach` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The flag indicates whether the `DETACH_PARTS` task is in the queue.
- `is_currently_executing` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The flag indicates whether a specific task is being performed right now.
- `num_tries` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of failed attempts to complete the task.
- `last_exception` ([String](../../sql-reference/data-types/string.md)) — Text message about the last error that occurred (if any).
- `last_attempt_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — Date and time when the task was last attempted.
- `num_postponed` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of postponed tasks.
- `postpone_reason` ([String](../../sql-reference/data-types/string.md)) — The reason why the task was postponed.
- `last_postpone_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — Date and time when the task was last postponed.
- `merge_type` ([String](../../sql-reference/data-types/string.md)) — Type of the current merge. Empty if it's a mutation.
**Example**
``` sql
SELECT * FROM system.replication_queue LIMIT 1 FORMAT Vertical;
```
``` text
Row 1:
──────
database: merge
table: visits_v2
replica_name: mtgiga001-1t.metrika.yandex.net
position: 15
node_name: queue-0009325559
type: MERGE_PARTS
create_time: 2020-12-07 14:04:21
required_quorum: 0
source_replica: mtgiga001-1t.metrika.yandex.net
new_part_name: 20201130_121373_121384_2
parts_to_merge: ['20201130_121373_121378_1','20201130_121379_121379_0','20201130_121380_121380_0','20201130_121381_121381_0','20201130_121382_121382_0','20201130_121383_121383_0','20201130_121384_121384_0']
is_detach: 0
is_currently_executing: 0
num_tries: 36
last_exception: Code: 226, e.displayText() = DB::Exception: Marks file '/opt/clickhouse/data/merge/visits_v2/tmp_fetch_20201130_121373_121384_2/CounterID.mrk' doesn't exist (version 20.8.7.15 (official build))
last_attempt_time: 2020-12-08 17:35:54
num_postponed: 0
postpone_reason:
last_postpone_time: 1970-01-01 03:00:00
```
**See Also**
- [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md/#query-language-system-replicated)
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/replication_queue) <!--hide-->

View File

@ -25,6 +25,10 @@ Example 2: `uniqArray(arr)` Counts the number of unique elements in all a
-If and -Array can be combined. However, Array must come first, then If. Examples: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. Due to this order, the cond argument wont be an array.
## -SimpleState {#agg-functions-combinator-simplestate}
If you apply this combinator, the aggregate function returns the same value but with a different type. This is an `SimpleAggregateFunction(...)` that can be stored in a table to work with [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) table engines.
## -State {#agg-functions-combinator-state}
If you apply this combinator, the aggregate function doesnt return the resulting value (such as the number of unique values for the [uniq](../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function), but an intermediate state of the aggregation (for `uniq`, this is the hash table for calculating the number of unique values). This is an `AggregateFunction(...)` that can be used for further processing or stored in a table to finish aggregating later.

View File

@ -1288,12 +1288,30 @@ Returns the index of the first element in the `arr1` array for which `func` retu
Note that the `arrayFirstIndex` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it cant be omitted.
## arrayMin(\[func,\] arr1, …) {#array-min}
Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements.
Note that the `arrayMin` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
## arrayMax(\[func,\] arr1, …) {#array-max}
Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements.
Note that the `arrayMax` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
## arraySum(\[func,\] arr1, …) {#array-sum}
Returns the sum of the `func` values. If the function is omitted, it just returns the sum of the array elements.
Note that the `arraySum` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
## arrayAvg(\[func,\] arr1, …) {#array-avg}
Returns the sum of the `func` values. If the function is omitted, it just returns the average of the array elements.
Note that the `arrayAvg` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
## arrayCumSum(\[func,\] arr1, …) {#arraycumsumfunc-arr1}
Returns an array of partial sums of elements in the source array (a running sum). If the `func` function is specified, then the values of the array elements are converted by this function before summing.

View File

@ -430,6 +430,63 @@ SELECT toTypeName(CAST(x, 'Nullable(UInt16)')) FROM t_null
- [cast_keep_nullable](../../operations/settings/settings.md#cast_keep_nullable) setting
## accurateCast(x, T) {#type_conversion_function-accurate-cast}
Converts x to the t data type. The differente from cast(x, T) is that accurateCast
does not allow overflow of numeric types during cast if type value x does not fit
bounds of type T.
Example
``` sql
SELECT cast(-1, 'UInt8') as uint8;
```
``` text
┌─uint8─┐
│ 255 │
└───────┘
```
```sql
SELECT accurateCast(-1, 'UInt8') as uint8;
```
``` text
Code: 70. DB::Exception: Received from localhost:9000. DB::Exception: Value in column Int8 cannot be safely converted into type UInt8: While processing accurateCast(-1, 'UInt8') AS uint8.
```
## accurateCastOrNull(x, T) {#type_conversion_function-accurate-cast_or_null}
Converts x to the t data type. Always returns nullable type and returns NULL
if the casted value is not representable in the target type.
Example:
``` sql
SELECT
accurateCastOrNull(-1, 'UInt8') as uint8,
accurateCastOrNull(128, 'Int8') as int8,
accurateCastOrNull('Test', 'FixedString(2)') as fixed_string
```
``` text
┌─uint8─┬─int8─┬─fixed_string─┐
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │
└───────┴──────┴──────────────┘┘
```
``` sql
SELECT toTypeName(accurateCastOrNull(5, 'UInt8'))
```
``` text
┌─toTypeName(accurateCastOrNull(5, 'UInt8'))─┐
│ Nullable(UInt8) │
└────────────────────────────────────────────┘
```
## toInterval(Year\|Quarter\|Month\|Week\|Day\|Hour\|Minute\|Second) {#function-tointerval}
Converts a Number type argument to an [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type.

View File

@ -204,7 +204,7 @@ SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
## Managing ReplicatedMergeTree Tables {#query-language-system-replicated}
ClickHouse can manage background replication related processes in [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication/#table_engines-replication) tables.
ClickHouse can manage background replication related processes in [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md#table_engines-replication) tables.
### STOP FETCHES {#query_language-system-stop-fetches}

View File

@ -59,7 +59,7 @@ La línea de comandos se basa en replxx (similar a readline). En otr
De forma predeterminada, el formato utilizado es PrettyCompact. Puede cambiar el formato en la cláusula FORMAT de la consulta o especificando `\G` al final de la consulta, utilizando el `--format` o `--vertical` en la línea de comandos, o utilizando el archivo de configuración del cliente.
Para salir del cliente, presione Ctrl+D (o Ctrl+C) o introduzca una de las siguientes opciones en lugar de una consulta: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q”
Para salir del cliente, presione Ctrl+D o introduzca una de las siguientes opciones en lugar de una consulta: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q”
Al procesar una consulta, el cliente muestra:

View File

@ -59,7 +59,7 @@ $ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FOR
既定では、使用される形式はPrettyCompactです。 クエリのFORMAT句で書式を変更するか、次のように指定することができます `\G` クエリの最後に、 `--format` または `--vertical` コマンドラインでの引数、またはクライアント構成ファイルの使用。
クライアントを終了するには、Ctrl+D(またはCtrl+C)を押すか、クエリの代わりに次のいずれかを入力します: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q”
クライアントを終了するには、Ctrl+Dを押すか、クエリの代わりに次のいずれかを入力します: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q”
が処理クエリー、クライアントを示し:

View File

@ -0,0 +1,25 @@
---
title: General questions about ClickHouse
toc_hidden_folder: true
toc_priority: 1
toc_title: Общие вопросы
---
# Общие вопросы о ClickHouse {#obshchie-voprosy}
Вопросы:
- Что такое ClickHouse?
- Почему ClickHouse такой быстрый?
- Кто пользуется ClickHouse?
- Что обозначает название “ClickHouse”?
- Что значит “Не тормозит”?
- Что такое OLAP?
- Что такое колоночная база данных?
- [Почему бы не использовать системы типа MapReduce?](mapreduce.md)
!!! info "Если вы не нашли то, что искали:"
Загляните в другие категории F.A.Q. или поищите в других разделах документации, ориентируйтесь по оглавлению слева.
{## [Original article](https://clickhouse.tech/docs/ru/faq/general/) ##}

View File

@ -1,8 +1,12 @@
# Общие вопросы {#obshchie-voprosy}
---
title: Why not use something like MapReduce?
toc_hidden: true
toc_priority: 110
---
## Почему бы не использовать системы типа MapReduce? {#pochemu-by-ne-ispolzovat-sistemy-tipa-mapreduce}
Системами типа MapReduce будем называть системы распределённых вычислений, в которых операция reduce сделана на основе распределённой сортировки. Наиболее распространённым opensource решением данного класса является [Apache Hadoop](http://hadoop.apache.org). Яндекс использует собственное решение — YT.
Системами типа MapReduce будем называть системы распределённых вычислений, в которых операция reduce сделана на основе распределённой сортировки. Наиболее распространённым opensource решением данного класса является [Apache Hadoop](http://hadoop.apache.org). Яндекс использует собственное решение — YT.
Такие системы не подходят для онлайн запросов в силу слишком большой latency. То есть, не могут быть использованы в качестве бэкенда для веб-интерфейса.
Такие системы не подходят для обновления данных в реальном времени.
@ -10,47 +14,3 @@
Распределённая сортировка является основной причиной тормозов при выполнении несложных map-reduce задач.
Большинство реализаций MapReduce позволяют выполнять произвольный код на кластере. Но для OLAP задач лучше подходит декларативный язык запросов, который позволяет быстро проводить исследования. Для примера, для Hadoop существует Hive и Pig. Также смотрите Cloudera Impala, Shark (устаревший) для Spark, а также Spark SQL, Presto, Apache Drill. Впрочем, производительность при выполнении таких задач является сильно неоптимальной по сравнению со специализированными системами, а сравнительно высокая latency не позволяет использовать эти системы в качестве бэкенда для веб-интерфейса.
## Что делать, если у меня проблема с кодировками при использовании Oracle через ODBC? {#oracle-odbc-encodings}
Если вы используете Oracle через драйвер ODBC в качестве источника внешних словарей, необходимо задать правильное значение для переменной окружения `NLS_LANG` в `/etc/default/clickhouse`. Подробнее читайте в [Oracle NLS_LANG FAQ](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html).
**Пример**
``` sql
NLS_LANG=RUSSIAN_RUSSIA.UTF8
```
## Как экспортировать данные из ClickHouse в файл? {#how-to-export-to-file}
### Секция INTO OUTFILE {#sektsiia-into-outfile}
Добавьте секцию [INTO OUTFILE](../sql-reference/statements/select/into-outfile.md#into-outfile-clause) к своему запросу.
Например:
``` sql
SELECT * FROM table INTO OUTFILE 'file'
```
По умолчанию, для выдачи данных ClickHouse использует формат [TabSeparated](../interfaces/formats.md#tabseparated). Чтобы выбрать [формат данных](../interfaces/formats.md), используйте [секцию FORMAT](../sql-reference/statements/select/format.md#format-clause).
Например:
``` sql
SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV
```
### Таблица с движком File {#tablitsa-s-dvizhkom-file}
Смотрите [File](../engines/table-engines/special/file.md).
### Перенаправление в командой строке {#perenapravlenie-v-komandoi-stroke}
``` sql
$ clickhouse-client --query "SELECT * from table" --format FormatName > result.txt
```
Смотрите [clickhouse-client](../interfaces/cli.md).
[Оригинальная статья](https://clickhouse.tech/docs/en/faq/general/) <!--hide-->

View File

@ -4,3 +4,14 @@ toc_hidden: true
toc_priority: 76
---
# Содержание F.A.Q. {#soderzhanie}
В этом разделе документации собрали вопросы о ClickHouse, которые задают чаще всего.
Категории:
- **[Общие вопросы](../faq/general/index.md)**
- **[Применение](../faq/use-cases/index.md)**
- **[Операции](../faq/operations/index.md)**
- **[Интеграция](../faq/integration/index.md)**

View File

@ -0,0 +1,37 @@
---
title: How do I export data from ClickHouse to a file?
toc_hidden: true
toc_priority: 10
---
## Как экспортировать данные из ClickHouse в файл? {#how-to-export-to-file-rus}
### Секция INTO OUTFILE {#sektsiia-into-outfile-rus}
Добавьте секцию [INTO OUTFILE](../../sql-reference/statements/select/into-outfile.md#into-outfile-clause) к своему запросу.
Например:
``` sql
SELECT * FROM table INTO OUTFILE 'file'
```
По умолчанию, для выдачи данных ClickHouse использует формат [TabSeparated](../../interfaces/formats.md#tabseparated). Чтобы выбрать [формат данных](../../interfaces/formats.md), используйте секцию [FORMAT](../../sql-reference/statements/select/format.md#format-clause).
Например:
``` sql
SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV
```
## Таблица с движком File {#using-a-file-engine-table}
Смотрите [File](../../engines/table-engines/special/file.md).
## Перенаправление в командой строке {#using-command-line-redirection}
``` bash
$ clickhouse-client --query "SELECT * from table" --format FormatName > result.txt
```
Смотрите [clickhouse-client](../../interfaces/cli.md).

View File

@ -0,0 +1,19 @@
---
title: Questions about integrating ClickHouse and other systems
toc_hidden_folder: true
toc_priority: 4
toc_title: Интеграция
---
# Вопросы об интеграции ClickHouse с другими системами {#question-about-integrating-clickhouse-and-other-systems-rus}
Вопросы:
- [Как экспортировать данные из ClickHouse в файл?](file-export.md)
- Как импортировать JSON в ClickHouse?
- [Что делать, если у меня проблема с кодировками при использовании Oracle через ODBC?](oracle-odbc.md)
!!! info "Если вы не нашли то, что искали"
Загляните в другие подразделы F.A.Q. или поищите в остальных разделах документации, ориентируйтесь по оглавлению слева.
{## [Original article](https://clickhouse.tech/docs/ru/faq/integration/) ##}

View File

@ -0,0 +1,15 @@
---
title: What if I have a problem with encodings when using Oracle via ODBC?
toc_hidden: true
toc_priority: 20
---
## Что делать, если у меня проблема с кодировками при использовании Oracle через ODBC? {#oracle-odbc-encodings-rus}
Если вы используете Oracle через драйвер ODBC в качестве источника внешних словарей, необходимо задать правильное значение для переменной окружения `NLS_LANG` в `/etc/default/clickhouse`. Подробнее читайте в [Oracle NLS_LANG FAQ](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html).
**Пример**
``` sql
NLS_LANG=RUSSIAN_RUSSIA.UTF8
```

View File

@ -0,0 +1,18 @@
---
title: Question about operating ClickHouse servers and clusters
toc_hidden_folder: true
toc_priority: 3
toc_title: Операции
---
# Вопросы о производительности серверов и кластеров ClickHouse {#voprosy-ob-operating-clickhouse-servers-and-clusters}
Вопросы:
- Which ClickHouse version to use in production?
- Is it possible to delete old records from a ClickHouse table?
!!! info "Dont see what you were looking for?"
Check out [other F.A.Q. categories](../../faq/index.md) or browse around main documentation articles found in the left sidebar.
{## [Original article](https://clickhouse.tech/docs/en/faq/production/) ##}

View File

@ -0,0 +1,14 @@
---
title: Questions about ClickHouse use cases
toc_hidden_folder: true
toc_priority: 2
toc_title: Применение
---
# Вопросы о применении ClickHouse {#voprosy-o-primenenii}
Вопросы:
- Can I use ClickHouse as a time-series database?
- Can I use ClickHouse as a key-value storage?

View File

@ -63,7 +63,7 @@ $ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FOR
По умолчанию, в качестве формата, используется формат PrettyCompact (красивые таблички). Вы можете изменить формат с помощью секции FORMAT запроса, или с помощью указания `\G` на конце запроса, с помощью аргумента командной строки `--format` или `--vertical`, или с помощью конфигурационного файла клиента.
Чтобы выйти из клиента, нажмите Ctrl+D (или Ctrl+C), или наберите вместо запроса одно из: «exit», «quit», «logout», «учше», «йгше», «дщпщге», «exit;», «quit;», «logout;», «учшеж», «йгшеж», «дщпщгеж», «q», «й», «q», «Q», «:q», «й», «Й», «Жй»
Чтобы выйти из клиента, нажмите Ctrl+D, или наберите вместо запроса одно из: «exit», «quit», «logout», «учше», «йгше», «дщпщге», «exit;», «quit;», «logout;», «учшеж», «йгшеж», «дщпщгеж», «q», «й», «q», «Q», «:q», «й», «Й», «Жй»
При выполнении запроса, клиент показывает:

View File

@ -0,0 +1,81 @@
# system.replication_queue {#system_tables-replication_queue}
Содержит информацию о задачах из очередей репликации, хранящихся в ZooKeeper, для таблиц семейства `ReplicatedMergeTree`.
Столбцы:
- `database` ([String](../../sql-reference/data-types/string.md)) — имя базы данных.
- `table` ([String](../../sql-reference/data-types/string.md)) — имя таблицы.
- `replica_name` ([String](../../sql-reference/data-types/string.md)) — имя реплики в ZooKeeper. Разные реплики одной и той же таблицы имеют различные имена.
- `position` ([UInt32](../../sql-reference/data-types/int-uint.md)) — позиция задачи в очереди.
- `node_name` ([String](../../sql-reference/data-types/string.md)) — имя узла в ZooKeeper.
- `type` ([String](../../sql-reference/data-types/string.md)) — тип задачи в очереди: `GET_PARTS`, `MERGE_PARTS`, `DETACH_PARTS`, `DROP_PARTS` или `MUTATE_PARTS`.
- `create_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — дата и время отправки задачи на выполнение.
- `required_quorum` ([UInt32](../../sql-reference/data-types/int-uint.md)) — количество реплик, ожидающих завершения задачи, с подтверждением о завершении. Этот столбец актуален только для задачи `GET_PARTS`.
- `source_replica` ([String](../../sql-reference/data-types/string.md)) — имя исходной реплики.
- `new_part_name` ([String](../../sql-reference/data-types/string.md)) — имя нового куска.
- `parts_to_merge` ([Array](../../sql-reference/data-types/array.md) ([String](../../sql-reference/data-types/string.md))) — имена кусков, которые требуется смержить или обновить.
- `is_detach` ([UInt8](../../sql-reference/data-types/int-uint.md)) — флаг, указывающий на присутствие в очереди задачи `DETACH_PARTS`.
- `is_currently_executing` ([UInt8](../../sql-reference/data-types/int-uint.md)) — флаг, указывающий на выполнение конкретной задачи на данный момент.
- `num_tries` ([UInt32](../../sql-reference/data-types/int-uint.md)) — количество неудачных попыток выполнить задачу.
- `last_exception` ([String](../../sql-reference/data-types/string.md)) — текст сообщения о последней возникшей ошибке, если таковые имеются.
- `last_attempt_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — дата и время последней попытки выполнить задачу.
- `num_postponed` ([UInt32](../../sql-reference/data-types/int-uint.md)) — количество отложенных задач.
- `postpone_reason` ([String](../../sql-reference/data-types/string.md)) — причина, по которой была отложена задача.
- `last_postpone_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — дата и время, когда была отложена задача в последний раз.
- `merge_type` ([String](../../sql-reference/data-types/string.md)) — тип текущего слияния. Пусто, если это мутация.
**Пример**
``` sql
SELECT * FROM system.replication_queue LIMIT 1 FORMAT Vertical;
```
``` text
Row 1:
──────
database: merge
table: visits_v2
replica_name: mtgiga001-1t.metrika.yandex.net
position: 15
node_name: queue-0009325559
type: MERGE_PARTS
create_time: 2020-12-07 14:04:21
required_quorum: 0
source_replica: mtgiga001-1t.metrika.yandex.net
new_part_name: 20201130_121373_121384_2
parts_to_merge: ['20201130_121373_121378_1','20201130_121379_121379_0','20201130_121380_121380_0','20201130_121381_121381_0','20201130_121382_121382_0','20201130_121383_121383_0','20201130_121384_121384_0']
is_detach: 0
is_currently_executing: 0
num_tries: 36
last_exception: Code: 226, e.displayText() = DB::Exception: Marks file '/opt/clickhouse/data/merge/visits_v2/tmp_fetch_20201130_121373_121384_2/CounterID.mrk' doesn't exist (version 20.8.7.15 (official build))
last_attempt_time: 2020-12-08 17:35:54
num_postponed: 0
postpone_reason:
last_postpone_time: 1970-01-01 03:00:00
```
**Смотрите также**
- [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system.md/#query-language-system-replicated)
[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/replication_queue) <!--hide-->

View File

@ -199,7 +199,7 @@ SOURCE(ODBC(
ClickHouse получает от ODBC-драйвера информацию о квотировании и квотирует настройки в запросах к драйверу, поэтому имя таблицы нужно указывать в соответствии с регистром имени таблицы в базе данных.
Если у вас есть проблемы с кодировками при использовании Oracle, ознакомьтесь с соответствующим разделом [FAQ](../../../faq/general.md#oracle-odbc-encodings).
Если у вас есть проблемы с кодировками при использовании Oracle, ознакомьтесь с соответствующим разделом [FAQ](../../../faq/integration/oracle-odbc.md).
### Выявленная уязвимость в функционировании ODBC словарей {#vyiavlennaia-uiazvimost-v-funktsionirovanii-odbc-slovarei}

View File

@ -59,7 +59,7 @@ Komut satırı dayanmaktadır replxx (benzer readline). Başka bir d
Varsayılan olarak, kullanılan biçim PrettyCompact. Sorgunun biçim yan tümcesinde veya belirterek biçimi değiştirebilirsiniz `\G` sorgunun sonunda, `--format` veya `--vertical` komut satırında veya istemci yapılandırma dosyasını kullanarak bağımsız değişken.
İstemciden çıkmak için Ctrl+D (veya Ctrl+C) tuşlarına basın veya bir sorgu yerine aşağıdakilerden birini girin: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q”
İstemciden çıkmak için Ctrl+D tuşlarına basın veya bir sorgu yerine aşağıdakilerden birini girin: “exit”, “quit”, “logout”, “exit;”, “quit;”, “logout;”, “q”, “Q”, “:q”
Bir sorguyu işlerken, istemci şunları gösterir:

View File

@ -9,19 +9,21 @@ toc_title: "\u5176\u4ED6"
## ATTACH {#attach}
这个查询是完全一样的 `CREATE`,但是
与`CREATE`类似,但有所区别
- 而不是这个词 `CREATE` 它使用这个词 `ATTACH`.
- 查询不会在磁盘上创建数据,但假定数据已经在适当的位置,只是将有关表的信息添加到服务器。
执行附加查询后,服务器将知道表的存在
- 使用关键词 `ATTACH`
- 查询不会在磁盘上创建数据。但会假定数据已经在对应位置存放,同时将与表相关的信息添加到服务器。
执行 `ATTACH` 查询后,服务器将知道表已经被创建
如果表之前已分离 (`DETACH`),意味着其结构是已知的,可以使用速记而不限定该结构
如果表之前已分离 (`DETACH`),意味着其结构是已知的,可以使用简要的写法来建立表即不需要定义表结构的Schema细节
``` sql
ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster]
```
启动服务器时使用此查询。 服务器将表元数据作为文件存储 `ATTACH` 查询,它只是在启动时运行(除了在服务器上显式创建的系统表)。
启动服务器时会自动触发此查询。
服务器将表的元数据作为文件存储 `ATTACH` 查询,它只是在启动时运行。有些表例外,如系统表,它们是在服务器上显式指定的。
## CHECK TABLE {#check-table}
@ -31,13 +33,12 @@ ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster]
CHECK TABLE [db.]name
```
`CHECK TABLE` 查询将实际文件大小与存储在服务器上的预期值进行比较。 如果文件大小与存储的值不匹配,则表示数据已损坏。 例如,这可能是由查询执行期间的系统崩溃引起的。
`CHECK TABLE` 查询会比较存储在服务器上的实际文件大小与预期值。 如果文件大小与存储的值不匹配,则表示数据已损坏。 例如,这可能是由查询执行期间的系统崩溃引起的。
查询响应包含 `result` 具有单行的列。 该行的值为
[布尔值](../../sql-reference/data-types/boolean.md) 类型:
查询返回一行结果,列名为 `result`, 该行的值为 [布尔值](../../sql-reference/data-types/boolean.md) 类型:
- 0-表中的数据已损坏
- 1-数据保持完整性
- 0-表中的数据已损坏
- 1-数据保持完整性
`CHECK TABLE` 查询支持下表引擎:
@ -56,13 +57,14 @@ CHECK TABLE [db.]name
如果表已损坏,则可以将未损坏的数据复制到另一个表。 要做到这一点:
1. 创建一个与损坏的表结构相同的新表。 要做到这一点,请执行查询 `CREATE TABLE <new_table_name> AS <damaged_table_name>`.
1. 创建一个与损坏的表结构相同的新表。 请执行查询 `CREATE TABLE <new_table_name> AS <damaged_table_name>`.
2. 将 [max_threads](../../operations/settings/settings.md#settings-max_threads) 值设置为1以在单个线程中处理下一个查询。 要这样做,请运行查询 `SET max_threads = 1`.
3. 执行查询 `INSERT INTO <new_table_name> SELECT * FROM <damaged_table_name>`. 此请求将未损坏的数据从损坏的表复制到另一个表。 只有损坏部分之前的数据才会被复制。
4. 重新启动 `clickhouse-client` 以重置 `max_threads` 值。
## DESCRIBE TABLE {#misc-describe-table}
查看表的描述信息返回各列的Schema语法如下
``` sql
DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format]
```
@ -73,24 +75,25 @@ DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format]
- `type`— 列的类型。
- `default_type` — [默认表达式](create.md#create-default-values) (`DEFAULT`, `MATERIALIZED``ALIAS`)中使用的子句。 如果没有指定默认表达式,则列包含一个空字符串。
- `default_expression``DEFAULT` 子句中指定的值。
- `comment_expression` — 注释。
- `comment_expression` — 注释信息
嵌套数据结构以 “expanded” 格式输出。 每列分别显示,列名后加点号。
## DETACH {#detach}
从服务器中删除有关 name 表的信息。 服务器停止了解该表的存在。
从服务器中删除目标表信息(删除对象是表), 执行查询后,服务器视作该表已经不存在。
``` sql
DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]
```
这不会删除表的数据或元数据。 在下一次服务器启动时,服务器将读取元数据并再次查找该表。
同样,可以使用 `ATTACH` 查询重新连接一个 “detached” 的表(系统表除外,没有为它们存储元数据)。
也可以不停止服务器的情况下,使用前面介绍的 `ATTACH` 查询来重新关联该表(系统表除外,没有为它们存储元数据)。
## DROP {#drop}
删除已经存在的实体。如果指定 `IF EXISTS` 则如果实体不存在,则不返回错误。
建议使用时添加 `IF EXISTS` 修饰符。
## DROP DATABASE {#drop-database}
@ -135,7 +138,7 @@ DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
删除角色。
已删除的角色将从授予该角色的所有实体撤销
同时该角色所拥有的权限也会被收回
语法:
@ -199,6 +202,8 @@ EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT
## KILL QUERY {#kill-query-statement}
``` sql
KILL QUERY [ON CLUSTER cluster]
WHERE <where expression to SELECT FROM system.processes query>
@ -219,16 +224,17 @@ KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'
KILL QUERY WHERE user='username' SYNC
```
只读用户只能停止自己的查询。
只读用户只能停止自己提交的查询。
默认情况下,使用异步版本的查询 (`ASYNC`),不等待确认查询已停止。
默认情况下,使用异步版本的查询 (`ASYNC`),不需要等待确认查询已停止。
同步版本 (`SYNC`)等待所有查询停止,并在停止时显示有关每个进程的信息。
响应包含 `kill_status` 列,该列可以采用以下值:
而相对的,终止同步版本 (`SYNC`)的查询会显示每步停止时间。
返回信息包含 `kill_status` 列,该列可以采用以下值:
1. finished 查询已成功终止。
2. waiting 发送查询信号终止后,等待查询结束。
3. 其他值解释为什么查询不能停止。
3. 其他值,会解释为什么查询不能停止。
测试查询 (`TEST`)仅检查用户的权限,并显示要停止的查询列表。

View File

@ -2335,7 +2335,7 @@ public:
"Suggestion limit for how many databases, tables and columns to fetch.")
("multiline,m", "multiline")
("multiquery,n", "multiquery")
("queries-file,qf", po::value<std::string>(), "file path with queries to execute")
("queries-file", po::value<std::string>(), "file path with queries to execute")
("format,f", po::value<std::string>(), "default output format")
("testmode,T", "enable test hints in comments")
("ignore-error", "do not stop processing in multiquery mode")

View File

@ -115,7 +115,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques
std::string name = schema_name.empty() ? backQuoteIfNeed(table_name) : backQuoteIfNeed(schema_name) + "." + backQuoteIfNeed(table_name);
WriteBufferFromOwnString buf;
std::string input = "SELECT * FROM " + name + " WHERE 1 = 0";
ParserQueryWithOutput parser;
ParserQueryWithOutput parser(input.data() + input.size());
ASTPtr select = parseQuery(parser, input.data(), input.data() + input.size(), "", context_settings.max_query_size, context_settings.max_parser_depth);
IAST::FormatSettings settings(buf, true);

View File

@ -139,6 +139,28 @@ void setupTmpPath(Poco::Logger * log, const std::string & path)
}
}
int waitServersToFinish(std::vector<DB::ProtocolServerAdapter> & servers, size_t seconds_to_wait)
{
const int sleep_max_ms = 1000 * seconds_to_wait;
const int sleep_one_ms = 100;
int sleep_current_ms = 0;
int current_connections = 0;
while (sleep_current_ms < sleep_max_ms)
{
current_connections = 0;
for (auto & server : servers)
{
server.stop();
current_connections += server.currentConnections();
}
if (!current_connections)
break;
sleep_current_ms += sleep_one_ms;
std::this_thread::sleep_for(std::chrono::milliseconds(sleep_one_ms));
}
return current_connections;
}
}
namespace DB
@ -366,7 +388,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
LOG_WARNING(log, "Server was built in debug mode. It will work slowly.");
#endif
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER)
#if defined(SANITIZER)
LOG_WARNING(log, "Server was built with sanitizer. It will work slowly.");
#endif
@ -794,8 +816,29 @@ int Server::main(const std::vector<std::string> & /*args*/)
LOG_DEBUG(log, "Shut down storages.");
for (auto & server : servers_to_start_before_tables)
server.stop();
if (!servers_to_start_before_tables.empty())
{
LOG_DEBUG(log, "Waiting for current connections to servers for tables to finish.");
int current_connections = 0;
for (auto & server : servers_to_start_before_tables)
{
server.stop();
current_connections += server.currentConnections();
}
if (current_connections)
LOG_INFO(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections);
else
LOG_INFO(log, "Closed all listening sockets.");
if (current_connections > 0)
current_connections = waitServersToFinish(servers_to_start_before_tables, config().getInt("shutdown_wait_unfinished", 5));
if (current_connections)
LOG_INFO(log, "Closed connections to servers for tables. But {} remain. Probably some tables of other users cannot finish their connections after context shutdown.", current_connections);
else
LOG_INFO(log, "Closed connections to servers for tables.");
}
/** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available.
* At this moment, no one could own shared part of Context.
@ -1167,24 +1210,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->getProcessList().killAllQueries();
if (current_connections)
{
const int sleep_max_ms = 1000 * config().getInt("shutdown_wait_unfinished", 5);
const int sleep_one_ms = 100;
int sleep_current_ms = 0;
while (sleep_current_ms < sleep_max_ms)
{
current_connections = 0;
for (auto & server : servers)
{
server.stop();
current_connections += server.currentConnections();
}
if (!current_connections)
break;
sleep_current_ms += sleep_one_ms;
std::this_thread::sleep_for(std::chrono::milliseconds(sleep_one_ms));
}
}
current_connections = waitServersToFinish(servers, config().getInt("shutdown_wait_unfinished", 5));
if (current_connections)
LOG_INFO(log, "Closed connections. But {} remain."

View File

@ -48,97 +48,121 @@
-->
</logger>
<send_crash_reports>
<!-- Changing <enabled> to true allows sending crash reports to -->
<!-- the ClickHouse core developers team via Sentry https://sentry.io -->
<!-- Doing so at least in pre-production environments is highly appreciated -->
<enabled>false</enabled>
<!-- Change <anonymize> to true if you don't feel comfortable attaching the server hostname to the crash report -->
<anonymize>false</anonymize>
<!-- Default endpoint should be changed to different Sentry DSN only if you have -->
<!-- some in-house engineers or hired consultants who're going to debug ClickHouse issues for you -->
<endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
</send_crash_reports>
<!-- It is the name that will be shown in the clickhouse-client.
By default, anything with "production" will be highlighted in red in query prompt.
-->
<!--display_name>production</display_name-->
<!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
<!-- Port for HTTP API. See also 'https_port' for secure connections.
This interface is also used by ODBC and JDBC drivers (DataGrip, Dbeaver, ...)
and by most of web interfaces (embedded UI, Grafana, Redash, ...).
-->
<http_port>8123</http_port>
<!-- Port for interaction by native protocol with:
- clickhouse-client and other native ClickHouse tools (clickhouse-benchmark, clickhouse-copier);
- clickhouse-server with other clickhouse-servers for distributed query processing;
- ClickHouse drivers and applications supporting native protocol
(this protocol is also informally called as "the TCP protocol");
See also 'tcp_port_secure' for secure connections.
-->
<tcp_port>9000</tcp_port>
<!-- Compatibility with MySQL protocol.
ClickHouse will pretend to be MySQL for applications connecting to this port.
-->
<mysql_port>9004</mysql_port>
<!-- For HTTPS and SSL over native protocol. -->
<!--
<https_port>8443</https_port>
<tcp_port_secure>9440</tcp_port_secure>
<!-- Compatibility with PostgreSQL protocol.
ClickHouse will pretend to be PostgreSQL for applications connecting to this port.
-->
<!-- <postgresql_port>9005</postgresql_port> -->
<!-- TCP with PROXY protocol (PROXY header sent for every connection) -->
<!--
<tcp_with_proxy_port>9010</tcp_with_proxy_port>
<!-- HTTP API with TLS (HTTPS).
You have to configure certificate to enable this interface.
See the openSSL section below.
-->
<!-- <https_port>8443</https_port> -->
<!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
<openSSL>
<server> <!-- Used for https server AND secure tcp port -->
<!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
<certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
<privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
<!-- openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 -->
<dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
<verificationMode>none</verificationMode>
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
</server>
<client> <!-- Used for connecting to https dictionary source and secured Zookeeper communication -->
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
<!-- Use for self-signed: <verificationMode>none</verificationMode> -->
<invalidCertificateHandler>
<!-- Use for self-signed: <name>AcceptCertificateHandler</name> -->
<name>RejectCertificateHandler</name>
</invalidCertificateHandler>
</client>
</openSSL>
<!-- Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -->
<!--
<http_server_default_response><![CDATA[<html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>]]></http_server_default_response>
<!-- Native interface with TLS.
You have to configure certificate to enable this interface.
See the openSSL section below.
-->
<!-- <tcp_port_secure>9440</tcp_port_secure> -->
<!-- Port for communication between replicas. Used for data exchange. -->
<!-- Native interface wrapped with PROXYv1 protocol
PROXYv1 header sent for every connection.
ClickHouse will extract information about proxy-forwarded client address from the header.
-->
<!-- <tcp_with_proxy_port>9011</tcp_with_proxy_port> -->
<!-- Port for communication between replicas. Used for data exchange.
It provides low-level data access between servers.
This port should not be accessible from untrusted networks.
See also 'interserver_http_credentials'.
Data transferred over connections to this port should not go through untrusted networks.
See also 'interserver_https_port'.
-->
<interserver_http_port>9009</interserver_http_port>
<!-- Port for communication between replicas with TLS.
You have to configure certificate to enable this interface.
See the openSSL section below.
See also 'interserver_http_credentials'.
-->
<!-- <interserver_https_port>9010</interserver_https_port> -->
<!-- Hostname that is used by other replicas to request this server.
If not specified, than it is determined analogous to 'hostname -f' command.
This setting could be used to switch replication to another network interface.
This setting could be used to switch replication to another network interface
(the server may be connected to multiple networks via multiple addresses)
-->
<!--
<interserver_http_host>example.yandex.ru</interserver_http_host>
-->
<!-- Listen specified host. use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. -->
<!-- You can specify credentials for authenthication between replicas.
This is required when interserver_https_port is accessible from untrusted networks,
and also recommended to avoid SSRF attacks from possibly compromised services in your network.
-->
<!--<interserver_http_credentials>
<user>interserver</user>
<password></password>
</interserver_http_credentials>-->
<!-- Listen specified address.
Use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere.
Notes:
If you open connections from wildcard address, make sure that at least one of the following measures applied:
- server is protected by firewall and not accessible from untrusted networks;
- all users are restricted to subset of network addresses (see users.xml);
- all users have strong passwords, only secure (TLS) interfaces are accessible, or connections are only made via TLS interfaces.
- users without password have readonly access.
See also: https://www.shodan.io/search?query=clickhouse
-->
<!-- <listen_host>::</listen_host> -->
<!-- Same for hosts with disabled ipv6: -->
<!-- Same for hosts without support for IPv6: -->
<!-- <listen_host>0.0.0.0</listen_host> -->
<!-- Default values - try listen localhost on ipv4 and ipv6: -->
<!-- Default values - try listen localhost on IPv4 and IPv6. -->
<!--
<listen_host>::1</listen_host>
<listen_host>127.0.0.1</listen_host>
-->
<!-- Don't exit if ipv6 or ipv4 unavailable, but listen_host with this protocol specified -->
<!-- Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. -->
<!-- <listen_try>0</listen_try> -->
<!-- Allow listen on same address:port -->
<!-- Allow multiple servers to listen on the same address:port. This is not recommended.
-->
<!-- <listen_reuse_port>0</listen_reuse_port> -->
<!-- <listen_backlog>64</listen_backlog> -->
<max_connections>4096</max_connections>
<!-- For 'Connection: keep-alive' in HTTP 1.1 -->
<keep_alive_timeout>3</keep_alive_timeout>
<!-- gRPC protocol (see src/Server/grpc_protos/clickhouse_grpc.proto for the API) -->
@ -172,6 +196,43 @@
<verbose_logs>false</verbose_logs>
</grpc>
<!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
<openSSL>
<server> <!-- Used for https server AND secure tcp port -->
<!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
<certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
<privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
<!-- dhparams are optional. You can delete the <dhParamsFile> element.
To generate dhparams, use the following command:
openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
Only file format with BEGIN DH PARAMETERS is supported.
-->
<dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
<verificationMode>none</verificationMode>
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
</server>
<client> <!-- Used for connecting to https dictionary source and secured Zookeeper communication -->
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
<!-- Use for self-signed: <verificationMode>none</verificationMode> -->
<invalidCertificateHandler>
<!-- Use for self-signed: <name>AcceptCertificateHandler</name> -->
<name>RejectCertificateHandler</name>
</invalidCertificateHandler>
</client>
</openSSL>
<!-- Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -->
<!--
<http_server_default_response><![CDATA[<html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>]]></http_server_default_response>
-->
<!-- Maximum number of concurrent queries. -->
<max_concurrent_queries>100</max_concurrent_queries>
@ -605,7 +666,7 @@
event_date + INTERVAL 1 WEEK
event_date + INTERVAL 7 DAY DELETE
event_date + INTERVAL 2 WEEK TO DISK 'bbb'
<ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
-->
@ -887,6 +948,18 @@
</http_handlers>
-->
<send_crash_reports>
<!-- Changing <enabled> to true allows sending crash reports to -->
<!-- the ClickHouse core developers team via Sentry https://sentry.io -->
<!-- Doing so at least in pre-production environments is highly appreciated -->
<enabled>false</enabled>
<!-- Change <anonymize> to true if you don't feel comfortable attaching the server hostname to the crash report -->
<anonymize>false</anonymize>
<!-- Default endpoint should be changed to different Sentry DSN only if you have -->
<!-- some in-house engineers or hired consultants who're going to debug ClickHouse issues for you -->
<endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
</send_crash_reports>
<!-- Uncomment to disable ClickHouse internal DNS caching. -->
<!-- <disable_internal_dns_cache>1</disable_internal_dns_cache> -->
</yandex>

View File

@ -392,9 +392,12 @@ bool ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &...
if (!getUser())
return access_denied("User has been dropped", ErrorCodes::UNKNOWN_USER);
/// If the current user was allowed to create a temporary table
/// then he is allowed to do with it whatever he wants.
if ((sizeof...(args) >= 2) && (getDatabase(args...) == DatabaseCatalog::TEMPORARY_DATABASE))
/// Access to temporary tables is controlled in an unusual way, not like normal tables.
/// Creating of temporary tables is controlled by AccessType::CREATE_TEMPORARY_TABLES grant,
/// and other grants are considered as always given.
/// The DatabaseCatalog class won't resolve StorageID for temporary tables
/// which shouldn't be accessed.
if (getDatabase(args...) == DatabaseCatalog::TEMPORARY_DATABASE)
return access_granted();
auto acs = getAccessRightsWithImplicit();

View File

@ -156,6 +156,25 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
{
const String & setting_name = change.name;
if (setting_name == "profile")
{
/// TODO Check profile settings in Context::setProfile(...), not here. It will be backward incompatible.
const String & profile_name = change.value.safeGet<String>();
const auto & profile_settings_changes = manager->getProfileSettings(profile_name);
try
{
/// NOTE We cannot use CLAMP_ON_VIOLATION here, because we cannot modify elements of profile_settings_changes
for (auto change_copy : *profile_settings_changes)
checkImpl(current_settings, change_copy, THROW_ON_VIOLATION);
}
catch (Exception & e)
{
e.addMessage(", while trying to set settings profile {}", profile_name);
throw;
}
return true;
}
bool cannot_cast;
auto cast_value = [&](const Field & x) -> Field
{

View File

@ -18,6 +18,7 @@
#include <Poco/String.h>
#include "registerAggregateFunctions.h"
#include <Functions/FunctionFactory.h>
namespace DB
{
@ -135,12 +136,17 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
return combinator->transformAggregateFunction(nested_function, out_properties, argument_types, parameters);
}
String extra_info;
if (FunctionFactory::instance().hasNameOrAlias(name))
extra_info = ". There is an ordinary function with the same name, but aggregate function is expected here";
auto hints = this->getHints(name);
if (!hints.empty())
throw Exception(fmt::format("Unknown aggregate function {}. Maybe you meant: {}", name, toString(hints)),
ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION);
throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION,
"Unknown aggregate function {}{}. Maybe you meant: {}", name, extra_info, toString(hints));
else
throw Exception(fmt::format("Unknown aggregate function {}", name), ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION);
throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION, "Unknown aggregate function {}{}", name, extra_info);
}

View File

@ -0,0 +1,32 @@
#include <AggregateFunctions/AggregateFunctionCombinatorFactory.h>
#include <AggregateFunctions/AggregateFunctionSimpleState.h>
namespace DB
{
namespace
{
class AggregateFunctionCombinatorSimpleState final : public IAggregateFunctionCombinator
{
public:
String getName() const override { return "SimpleState"; }
DataTypes transformArguments(const DataTypes & arguments) const override { return arguments; }
AggregateFunctionPtr transformAggregateFunction(
const AggregateFunctionPtr & nested_function,
const AggregateFunctionProperties &,
const DataTypes & arguments,
const Array & params) const override
{
return std::make_shared<AggregateFunctionSimpleState>(nested_function, arguments, params);
}
};
}
void registerAggregateFunctionCombinatorSimpleState(AggregateFunctionCombinatorFactory & factory)
{
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorSimpleState>());
}
}

View File

@ -0,0 +1,77 @@
#pragma once
#include <AggregateFunctions/IAggregateFunction.h>
#include <DataTypes/DataTypeCustomSimpleAggregateFunction.h>
#include <DataTypes/DataTypeFactory.h>
namespace DB
{
/** Not an aggregate function, but an adapter of aggregate functions.
* Aggregate functions with the `SimpleState` suffix is almost identical to the corresponding ones,
* except the return type becomes DataTypeCustomSimpleAggregateFunction.
*/
class AggregateFunctionSimpleState final : public IAggregateFunctionHelper<AggregateFunctionSimpleState>
{
private:
AggregateFunctionPtr nested_func;
DataTypes arguments;
Array params;
public:
AggregateFunctionSimpleState(AggregateFunctionPtr nested_, const DataTypes & arguments_, const Array & params_)
: IAggregateFunctionHelper<AggregateFunctionSimpleState>(arguments_, params_)
, nested_func(nested_)
, arguments(arguments_)
, params(params_)
{
}
String getName() const override { return nested_func->getName() + "SimpleState"; }
DataTypePtr getReturnType() const override
{
DataTypeCustomSimpleAggregateFunction::checkSupportedFunctions(nested_func);
// Need to make a clone because it'll be customized.
auto storage_type = DataTypeFactory::instance().get(nested_func->getReturnType()->getName());
DataTypeCustomNamePtr custom_name
= std::make_unique<DataTypeCustomSimpleAggregateFunction>(nested_func, DataTypes{nested_func->getReturnType()}, params);
storage_type->setCustomization(std::make_unique<DataTypeCustomDesc>(std::move(custom_name), nullptr));
return storage_type;
}
void create(AggregateDataPtr place) const override { nested_func->create(place); }
void destroy(AggregateDataPtr place) const noexcept override { nested_func->destroy(place); }
bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); }
size_t sizeOfData() const override { return nested_func->sizeOfData(); }
size_t alignOfData() const override { return nested_func->alignOfData(); }
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override
{
nested_func->add(place, columns, row_num, arena);
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override { nested_func->merge(place, rhs, arena); }
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override { nested_func->serialize(place, buf); }
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override
{
nested_func->deserialize(place, buf, arena);
}
void insertResultInto(AggregateDataPtr place, IColumn & to, Arena * arena) const override
{
nested_func->insertResultInto(place, to, arena);
}
bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); }
AggregateFunctionPtr getNestedFunction() const { return nested_func; }
};
}

View File

@ -47,6 +47,7 @@ class AggregateFunctionCombinatorFactory;
void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
void registerAggregateFunctionCombinatorArray(AggregateFunctionCombinatorFactory &);
void registerAggregateFunctionCombinatorForEach(AggregateFunctionCombinatorFactory &);
void registerAggregateFunctionCombinatorSimpleState(AggregateFunctionCombinatorFactory &);
void registerAggregateFunctionCombinatorState(AggregateFunctionCombinatorFactory &);
void registerAggregateFunctionCombinatorMerge(AggregateFunctionCombinatorFactory &);
void registerAggregateFunctionCombinatorNull(AggregateFunctionCombinatorFactory &);
@ -104,6 +105,7 @@ void registerAggregateFunctions()
registerAggregateFunctionCombinatorIf(factory);
registerAggregateFunctionCombinatorArray(factory);
registerAggregateFunctionCombinatorForEach(factory);
registerAggregateFunctionCombinatorSimpleState(factory);
registerAggregateFunctionCombinatorState(factory);
registerAggregateFunctionCombinatorMerge(factory);
registerAggregateFunctionCombinatorNull(factory);

View File

@ -41,6 +41,7 @@ SRCS(
AggregateFunctionRetention.cpp
AggregateFunctionSequenceMatch.cpp
AggregateFunctionSimpleLinearRegression.cpp
AggregateFunctionSimpleState.cpp
AggregateFunctionState.cpp
AggregateFunctionStatistics.cpp
AggregateFunctionStatisticsSimple.cpp

View File

@ -88,6 +88,10 @@ if (USE_AWS_S3)
add_headers_and_sources(dbms Disks/S3)
endif()
if (USE_HDFS)
add_headers_and_sources(dbms Storages/HDFS)
endif()
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
@ -389,8 +393,8 @@ if (USE_GRPC)
endif()
if (USE_HDFS)
target_link_libraries (clickhouse_common_io PUBLIC ${HDFS3_LIBRARY})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR})
dbms_target_link_libraries(PRIVATE ${HDFS3_LIBRARY})
dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR})
endif()
if (USE_AWS_S3)

View File

@ -5,8 +5,9 @@
#include <Poco/Net/StreamSocket.h>
#include <Common/Throttler.h>
#include <Common/config.h>
#if !defined(ARCADIA_BUILD)
# include <Common/config.h>
#endif
#include <Core/Block.h>
#include <Core/Defines.h>
#include <IO/Progress.h>

241
src/Columns/ColumnMap.cpp Normal file
View File

@ -0,0 +1,241 @@
#include <Columns/ColumnMap.h>
#include <Columns/IColumnImpl.h>
#include <DataStreams/ColumnGathererStream.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#include <ext/map.h>
#include <ext/range.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include <Common/WeakHash.h>
#include <Core/Field.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
}
std::string ColumnMap::getName() const
{
WriteBufferFromOwnString res;
const auto & nested_tuple = getNestedData();
res << "Map(" << nested_tuple.getColumn(0).getName()
<< ", " << nested_tuple.getColumn(1).getName() << ")";
return res.str();
}
ColumnMap::ColumnMap(MutableColumnPtr && nested_)
: nested(std::move(nested_))
{
const auto * column_array = typeid_cast<const ColumnArray *>(nested.get());
if (!column_array)
throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnMap can be created only from array of tuples");
const auto * column_tuple = typeid_cast<const ColumnTuple *>(column_array->getDataPtr().get());
if (!column_tuple)
throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnMap can be created only from array of tuples");
if (column_tuple->getColumns().size() != 2)
throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnMap should contain only 2 subcolumns: keys and values");
for (const auto & column : column_tuple->getColumns())
if (isColumnConst(*column))
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnMap cannot have ColumnConst as its element");
}
MutableColumnPtr ColumnMap::cloneEmpty() const
{
return ColumnMap::create(nested->cloneEmpty());
}
MutableColumnPtr ColumnMap::cloneResized(size_t new_size) const
{
return ColumnMap::create(nested->cloneResized(new_size));
}
Field ColumnMap::operator[](size_t n) const
{
auto array = DB::get<Array>((*nested)[n]);
return Map(std::make_move_iterator(array.begin()), std::make_move_iterator(array.end()));
}
void ColumnMap::get(size_t n, Field & res) const
{
const auto & offsets = getNestedColumn().getOffsets();
size_t offset = offsets[n - 1];
size_t size = offsets[n] - offsets[n - 1];
res = Map(size);
auto & map = DB::get<Map &>(res);
for (size_t i = 0; i < size; ++i)
getNestedData().get(offset + i, map[i]);
}
StringRef ColumnMap::getDataAt(size_t) const
{
throw Exception("Method getDataAt is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void ColumnMap::insertData(const char *, size_t)
{
throw Exception("Method insertData is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void ColumnMap::insert(const Field & x)
{
const auto & map = DB::get<const Map &>(x);
nested->insert(Array(map.begin(), map.end()));
}
void ColumnMap::insertDefault()
{
nested->insertDefault();
}
void ColumnMap::popBack(size_t n)
{
nested->popBack(n);
}
StringRef ColumnMap::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const
{
return nested->serializeValueIntoArena(n, arena, begin);
}
const char * ColumnMap::deserializeAndInsertFromArena(const char * pos)
{
return nested->deserializeAndInsertFromArena(pos);
}
void ColumnMap::updateHashWithValue(size_t n, SipHash & hash) const
{
nested->updateHashWithValue(n, hash);
}
void ColumnMap::updateWeakHash32(WeakHash32 & hash) const
{
nested->updateWeakHash32(hash);
}
void ColumnMap::updateHashFast(SipHash & hash) const
{
nested->updateHashFast(hash);
}
void ColumnMap::insertRangeFrom(const IColumn & src, size_t start, size_t length)
{
nested->insertRangeFrom(
assert_cast<const ColumnMap &>(src).getNestedColumn(),
start, length);
}
ColumnPtr ColumnMap::filter(const Filter & filt, ssize_t result_size_hint) const
{
auto filtered = nested->filter(filt, result_size_hint);
return ColumnMap::create(filtered);
}
ColumnPtr ColumnMap::permute(const Permutation & perm, size_t limit) const
{
auto permuted = nested->permute(perm, limit);
return ColumnMap::create(std::move(permuted));
}
ColumnPtr ColumnMap::index(const IColumn & indexes, size_t limit) const
{
auto res = nested->index(indexes, limit);
return ColumnMap::create(std::move(res));
}
ColumnPtr ColumnMap::replicate(const Offsets & offsets) const
{
auto replicated = nested->replicate(offsets);
return ColumnMap::create(std::move(replicated));
}
MutableColumns ColumnMap::scatter(ColumnIndex num_columns, const Selector & selector) const
{
auto scattered_columns = nested->scatter(num_columns, selector);
MutableColumns res;
res.reserve(num_columns);
for (auto && scattered : scattered_columns)
res.push_back(ColumnMap::create(std::move(scattered)));
return res;
}
int ColumnMap::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
{
const auto & rhs_map = assert_cast<const ColumnMap &>(rhs);
return nested->compareAt(n, m, rhs_map.getNestedColumn(), nan_direction_hint);
}
void ColumnMap::compareColumn(const IColumn & rhs, size_t rhs_row_num,
PaddedPODArray<UInt64> * row_indexes, PaddedPODArray<Int8> & compare_results,
int direction, int nan_direction_hint) const
{
return doCompareColumn<ColumnMap>(assert_cast<const ColumnMap &>(rhs), rhs_row_num, row_indexes,
compare_results, direction, nan_direction_hint);
}
void ColumnMap::getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const
{
nested->getPermutation(reverse, limit, nan_direction_hint, res);
}
void ColumnMap::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const
{
nested->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range);
}
void ColumnMap::gather(ColumnGathererStream & gatherer)
{
gatherer.gather(*this);
}
void ColumnMap::reserve(size_t n)
{
nested->reserve(n);
}
size_t ColumnMap::byteSize() const
{
return nested->byteSize();
}
size_t ColumnMap::allocatedBytes() const
{
return nested->allocatedBytes();
}
void ColumnMap::protect()
{
nested->protect();
}
void ColumnMap::getExtremes(Field & min, Field & max) const
{
nested->getExtremes(min, max);
}
void ColumnMap::forEachSubcolumn(ColumnCallback callback)
{
nested->forEachSubcolumn(callback);
}
bool ColumnMap::structureEquals(const IColumn & rhs) const
{
if (const auto * rhs_map = typeid_cast<const ColumnMap *>(&rhs))
return nested->structureEquals(*rhs_map->nested);
return false;
}
}

92
src/Columns/ColumnMap.h Normal file
View File

@ -0,0 +1,92 @@
#pragma once
#include <Core/Block.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnTuple.h>
namespace DB
{
/** Column, that stores a nested Array(Tuple(key, value)) column.
*/
class ColumnMap final : public COWHelper<IColumn, ColumnMap>
{
private:
friend class COWHelper<IColumn, ColumnMap>;
WrappedPtr nested;
explicit ColumnMap(MutableColumnPtr && nested_);
ColumnMap(const ColumnMap &) = default;
public:
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
*/
using Base = COWHelper<IColumn, ColumnMap>;
static Ptr create(const ColumnPtr & keys, const ColumnPtr & values, const ColumnPtr & offsets)
{
auto nested_column = ColumnArray::create(ColumnTuple::create(Columns{keys, values}), offsets);
return ColumnMap::create(nested_column);
}
static Ptr create(const ColumnPtr & column) { return ColumnMap::create(column->assumeMutable()); }
static Ptr create(ColumnPtr && arg) { return create(arg); }
template <typename Arg, typename = typename std::enable_if<std::is_rvalue_reference<Arg &&>::value>::type>
static MutablePtr create(Arg && arg) { return Base::create(std::forward<Arg>(arg)); }
std::string getName() const override;
const char * getFamilyName() const override { return "Map"; }
TypeIndex getDataType() const override { return TypeIndex::Map; }
MutableColumnPtr cloneEmpty() const override;
MutableColumnPtr cloneResized(size_t size) const override;
size_t size() const override { return nested->size(); }
Field operator[](size_t n) const override;
void get(size_t n, Field & res) const override;
StringRef getDataAt(size_t n) const override;
void insertData(const char * pos, size_t length) override;
void insert(const Field & x) override;
void insertDefault() override;
void popBack(size_t n) override;
StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
const char * deserializeAndInsertFromArena(const char * pos) override;
void updateHashWithValue(size_t n, SipHash & hash) const override;
void updateWeakHash32(WeakHash32 & hash) const override;
void updateHashFast(SipHash & hash) const override;
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
ColumnPtr replicate(const Offsets & offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
void gather(ColumnGathererStream & gatherer_stream) override;
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
void compareColumn(const IColumn & rhs, size_t rhs_row_num,
PaddedPODArray<UInt64> * row_indexes, PaddedPODArray<Int8> & compare_results,
int direction, int nan_direction_hint) const override;
void getExtremes(Field & min, Field & max) const override;
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const override;
void reserve(size_t n) override;
size_t byteSize() const override;
size_t allocatedBytes() const override;
void protect() override;
void forEachSubcolumn(ColumnCallback callback) override;
bool structureEquals(const IColumn & rhs) const override;
const ColumnArray & getNestedColumn() const { return assert_cast<const ColumnArray &>(*nested); }
ColumnArray & getNestedColumn() { return assert_cast<ColumnArray &>(*nested); }
const ColumnTuple & getNestedData() const { return assert_cast<const ColumnTuple &>(getNestedColumn().getData()); }
ColumnTuple & getNestedData() { return assert_cast<ColumnTuple &>(getNestedColumn().getData()); }
};
}

View File

@ -24,6 +24,7 @@ SRCS(
ColumnFixedString.cpp
ColumnFunction.cpp
ColumnLowCardinality.cpp
ColumnMap.cpp
ColumnNullable.cpp
ColumnString.cpp
ColumnTuple.cpp

View File

@ -26,6 +26,7 @@
#define DISABLE_MREMAP 1
#endif
#include <common/mremap.h>
#include <common/getPageSize.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
@ -59,7 +60,6 @@
*/
extern const size_t MMAP_THRESHOLD;
static constexpr size_t MMAP_MIN_ALIGNMENT = 4096;
static constexpr size_t MALLOC_MIN_ALIGNMENT = 8;
namespace DB
@ -194,10 +194,11 @@ private:
void * allocNoTrack(size_t size, size_t alignment)
{
void * buf;
size_t mmap_min_alignment = ::getPageSize();
if (size >= MMAP_THRESHOLD)
{
if (alignment > MMAP_MIN_ALIGNMENT)
if (alignment > mmap_min_alignment)
throw DB::Exception(fmt::format("Too large alignment {}: more than page size when allocating {}.",
ReadableSize(alignment), ReadableSize(size)), DB::ErrorCodes::BAD_ARGUMENTS);

View File

@ -83,10 +83,11 @@ private:
/// Last contiguous chunk of memory.
Chunk * head;
size_t size_in_bytes;
size_t page_size;
static size_t roundUpToPageSize(size_t s)
static size_t roundUpToPageSize(size_t s, size_t page_size)
{
return (s + 4096 - 1) / 4096 * 4096;
return (s + page_size - 1) / page_size * page_size;
}
/// If chunks size is less than 'linear_growth_threshold', then use exponential growth, otherwise - linear growth
@ -113,7 +114,7 @@ private:
}
assert(size_after_grow >= min_next_size);
return roundUpToPageSize(size_after_grow);
return roundUpToPageSize(size_after_grow, page_size);
}
/// Add next contiguous chunk of memory with size not less than specified.
@ -129,7 +130,8 @@ private:
public:
Arena(size_t initial_size_ = 4096, size_t growth_factor_ = 2, size_t linear_growth_threshold_ = 128 * 1024 * 1024)
: growth_factor(growth_factor_), linear_growth_threshold(linear_growth_threshold_),
head(new Chunk(initial_size_, nullptr)), size_in_bytes(head->size())
head(new Chunk(initial_size_, nullptr)), size_in_bytes(head->size()),
page_size(static_cast<size_t>(::getPageSize()))
{
}

View File

@ -13,6 +13,8 @@
#include <boost/noncopyable.hpp>
#include <ext/scope_guard.h>
#include <common/getPageSize.h>
#include <Common/Exception.h>
#include <Common/randomSeed.h>
#include <Common/formatReadable.h>
@ -326,8 +328,6 @@ private:
return (x + (rounding - 1)) / rounding * rounding;
}
static constexpr size_t page_size = 4096;
/// Sizes and addresses of allocated memory will be aligned to specified boundary.
static constexpr size_t alignment = 16;
@ -505,6 +505,7 @@ private:
/// If nothing was found and total size of allocated chunks plus required size is lower than maximum,
/// allocate a new chunk.
size_t page_size = static_cast<size_t>(::getPageSize());
size_t required_chunk_size = std::max(min_chunk_size, roundUp(size, page_size));
if (total_chunks_size + required_chunk_size <= max_total_size)
{

View File

@ -16,9 +16,11 @@
#include <Common/StringUtils/StringUtils.h>
#include <Common/Exception.h>
#include <common/getResource.h>
#include <common/errnoToString.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#define PREPROCESSED_SUFFIX "-preprocessed"
@ -234,7 +236,7 @@ static std::string layerFromHost()
{
utsname buf;
if (uname(&buf))
throw Poco::Exception(std::string("uname failed: ") + std::strerror(errno));
throw Poco::Exception(std::string("uname failed: ") + errnoToString(errno));
std::string layer = numberFromHost(buf.nodename);
if (layer.empty())

View File

@ -529,6 +529,7 @@
M(560, ZSTD_ENCODER_FAILED) \
M(561, ZSTD_DECODER_FAILED) \
M(562, TLD_LIST_NOT_FOUND) \
M(563, CANNOT_READ_MAP_FROM_TEXT) \
\
M(999, KEEPER_EXCEPTION) \
M(1000, POCO_EXCEPTION) \

View File

@ -93,6 +93,22 @@ String FieldVisitorDump::operator() (const Tuple & x) const
return wb.str();
}
String FieldVisitorDump::operator() (const Map & x) const
{
WriteBufferFromOwnString wb;
wb << "Map_(";
for (auto it = x.begin(); it != x.end(); ++it)
{
if (it != x.begin())
wb << ", ";
wb << applyVisitor(*this, *it);
}
wb << ')';
return wb.str();
}
String FieldVisitorDump::operator() (const AggregateFunctionStateData & x) const
{
WriteBufferFromOwnString wb;
@ -176,6 +192,82 @@ String FieldVisitorToString::operator() (const Tuple & x) const
return wb.str();
}
String FieldVisitorToString::operator() (const Map & x) const
{
WriteBufferFromOwnString wb;
wb << '(';
for (auto it = x.begin(); it != x.end(); ++it)
{
if (it != x.begin())
wb << ", ";
wb << applyVisitor(*this, *it);
}
wb << ')';
return wb.str();
}
void FieldVisitorWriteBinary::operator() (const Null &, WriteBuffer &) const { }
void FieldVisitorWriteBinary::operator() (const UInt64 & x, WriteBuffer & buf) const { DB::writeVarUInt(x, buf); }
void FieldVisitorWriteBinary::operator() (const Int64 & x, WriteBuffer & buf) const { DB::writeVarInt(x, buf); }
void FieldVisitorWriteBinary::operator() (const Float64 & x, WriteBuffer & buf) const { DB::writeFloatBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const String & x, WriteBuffer & buf) const { DB::writeStringBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const UInt128 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const Int128 & x, WriteBuffer & buf) const { DB::writeVarInt(x, buf); }
void FieldVisitorWriteBinary::operator() (const UInt256 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const Int256 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal32> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal64> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal256> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const
{
DB::writeStringBinary(x.name, buf);
DB::writeStringBinary(x.data, buf);
}
void FieldVisitorWriteBinary::operator() (const Array & x, WriteBuffer & buf) const
{
const size_t size = x.size();
DB::writeBinary(size, buf);
for (size_t i = 0; i < size; ++i)
{
const UInt8 type = x[i].getType();
DB::writeBinary(type, buf);
Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, x[i]);
}
}
void FieldVisitorWriteBinary::operator() (const Tuple & x, WriteBuffer & buf) const
{
const size_t size = x.size();
DB::writeBinary(size, buf);
for (size_t i = 0; i < size; ++i)
{
const UInt8 type = x[i].getType();
DB::writeBinary(type, buf);
Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, x[i]);
}
}
void FieldVisitorWriteBinary::operator() (const Map & x, WriteBuffer & buf) const
{
const size_t size = x.size();
DB::writeBinary(size, buf);
for (size_t i = 0; i < size; ++i)
{
const UInt8 type = x[i].getType();
writeBinary(type, buf);
Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, x[i]);
}
}
FieldVisitorHash::FieldVisitorHash(SipHash & hash_) : hash(hash_) {}
@ -238,6 +330,16 @@ void FieldVisitorHash::operator() (const Tuple & x) const
applyVisitor(*this, elem);
}
void FieldVisitorHash::operator() (const Map & x) const
{
UInt8 type = Field::Types::Map;
hash.update(type);
hash.update(x.size());
for (const auto & elem : x)
applyVisitor(*this, elem);
}
void FieldVisitorHash::operator() (const Array & x) const
{
UInt8 type = Field::Types::Array;

View File

@ -77,6 +77,7 @@ public:
String operator() (const String & x) const;
String operator() (const Array & x) const;
String operator() (const Tuple & x) const;
String operator() (const Map & x) const;
String operator() (const DecimalField<Decimal32> & x) const;
String operator() (const DecimalField<Decimal64> & x) const;
String operator() (const DecimalField<Decimal128> & x) const;
@ -88,6 +89,30 @@ public:
};
class FieldVisitorWriteBinary
{
public:
void operator() (const Null & x, WriteBuffer & buf) const;
void operator() (const UInt64 & x, WriteBuffer & buf) const;
void operator() (const UInt128 & x, WriteBuffer & buf) const;
void operator() (const Int64 & x, WriteBuffer & buf) const;
void operator() (const Int128 & x, WriteBuffer & buf) const;
void operator() (const Float64 & x, WriteBuffer & buf) const;
void operator() (const String & x, WriteBuffer & buf) const;
void operator() (const Array & x, WriteBuffer & buf) const;
void operator() (const Tuple & x, WriteBuffer & buf) const;
void operator() (const Map & x, WriteBuffer & buf) const;
void operator() (const DecimalField<Decimal32> & x, WriteBuffer & buf) const;
void operator() (const DecimalField<Decimal64> & x, WriteBuffer & buf) const;
void operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const;
void operator() (const DecimalField<Decimal256> & x, WriteBuffer & buf) const;
void operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const;
void operator() (const UInt256 & x, WriteBuffer & buf) const;
void operator() (const Int256 & x, WriteBuffer & buf) const;
};
/** Print readable and unique text dump of field type and value. */
class FieldVisitorDump : public StaticVisitor<String>
{
@ -101,6 +126,7 @@ public:
String operator() (const String & x) const;
String operator() (const Array & x) const;
String operator() (const Tuple & x) const;
String operator() (const Map & x) const;
String operator() (const DecimalField<Decimal32> & x) const;
String operator() (const DecimalField<Decimal64> & x) const;
String operator() (const DecimalField<Decimal128> & x) const;
@ -137,6 +163,11 @@ public:
throw Exception("Cannot convert Tuple to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE);
}
T operator() (const Map &) const
{
throw Exception("Cannot convert Map to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE);
}
T operator() (const UInt64 & x) const { return T(x); }
T operator() (const Int64 & x) const { return T(x); }
T operator() (const Int128 & x) const { return T(x); }
@ -226,6 +257,7 @@ public:
void operator() (const String & x) const;
void operator() (const Array & x) const;
void operator() (const Tuple & x) const;
void operator() (const Map & x) const;
void operator() (const DecimalField<Decimal32> & x) const;
void operator() (const DecimalField<Decimal64> & x) const;
void operator() (const DecimalField<Decimal128> & x) const;
@ -268,6 +300,7 @@ public:
bool operator() (String &) const { throw Exception("Cannot sum Strings", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Array &) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Tuple &) const { throw Exception("Cannot sum Tuples", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Map &) const { throw Exception("Cannot sum Maps", ErrorCodes::LOGICAL_ERROR); }
bool operator() (UInt128 &) const { throw Exception("Cannot sum UUIDs", ErrorCodes::LOGICAL_ERROR); }
bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); }

View File

@ -106,6 +106,11 @@ public:
return aliases.count(name) || case_insensitive_aliases.count(name);
}
bool hasNameOrAlias(const String & name) const
{
return getMap().count(name) || getCaseInsensitiveMap().count(name) || isAlias(name);
}
virtual ~IFactoryWithAliases() override {}
private:

View File

@ -8,10 +8,11 @@
#include "MemoryStatisticsOS.h"
#include <common/logger_useful.h>
#include <common/getPageSize.h>
#include <Common/Exception.h>
#include <IO/ReadBufferFromMemory.h>
#include <IO/ReadHelpers.h>
#include <common/logger_useful.h>
namespace DB
@ -26,7 +27,6 @@ namespace ErrorCodes
}
static constexpr auto filename = "/proc/self/statm";
static constexpr size_t PAGE_SIZE = 4096;
MemoryStatisticsOS::MemoryStatisticsOS()
{
@ -93,11 +93,12 @@ MemoryStatisticsOS::Data MemoryStatisticsOS::get() const
skipWhitespaceIfAny(in);
readIntText(data.data_and_stack, in);
data.virt *= PAGE_SIZE;
data.resident *= PAGE_SIZE;
data.shared *= PAGE_SIZE;
data.code *= PAGE_SIZE;
data.data_and_stack *= PAGE_SIZE;
size_t page_size = static_cast<size_t>(::getPageSize());
data.virt *= page_size;
data.resident *= page_size;
data.shared *= page_size;
data.code *= page_size;
data.data_and_stack *= page_size;
return data;
}

View File

@ -1,5 +1,6 @@
#pragma once
#include <common/getPageSize.h>
#include <Common/Exception.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/UTF8Helpers.h>
@ -37,7 +38,7 @@ struct StringSearcherBase
{
#ifdef __SSE2__
static constexpr auto n = sizeof(__m128i);
const int page_size = getpagesize();
const int page_size = ::getPageSize();
bool pageSafe(const void * const ptr) const
{

View File

@ -21,6 +21,9 @@
#include <sys/types.h>
#include <dirent.h>
#include <common/errnoToString.h>
namespace DB
{
@ -246,7 +249,7 @@ static void enablePerfEvent(int event_fd)
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Can't enable perf event with file descriptor {}: '{}' ({})",
event_fd, strerror(errno), errno);
event_fd, errnoToString(errno), errno);
}
}
@ -256,7 +259,7 @@ static void disablePerfEvent(int event_fd)
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Can't disable perf event with file descriptor {}: '{}' ({})",
event_fd, strerror(errno), errno);
event_fd, errnoToString(errno), errno);
}
}
@ -266,7 +269,7 @@ static void releasePerfEvent(int event_fd)
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Can't close perf event file descriptor {}: {} ({})",
event_fd, strerror(errno), errno);
event_fd, errnoToString(errno), errno);
}
}
@ -284,7 +287,7 @@ static bool validatePerfEventDescriptor(int & fd)
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Error while checking availability of event descriptor {}: {} ({})",
fd, strerror(errno), errno);
fd, errnoToString(errno), errno);
disablePerfEvent(fd);
releasePerfEvent(fd);
@ -391,7 +394,7 @@ bool PerfEventsCounters::processThreadLocalChanges(const std::string & needed_ev
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Failed to open perf event {} (event_type={}, event_config={}): "
"'{}' ({})", event_info.settings_name, event_info.event_type,
event_info.event_config, strerror(errno), errno);
event_info.event_config, errnoToString(errno), errno);
}
}
@ -477,7 +480,7 @@ void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile
{
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
"Can't read event value from file descriptor {}: '{}' ({})",
fd, strerror(errno), errno);
fd, errnoToString(errno), errno);
current_values[i] = {};
}
}

View File

@ -2,11 +2,14 @@
#include <Common/ThreadProfileEvents.h>
#include <Common/QueryProfiler.h>
#include <Common/ThreadStatus.h>
#include <common/errnoToString.h>
#include <Interpreters/OpenTelemetrySpanLog.h>
#include <Poco/Logger.h>
#include <common/getThreadId.h>
#include <signal.h>
namespace DB
{
@ -21,6 +24,11 @@ namespace ErrorCodes
thread_local ThreadStatus * current_thread = nullptr;
thread_local ThreadStatus * main_thread = nullptr;
#if !defined(SANITIZER) && !defined(ARCADIA_BUILD)
alignas(4096) static thread_local char alt_stack[4096];
static thread_local bool has_alt_stack = false;
#endif
ThreadStatus::ThreadStatus()
: thread_id{getThreadId()}
@ -34,6 +42,46 @@ ThreadStatus::ThreadStatus()
/// NOTE: It is important not to do any non-trivial actions (like updating ProfileEvents or logging) before ThreadStatus is created
/// Otherwise it could lead to SIGSEGV due to current_thread dereferencing
/// Will set alternative signal stack to provide diagnostics for stack overflow errors.
/// If not already installed for current thread.
/// Sanitizer makes larger stack usage and also it's incompatible with alternative stack by default (it sets up and relies on its own).
#if !defined(SANITIZER) && !defined(ARCADIA_BUILD)
if (!has_alt_stack)
{
/// Don't repeat tries even if not installed successfully.
has_alt_stack = true;
/// We have to call 'sigaltstack' before first 'sigaction'. (It does not work other way, for unknown reason).
stack_t altstack_description{};
altstack_description.ss_sp = alt_stack;
altstack_description.ss_flags = 0;
altstack_description.ss_size = sizeof(alt_stack);
if (0 != sigaltstack(&altstack_description, nullptr))
{
LOG_WARNING(log, "Cannot set alternative signal stack for thread, {}", errnoToString(errno));
}
else
{
/// Obtain existing sigaction and modify it by adding a flag.
struct sigaction action{};
if (0 != sigaction(SIGSEGV, nullptr, &action))
{
LOG_WARNING(log, "Cannot obtain previous signal action to set alternative signal stack for thread, {}", errnoToString(errno));
}
else if (!(action.sa_flags & SA_ONSTACK))
{
action.sa_flags |= SA_ONSTACK;
if (0 != sigaction(SIGSEGV, &action, nullptr))
{
LOG_WARNING(log, "Cannot set action with alternative signal stack for thread, {}", errnoToString(errno));
}
}
}
}
#endif
}
ThreadStatus::~ThreadStatus()

View File

@ -28,23 +28,28 @@ struct UInt128
UInt64 low;
UInt64 high;
/// TODO: Make this constexpr. Currently it is used in unions
/// and union cannot contain member with non trivial constructor
/// constructor must be non user provided but compiler cannot constexpr constructor
/// if members low and high are not initialized, if we default member initialize them
/// constructor becomes non trivial.
UInt128() = default;
explicit UInt128(const UInt64 low_, const UInt64 high_) : low(low_), high(high_) {}
explicit constexpr UInt128(const UInt64 low_, const UInt64 high_) : low(low_), high(high_) { }
/// We need Int128 to UInt128 conversion or AccurateComparison will call greaterOp<Int128, UInt64> instead of greaterOp<Int128, UInt128>
explicit UInt128(const Int128 rhs) : low(rhs), high(rhs >> 64) {}
explicit UInt128(const Int64 rhs) : low(rhs), high() {}
explicit UInt128(const Int32 rhs) : low(rhs), high() {}
explicit UInt128(const Int16 rhs) : low(rhs), high() {}
explicit UInt128(const Int8 rhs) : low(rhs), high() {}
explicit UInt128(const UInt8 rhs) : low(rhs), high() {}
explicit UInt128(const UInt16 rhs) : low(rhs), high() {}
explicit UInt128(const UInt32 rhs) : low(rhs), high() {}
explicit UInt128(const UInt64 rhs) : low(rhs), high() {}
explicit UInt128(const Float32 rhs) : low(rhs), high() {}
explicit UInt128(const Float64 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Int128 rhs) : low(rhs), high(rhs >> 64) {}
explicit constexpr UInt128(const Int64 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Int32 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Int16 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Int8 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const UInt8 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const UInt16 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const UInt32 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const UInt64 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Float32 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Float64 rhs) : low(rhs), high() {}
auto tuple() const { return std::tie(high, low); }
constexpr auto tuple() const { return std::tie(high, low); }
String toHexString() const
{
@ -53,31 +58,31 @@ struct UInt128
return res;
}
bool inline operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); }
bool inline operator!= (const UInt128 rhs) const { return tuple() != rhs.tuple(); }
bool inline operator< (const UInt128 rhs) const { return tuple() < rhs.tuple(); }
bool inline operator<= (const UInt128 rhs) const { return tuple() <= rhs.tuple(); }
bool inline operator> (const UInt128 rhs) const { return tuple() > rhs.tuple(); }
bool inline operator>= (const UInt128 rhs) const { return tuple() >= rhs.tuple(); }
constexpr bool operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); }
constexpr bool operator!= (const UInt128 rhs) const { return tuple() != rhs.tuple(); }
constexpr bool operator< (const UInt128 rhs) const { return tuple() < rhs.tuple(); }
constexpr bool operator<= (const UInt128 rhs) const { return tuple() <= rhs.tuple(); }
constexpr bool operator> (const UInt128 rhs) const { return tuple() > rhs.tuple(); }
constexpr bool operator>= (const UInt128 rhs) const { return tuple() >= rhs.tuple(); }
bool inline operator == (const Int128 rhs) const { return *this == UInt128(rhs, rhs >> 64) && rhs >= 0; }
bool inline operator != (const Int128 rhs) const { return *this != UInt128(rhs, rhs >> 64) || rhs < 0; }
bool inline operator >= (const Int128 rhs) const { return *this >= UInt128(rhs, rhs >> 64) || rhs < 0; }
bool inline operator > (const Int128 rhs) const { return *this > UInt128(rhs, rhs >> 64) || rhs < 0; }
bool inline operator <= (const Int128 rhs) const { return *this <= UInt128(rhs, rhs >> 64) && rhs >= 0; }
bool inline operator < (const Int128 rhs) const { return *this < UInt128(rhs, rhs >> 64) && rhs >= 0; }
constexpr bool operator == (const Int128 rhs) const { return *this == UInt128(rhs, rhs >> 64) && rhs >= 0; }
constexpr bool operator != (const Int128 rhs) const { return *this != UInt128(rhs, rhs >> 64) || rhs < 0; }
constexpr bool operator >= (const Int128 rhs) const { return *this >= UInt128(rhs, rhs >> 64) || rhs < 0; }
constexpr bool operator > (const Int128 rhs) const { return *this > UInt128(rhs, rhs >> 64) || rhs < 0; }
constexpr bool operator <= (const Int128 rhs) const { return *this <= UInt128(rhs, rhs >> 64) && rhs >= 0; }
constexpr bool operator < (const Int128 rhs) const { return *this < UInt128(rhs, rhs >> 64) && rhs >= 0; }
bool inline operator > (const Int256 rhs) const { return (rhs < 0) || ((Int256(high) << 64) + low) > rhs; }
bool inline operator > (const UInt256 rhs) const { return ((UInt256(high) << 64) + low) > rhs; }
bool inline operator < (const Int256 rhs) const { return (rhs >= 0) && ((Int256(high) << 64) + low) < rhs; }
bool inline operator < (const UInt256 rhs) const { return ((UInt256(high) << 64) + low) < rhs; }
constexpr bool operator > (const Int256 rhs) const { return (rhs < 0) || ((Int256(high) << 64) + low) > rhs; }
constexpr bool operator > (const UInt256 rhs) const { return ((UInt256(high) << 64) + low) > rhs; }
constexpr bool operator < (const Int256 rhs) const { return (rhs >= 0) && ((Int256(high) << 64) + low) < rhs; }
constexpr bool operator < (const UInt256 rhs) const { return ((UInt256(high) << 64) + low) < rhs; }
template <typename T> bool inline operator== (const T rhs) const { return *this == UInt128(rhs); }
template <typename T> bool inline operator!= (const T rhs) const { return *this != UInt128(rhs); }
template <typename T> bool inline operator>= (const T rhs) const { return *this >= UInt128(rhs); }
template <typename T> bool inline operator> (const T rhs) const { return *this > UInt128(rhs); }
template <typename T> bool inline operator<= (const T rhs) const { return *this <= UInt128(rhs); }
template <typename T> bool inline operator< (const T rhs) const { return *this < UInt128(rhs); }
template <typename T> constexpr bool operator== (const T rhs) const { return *this == UInt128(rhs); }
template <typename T> constexpr bool operator!= (const T rhs) const { return *this != UInt128(rhs); }
template <typename T> constexpr bool operator>= (const T rhs) const { return *this >= UInt128(rhs); }
template <typename T> constexpr bool operator> (const T rhs) const { return *this > UInt128(rhs); }
template <typename T> constexpr bool operator<= (const T rhs) const { return *this <= UInt128(rhs); }
template <typename T> constexpr bool operator< (const T rhs) const { return *this < UInt128(rhs); }
template <typename T> explicit operator T() const
{
@ -91,15 +96,15 @@ struct UInt128
#pragma GCC diagnostic pop
#endif
UInt128 & operator= (const UInt64 rhs) { low = rhs; high = 0; return *this; }
constexpr UInt128 & operator= (const UInt64 rhs) { low = rhs; high = 0; return *this; }
};
template <typename T> bool inline operator == (T a, const UInt128 b) { return b.operator==(a); }
template <typename T> bool inline operator != (T a, const UInt128 b) { return b.operator!=(a); }
template <typename T> bool inline operator >= (T a, const UInt128 b) { return b <= a; }
template <typename T> bool inline operator > (T a, const UInt128 b) { return b < a; }
template <typename T> bool inline operator <= (T a, const UInt128 b) { return b >= a; }
template <typename T> bool inline operator < (T a, const UInt128 b) { return b > a; }
template <typename T> constexpr bool operator == (T a, const UInt128 b) { return b.operator==(a); }
template <typename T> constexpr bool operator != (T a, const UInt128 b) { return b.operator!=(a); }
template <typename T> constexpr bool operator >= (T a, const UInt128 b) { return b <= a; }
template <typename T> constexpr bool operator > (T a, const UInt128 b) { return b < a; }
template <typename T> constexpr bool operator <= (T a, const UInt128 b) { return b >= a; }
template <typename T> constexpr bool operator < (T a, const UInt128 b) { return b > a; }
template <> inline constexpr bool IsNumber<UInt128> = true;
template <> struct TypeName<UInt128> { static constexpr const char * get() { return "UInt128"; } };
@ -246,4 +251,42 @@ template <> struct hash<DB::UInt128>
}
};
template<>
class numeric_limits<DB::UInt128>
{
public:
static constexpr bool is_specialized = true;
static constexpr bool is_signed = ::is_signed<DB::UInt128>::value;
static constexpr bool is_integer = ::is_integer<DB::UInt128>::value;
static constexpr bool is_exact = true;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr std::float_denorm_style has_denorm = std::denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr std::float_round_style round_style = std::round_toward_zero;
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr int digits = std::numeric_limits<UInt64>::digits * 2;
static constexpr int digits10 = digits * 0.30103 /*std::log10(2)*/;
static constexpr int max_digits10 = 0;
static constexpr int radix = 2;
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr DB::UInt128 min() noexcept { return DB::UInt128(0, 0); }
static constexpr DB::UInt128 max() noexcept
{
return DB::UInt128(std::numeric_limits<UInt64>::max(), std::numeric_limits<UInt64>::max());
}
static constexpr DB::UInt128 lowest() noexcept { return min(); }
};
}

View File

@ -798,6 +798,21 @@ void TestKeeperStorage::clearDeadWatches(int64_t session_id)
if (watches_for_path.empty())
watches.erase(watch);
}
auto list_watch = list_watches.find(watch_path);
if (list_watch != list_watches.end())
{
auto & list_watches_for_path = list_watch->second;
for (auto w_it = list_watches_for_path.begin(); w_it != list_watches_for_path.end();)
{
if (w_it->session_id == session_id)
w_it = list_watches_for_path.erase(w_it);
else
++w_it;
}
if (list_watches_for_path.empty())
list_watches.erase(list_watch);
}
}
sessions_and_watchers.erase(watches_it);
}

View File

@ -5,6 +5,7 @@
#cmakedefine01 USE_RE2_ST
#cmakedefine01 USE_SSL
#cmakedefine01 USE_HDFS
#cmakedefine01 USE_INTERNAL_HDFS3_LIBRARY
#cmakedefine01 USE_AWS_S3
#cmakedefine01 USE_BROTLI
#cmakedefine01 USE_UNWIND

View File

@ -7,7 +7,7 @@ ADDINCL (
GLOBAL clickhouse/src
contrib/libs/libcpuid
contrib/libs/libunwind/include
GLOBAL contrib/restricted/ryu
GLOBAL contrib/restricted/dragonbox
)
PEERDIR(
@ -18,7 +18,7 @@ PEERDIR(
contrib/libs/openssl
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/re2
contrib/restricted/ryu
contrib/restricted/dragonbox
)
INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc)

View File

@ -6,7 +6,7 @@ ADDINCL (
GLOBAL clickhouse/src
contrib/libs/libcpuid
contrib/libs/libunwind/include
GLOBAL contrib/restricted/ryu
GLOBAL contrib/restricted/dragonbox
)
PEERDIR(
@ -17,7 +17,7 @@ PEERDIR(
contrib/libs/openssl
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/re2
contrib/restricted/ryu
contrib/restricted/dragonbox
)
INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc)

View File

@ -515,11 +515,32 @@ inline bool NO_SANITIZE_UNDEFINED convertNumeric(From value, To & result)
return true;
}
/// Note that NaNs doesn't compare equal to anything, but they are still in range of any Float type.
if (isNaN(value) && std::is_floating_point_v<To>)
if constexpr (std::is_floating_point_v<From> && std::is_floating_point_v<To>)
{
result = value;
return true;
/// Note that NaNs doesn't compare equal to anything, but they are still in range of any Float type.
if (isNaN(value))
{
result = value;
return true;
}
if (value == std::numeric_limits<From>::infinity())
{
result = std::numeric_limits<To>::infinity();
return true;
}
if (value == -std::numeric_limits<From>::infinity())
{
result = -std::numeric_limits<To>::infinity();
return true;
}
}
if (accurate::greaterOp(value, std::numeric_limits<To>::max())
|| accurate::greaterOp(std::numeric_limits<To>::lowest(), value))
{
return false;
}
result = static_cast<To>(value);

View File

@ -206,27 +206,33 @@ inline typename DecimalType::NativeType getFractionalPart(const DecimalType & de
return getFractionalPartWithScaleMultiplier(decimal, scaleMultiplier<typename DecimalType::NativeType>(scale));
}
/// Decimal to integer/float conversion with the ability to convert into itself (returns the original value);
template <typename To, typename DecimalType>
To convertTo(const DecimalType & decimal, size_t scale)
/// Decimal to integer/float conversion
template <typename To, typename DecimalType, typename ReturnType>
ReturnType convertToImpl(const DecimalType & decimal, size_t scale, To & result)
{
if constexpr (std::is_same_v<To, DecimalType>)
return decimal;
using NativeT = typename DecimalType::NativeType;
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
if constexpr (std::is_floating_point_v<To>)
{
return static_cast<To>(decimal.value) / static_cast<To>(scaleMultiplier<NativeT>(scale));
result = static_cast<To>(decimal.value) / static_cast<To>(scaleMultiplier<NativeT>(scale));
}
else if constexpr (is_integer_v<To> && (sizeof(To) >= sizeof(NativeT)))
{
NativeT whole = getWholePart(decimal, scale);
if constexpr (is_unsigned_v<To>)
{
if (whole < 0)
throw Exception("Convert overflow", ErrorCodes::DECIMAL_OVERFLOW);
return static_cast<To>(whole);
{
if constexpr (throw_exception)
throw Exception("Convert overflow", ErrorCodes::DECIMAL_OVERFLOW);
else
return ReturnType(true);
}
}
result = static_cast<To>(whole);
}
else if constexpr (is_integer_v<To>)
{
@ -239,9 +245,34 @@ To convertTo(const DecimalType & decimal, size_t scale)
static const constexpr CastTo max_to = std::numeric_limits<ToNativeT>::max();
if (whole < min_to || whole > max_to)
throw Exception("Convert overflow", ErrorCodes::DECIMAL_OVERFLOW);
return static_cast<CastTo>(whole);
{
if constexpr (throw_exception)
throw Exception("Convert overflow", ErrorCodes::DECIMAL_OVERFLOW);
else
return ReturnType(true);
}
result = static_cast<CastTo>(whole);
}
return ReturnType(true);
}
template <typename To, typename DecimalType>
To convertTo(const DecimalType & decimal, size_t scale)
{
To result;
convertToImpl<To, DecimalType, void>(decimal, scale, result);
return result;
}
template <typename To, typename DecimalType>
bool tryConvertTo(const DecimalType & decimal, size_t scale, To & result)
{
return convertToImpl<To, DecimalType, bool>(decimal, scale, result);
}
template <bool is_multiply, bool is_division, typename T, typename U, template <typename> typename DecimalType>

View File

@ -17,6 +17,63 @@ namespace ErrorCodes
extern const int DECIMAL_OVERFLOW;
}
inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
{
switch (type)
{
case Field::Types::Null: {
return DB::Field();
}
case Field::Types::UInt64: {
UInt64 value;
DB::readVarUInt(value, buf);
return value;
}
case Field::Types::UInt128: {
UInt128 value;
DB::readBinary(value, buf);
return value;
}
case Field::Types::Int64: {
Int64 value;
DB::readVarInt(value, buf);
return value;
}
case Field::Types::Float64: {
Float64 value;
DB::readFloatBinary(value, buf);
return value;
}
case Field::Types::String: {
std::string value;
DB::readStringBinary(value, buf);
return value;
}
case Field::Types::Array: {
Array value;
DB::readBinary(value, buf);
return value;
}
case Field::Types::Tuple: {
Tuple value;
DB::readBinary(value, buf);
return value;
}
case Field::Types::Map: {
Map value;
DB::readBinary(value, buf);
return value;
}
case Field::Types::AggregateFunctionState: {
AggregateFunctionStateData value;
DB::readStringBinary(value.name, buf);
DB::readStringBinary(value.data, buf);
return value;
}
}
return DB::Field();
}
void readBinary(Array & x, ReadBuffer & buf)
{
size_t size;
@ -25,73 +82,7 @@ void readBinary(Array & x, ReadBuffer & buf)
DB::readBinary(size, buf);
for (size_t index = 0; index < size; ++index)
{
switch (type)
{
case Field::Types::Null:
{
x.push_back(DB::Field());
break;
}
case Field::Types::UInt64:
{
UInt64 value;
DB::readVarUInt(value, buf);
x.push_back(value);
break;
}
case Field::Types::UInt128:
{
UInt128 value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::Int64:
{
Int64 value;
DB::readVarInt(value, buf);
x.push_back(value);
break;
}
case Field::Types::Float64:
{
Float64 value;
DB::readFloatBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::String:
{
std::string value;
DB::readStringBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::Array:
{
Array value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::Tuple:
{
Tuple value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::AggregateFunctionState:
{
AggregateFunctionStateData value;
DB::readStringBinary(value.name, buf);
DB::readStringBinary(value.data, buf);
x.push_back(value);
break;
}
}
}
x.push_back(getBinaryValue(type, buf));
}
void writeBinary(const Array & x, WriteBuffer & buf)
@ -104,53 +95,7 @@ void writeBinary(const Array & x, WriteBuffer & buf)
DB::writeBinary(size, buf);
for (const auto & elem : x)
{
switch (type)
{
case Field::Types::Null: break;
case Field::Types::UInt64:
{
DB::writeVarUInt(get<UInt64>(elem), buf);
break;
}
case Field::Types::UInt128:
{
DB::writeBinary(get<UInt128>(elem), buf);
break;
}
case Field::Types::Int64:
{
DB::writeVarInt(get<Int64>(elem), buf);
break;
}
case Field::Types::Float64:
{
DB::writeFloatBinary(get<Float64>(elem), buf);
break;
}
case Field::Types::String:
{
DB::writeStringBinary(get<std::string>(elem), buf);
break;
}
case Field::Types::Array:
{
DB::writeBinary(get<Array>(elem), buf);
break;
}
case Field::Types::Tuple:
{
DB::writeBinary(get<Tuple>(elem), buf);
break;
}
case Field::Types::AggregateFunctionState:
{
DB::writeStringBinary(elem.get<AggregateFunctionStateData>().name, buf);
DB::writeStringBinary(elem.get<AggregateFunctionStateData>().data, buf);
break;
}
}
}
Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, elem);
}
void writeText(const Array & x, WriteBuffer & buf)
@ -168,93 +113,7 @@ void readBinary(Tuple & x, ReadBuffer & buf)
{
UInt8 type;
DB::readBinary(type, buf);
switch (type)
{
case Field::Types::Null:
{
x.push_back(DB::Field());
break;
}
case Field::Types::UInt64:
{
UInt64 value;
DB::readVarUInt(value, buf);
x.push_back(value);
break;
}
case Field::Types::UInt128:
{
UInt128 value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::Int64:
{
Int64 value;
DB::readVarInt(value, buf);
x.push_back(value);
break;
}
case Field::Types::Int128:
{
Int64 value;
DB::readVarInt(value, buf);
x.push_back(value);
break;
}
case Field::Types::Float64:
{
Float64 value;
DB::readFloatBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::String:
{
std::string value;
DB::readStringBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::UInt256:
{
UInt256 value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::Int256:
{
Int256 value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::Array:
{
Array value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::Tuple:
{
Tuple value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::AggregateFunctionState:
{
AggregateFunctionStateData value;
DB::readStringBinary(value.name, buf);
DB::readStringBinary(value.data, buf);
x.push_back(value);
break;
}
}
x.push_back(getBinaryValue(type, buf));
}
}
@ -267,67 +126,7 @@ void writeBinary(const Tuple & x, WriteBuffer & buf)
{
const UInt8 type = elem.getType();
DB::writeBinary(type, buf);
switch (type)
{
case Field::Types::Null: break;
case Field::Types::UInt64:
{
DB::writeVarUInt(get<UInt64>(elem), buf);
break;
}
case Field::Types::UInt128:
{
DB::writeBinary(get<UInt128>(elem), buf);
break;
}
case Field::Types::Int64:
{
DB::writeVarInt(get<Int64>(elem), buf);
break;
}
case Field::Types::Int128:
{
DB::writeVarInt(get<Int64>(elem), buf);
break;
}
case Field::Types::Float64:
{
DB::writeFloatBinary(get<Float64>(elem), buf);
break;
}
case Field::Types::String:
{
DB::writeStringBinary(get<std::string>(elem), buf);
break;
}
case Field::Types::UInt256:
{
DB::writeBinary(get<UInt256>(elem), buf);
break;
}
case Field::Types::Int256:
{
DB::writeBinary(get<Int256>(elem), buf);
break;
}
case Field::Types::Array:
{
DB::writeBinary(get<Array>(elem), buf);
break;
}
case Field::Types::Tuple:
{
DB::writeBinary(get<Tuple>(elem), buf);
break;
}
case Field::Types::AggregateFunctionState:
{
DB::writeStringBinary(elem.get<AggregateFunctionStateData>().name, buf);
DB::writeStringBinary(elem.get<AggregateFunctionStateData>().data, buf);
break;
}
}
Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, elem);
}
}
@ -336,6 +135,37 @@ void writeText(const Tuple & x, WriteBuffer & buf)
writeFieldText(DB::Field(x), buf);
}
void readBinary(Map & x, ReadBuffer & buf)
{
size_t size;
DB::readBinary(size, buf);
for (size_t index = 0; index < size; ++index)
{
UInt8 type;
DB::readBinary(type, buf);
x.push_back(getBinaryValue(type, buf));
}
}
void writeBinary(const Map & x, WriteBuffer & buf)
{
const size_t size = x.size();
DB::writeBinary(size, buf);
for (const auto & elem : x)
{
const UInt8 type = elem.getType();
DB::writeBinary(type, buf);
Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, elem);
}
}
void writeText(const Map & x, WriteBuffer & buf)
{
writeFieldText(DB::Field(x), buf);
}
template <typename T>
void readQuoted(DecimalField<T> & x, ReadBuffer & buf)
{
@ -530,6 +360,30 @@ Field Field::restoreFromDump(const std::string_view & dump_)
return tuple;
}
prefix = std::string_view{"Map_("};
if (dump.starts_with(prefix))
{
std::string_view tail = dump.substr(prefix.length());
trimLeft(tail);
Map map;
while (tail != ")")
{
size_t separator = tail.find_first_of(",)");
if (separator == std::string_view::npos)
show_error();
bool comma = (tail[separator] == ',');
std::string_view element = tail.substr(0, separator);
tail.remove_prefix(separator);
if (comma)
tail.remove_prefix(1);
trimLeft(tail);
if (!comma && tail != ")")
show_error();
map.push_back(Field::restoreFromDump(element));
}
return map;
}
prefix = std::string_view{"AggregateFunctionState_("};
if (dump.starts_with(prefix))
{

View File

@ -51,6 +51,9 @@ struct X : public FieldVector \
DEFINE_FIELD_VECTOR(Array);
DEFINE_FIELD_VECTOR(Tuple);
/// An array with the following structure: [(key1, value1), (key2, value2), ...]
DEFINE_FIELD_VECTOR(Map);
#undef DEFINE_FIELD_VECTOR
struct AggregateFunctionStateData
@ -206,6 +209,7 @@ template <> struct NearestFieldTypeImpl<std::string_view> { using Type = String;
template <> struct NearestFieldTypeImpl<String> { using Type = String; };
template <> struct NearestFieldTypeImpl<Array> { using Type = Array; };
template <> struct NearestFieldTypeImpl<Tuple> { using Type = Tuple; };
template <> struct NearestFieldTypeImpl<Map> { using Type = Map; };
template <> struct NearestFieldTypeImpl<bool> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<Null> { using Type = Null; };
@ -259,6 +263,7 @@ public:
Decimal256 = 23,
UInt256 = 24,
Int256 = 25,
Map = 26,
};
static const int MIN_NON_POD = 16;
@ -276,6 +281,7 @@ public:
case String: return "String";
case Array: return "Array";
case Tuple: return "Tuple";
case Map: return "Map";
case Decimal32: return "Decimal32";
case Decimal64: return "Decimal64";
case Decimal128: return "Decimal128";
@ -464,6 +470,7 @@ public:
case Types::String: return get<String>() < rhs.get<String>();
case Types::Array: return get<Array>() < rhs.get<Array>();
case Types::Tuple: return get<Tuple>() < rhs.get<Tuple>();
case Types::Map: return get<Map>() < rhs.get<Map>();
case Types::Decimal32: return get<DecimalField<Decimal32>>() < rhs.get<DecimalField<Decimal32>>();
case Types::Decimal64: return get<DecimalField<Decimal64>>() < rhs.get<DecimalField<Decimal64>>();
case Types::Decimal128: return get<DecimalField<Decimal128>>() < rhs.get<DecimalField<Decimal128>>();
@ -499,6 +506,7 @@ public:
case Types::String: return get<String>() <= rhs.get<String>();
case Types::Array: return get<Array>() <= rhs.get<Array>();
case Types::Tuple: return get<Tuple>() <= rhs.get<Tuple>();
case Types::Map: return get<Map>() <= rhs.get<Map>();
case Types::Decimal32: return get<DecimalField<Decimal32>>() <= rhs.get<DecimalField<Decimal32>>();
case Types::Decimal64: return get<DecimalField<Decimal64>>() <= rhs.get<DecimalField<Decimal64>>();
case Types::Decimal128: return get<DecimalField<Decimal128>>() <= rhs.get<DecimalField<Decimal128>>();
@ -536,6 +544,7 @@ public:
case Types::String: return get<String>() == rhs.get<String>();
case Types::Array: return get<Array>() == rhs.get<Array>();
case Types::Tuple: return get<Tuple>() == rhs.get<Tuple>();
case Types::Map: return get<Map>() == rhs.get<Map>();
case Types::UInt128: return get<UInt128>() == rhs.get<UInt128>();
case Types::Int128: return get<Int128>() == rhs.get<Int128>();
case Types::Decimal32: return get<DecimalField<Decimal32>>() == rhs.get<DecimalField<Decimal32>>();
@ -575,6 +584,7 @@ public:
case Types::String: return f(field.template get<String>());
case Types::Array: return f(field.template get<Array>());
case Types::Tuple: return f(field.template get<Tuple>());
case Types::Map: return f(field.template get<Map>());
case Types::Decimal32: return f(field.template get<DecimalField<Decimal32>>());
case Types::Decimal64: return f(field.template get<DecimalField<Decimal64>>());
case Types::Decimal128: return f(field.template get<DecimalField<Decimal128>>());
@ -600,7 +610,7 @@ public:
private:
std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which),
Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple,
Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple, Map,
DecimalField<Decimal32>, DecimalField<Decimal64>, DecimalField<Decimal128>, DecimalField<Decimal256>,
AggregateFunctionStateData,
UInt256, Int256
@ -699,6 +709,9 @@ private:
case Types::Tuple:
destroy<Tuple>();
break;
case Types::Map:
destroy<Map>();
break;
case Types::AggregateFunctionState:
destroy<AggregateFunctionStateData>();
break;
@ -729,6 +742,7 @@ template <> struct Field::TypeToEnum<Float64> { static const Types::Which value
template <> struct Field::TypeToEnum<String> { static const Types::Which value = Types::String; };
template <> struct Field::TypeToEnum<Array> { static const Types::Which value = Types::Array; };
template <> struct Field::TypeToEnum<Tuple> { static const Types::Which value = Types::Tuple; };
template <> struct Field::TypeToEnum<Map> { static const Types::Which value = Types::Map; };
template <> struct Field::TypeToEnum<DecimalField<Decimal32>>{ static const Types::Which value = Types::Decimal32; };
template <> struct Field::TypeToEnum<DecimalField<Decimal64>>{ static const Types::Which value = Types::Decimal64; };
template <> struct Field::TypeToEnum<DecimalField<Decimal128>>{ static const Types::Which value = Types::Decimal128; };
@ -747,6 +761,7 @@ template <> struct Field::EnumToType<Field::Types::Float64> { using Type = Float
template <> struct Field::EnumToType<Field::Types::String> { using Type = String; };
template <> struct Field::EnumToType<Field::Types::Array> { using Type = Array; };
template <> struct Field::EnumToType<Field::Types::Tuple> { using Type = Tuple; };
template <> struct Field::EnumToType<Field::Types::Map> { using Type = Map; };
template <> struct Field::EnumToType<Field::Types::Decimal32> { using Type = DecimalField<Decimal32>; };
template <> struct Field::EnumToType<Field::Types::Decimal64> { using Type = DecimalField<Decimal64>; };
template <> struct Field::EnumToType<Field::Types::Decimal128> { using Type = DecimalField<Decimal128>; };
@ -814,6 +829,7 @@ T safeGet(Field & field)
template <> struct TypeName<Array> { static std::string get() { return "Array"; } };
template <> struct TypeName<Tuple> { static std::string get() { return "Tuple"; } };
template <> struct TypeName<Map> { static std::string get() { return "Map"; } };
template <> struct TypeName<AggregateFunctionStateData> { static std::string get() { return "AggregateFunctionState"; } };
template <typename T>
@ -900,6 +916,12 @@ void writeBinary(const Tuple & x, WriteBuffer & buf);
void writeText(const Tuple & x, WriteBuffer & buf);
void readBinary(Map & x, ReadBuffer & buf);
[[noreturn]] inline void readText(Map &, ReadBuffer &) { throw Exception("Cannot read Map.", ErrorCodes::NOT_IMPLEMENTED); }
[[noreturn]] inline void readQuoted(Map &, ReadBuffer &) { throw Exception("Cannot read Map.", ErrorCodes::NOT_IMPLEMENTED); }
void writeBinary(const Map & x, WriteBuffer & buf);
void writeText(const Map & x, WriteBuffer & buf);
[[noreturn]] inline void writeQuoted(const Map &, WriteBuffer &) { throw Exception("Cannot write Map quoted.", ErrorCodes::NOT_IMPLEMENTED); }
__attribute__ ((noreturn)) inline void writeText(const AggregateFunctionStateData &, WriteBuffer &)
{

View File

@ -16,10 +16,8 @@ namespace ErrorCodes
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
}
IMPLEMENT_SETTINGS_TRAITS(SettingsTraits, LIST_OF_SETTINGS)
/** Set the settings from the profile (in the server configuration, many settings can be listed in one profile).
* The profile can also be set using the `set` functions, like the `profile` setting.
*/

View File

@ -166,7 +166,7 @@ class IColumn;
\
M(UInt64, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled.", 0) \
M(Milliseconds, insert_quorum_timeout, 600000, "", 0) \
M(Bool, insert_quorum_parallel, false, "For quorum INSERT queries - enable to make parallel inserts without linearizability", 0) \
M(Bool, insert_quorum_parallel, true, "For quorum INSERT queries - enable to make parallel inserts without linearizability", 0) \
M(UInt64, select_sequential_consistency, 0, "For SELECT queries from the replicated table, throw an exception if the replica does not have a chunk written with the quorum; do not read the parts that have not yet been written with the quorum.", 0) \
M(UInt64, table_function_remote_max_addresses, 1000, "The maximum number of different shards and the maximum number of replicas of one shard in the `remote` function.", 0) \
M(Milliseconds, read_backoff_min_latency_ms, 1000, "Setting to reduce the number of threads in case of slow reads. Pay attention only to reads that took at least that much time.", 0) \
@ -239,6 +239,8 @@ class IColumn;
* Almost all limits apply to each stream individually. \
*/ \
\
M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \
M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \
M(UInt64, max_rows_to_read, 0, "Limit on read rows from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \
M(UInt64, max_bytes_to_read, 0, "Limit on read bytes (after decompression) from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \
M(OverflowMode, read_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
@ -402,6 +404,7 @@ class IColumn;
M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \
M(Bool, optimize_skip_merged_partitions, false, "Skip partitions with one part with level > 0 in optimize final", 0) \
M(Bool, optimize_on_insert, true, "Do the same transformation for inserted block of data as if merge was done on this block.", 0) \
M(Bool, allow_experimental_map_type, false, "Allow data type Map", 0) \
\
M(Bool, use_antlr_parser, false, "Parse incoming queries using ANTLR-generated parser", 0) \
\

View File

@ -56,6 +56,7 @@ enum class TypeIndex
Function,
AggregateFunction,
LowCardinality,
Map,
};
#if !__clang__
#pragma GCC diagnostic pop
@ -267,6 +268,7 @@ inline constexpr const char * getTypeName(TypeIndex idx)
case TypeIndex::Function: return "Function";
case TypeIndex::AggregateFunction: return "AggregateFunction";
case TypeIndex::LowCardinality: return "LowCardinality";
case TypeIndex::Map: return "Map";
}
__builtin_unreachable();

View File

@ -3,6 +3,7 @@
#include <Client/ConnectionPool.h>
#include <Client/MultiplexedConnections.h>
#include <Storages/IStorage_fwd.h>
#include <Interpreters/Context.h>
#include <Interpreters/StorageID.h>
namespace DB
@ -93,7 +94,7 @@ private:
const String query;
String query_id = "";
const Context & context;
Context context;
ProgressCallback progress_callback;
ProfileInfoCallback profile_info_callback;

View File

@ -25,10 +25,19 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
static const std::vector<String> supported_functions{"any", "anyLast", "min",
"max", "sum", "sumWithOverflow", "groupBitAnd", "groupBitOr", "groupBitXor",
"sumMap", "minMap", "maxMap", "groupArrayArray", "groupUniqArrayArray"};
void DataTypeCustomSimpleAggregateFunction::checkSupportedFunctions(const AggregateFunctionPtr & function)
{
static const std::vector<String> supported_functions{"any", "anyLast", "min",
"max", "sum", "sumWithOverflow", "groupBitAnd", "groupBitOr", "groupBitXor",
"sumMap", "minMap", "maxMap", "groupArrayArray", "groupUniqArrayArray"};
// check function
if (std::find(std::begin(supported_functions), std::end(supported_functions), function->getName()) == std::end(supported_functions))
{
throw Exception("Unsupported aggregate function " + function->getName() + ", supported functions are " + boost::algorithm::join(supported_functions, ","),
ErrorCodes::BAD_ARGUMENTS);
}
}
String DataTypeCustomSimpleAggregateFunction::getName() const
{
@ -114,12 +123,7 @@ static std::pair<DataTypePtr, DataTypeCustomDescPtr> create(const ASTPtr & argum
AggregateFunctionProperties properties;
function = AggregateFunctionFactory::instance().get(function_name, argument_types, params_row, properties);
// check function
if (std::find(std::begin(supported_functions), std::end(supported_functions), function->getName()) == std::end(supported_functions))
{
throw Exception("Unsupported aggregate function " + function->getName() + ", supported functions are " + boost::algorithm::join(supported_functions, ","),
ErrorCodes::BAD_ARGUMENTS);
}
DataTypeCustomSimpleAggregateFunction::checkSupportedFunctions(function);
DataTypePtr storage_type = DataTypeFactory::instance().get(argument_types[0]->getName());

View File

@ -37,6 +37,7 @@ public:
const AggregateFunctionPtr getFunction() const { return function; }
String getName() const override;
static void checkSupportedFunctions(const AggregateFunctionPtr & function);
};
}

View File

@ -180,6 +180,7 @@ DataTypeFactory::DataTypeFactory()
registerDataTypeDomainIPv4AndIPv6(*this);
registerDataTypeDomainSimpleAggregateFunction(*this);
registerDataTypeDomainGeo(*this);
registerDataTypeMap(*this);
}
DataTypeFactory & DataTypeFactory::instance()

View File

@ -73,6 +73,7 @@ void registerDataTypeFixedString(DataTypeFactory & factory);
void registerDataTypeEnum(DataTypeFactory & factory);
void registerDataTypeArray(DataTypeFactory & factory);
void registerDataTypeTuple(DataTypeFactory & factory);
void registerDataTypeMap(DataTypeFactory & factory);
void registerDataTypeNullable(DataTypeFactory & factory);
void registerDataTypeNothing(DataTypeFactory & factory);
void registerDataTypeUUID(DataTypeFactory & factory);

View File

@ -0,0 +1,375 @@
#include <Common/StringUtils/StringUtils.h>
#include <Columns/ColumnMap.h>
#include <Columns/ColumnArray.h>
#include <Core/Field.h>
#include <Formats/FormatSettings.h>
#include <DataTypes/DataTypeMap.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeFactory.h>
#include <Parsers/IAST.h>
#include <Parsers/ASTNameTypePair.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include <Common/quoteString.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/ReadBufferFromString.h>
#include <IO/Operators.h>
#include <ext/map.h>
#include <ext/enumerate.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int CANNOT_READ_MAP_FROM_TEXT;
}
DataTypeMap::DataTypeMap(const DataTypes & elems_)
{
assert(elems_.size() == 2);
key_type = elems_[0];
value_type = elems_[1];
nested = std::make_shared<DataTypeArray>(
std::make_shared<DataTypeTuple>(DataTypes{key_type, value_type}, Names{"keys", "values"}));
}
DataTypeMap::DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & value_type_)
: key_type(key_type_), value_type(value_type_)
, nested(std::make_shared<DataTypeArray>(
std::make_shared<DataTypeTuple>(DataTypes{key_type_, value_type_}, Names{"keys", "values"}))) {}
std::string DataTypeMap::doGetName() const
{
WriteBufferFromOwnString s;
s << "Map(" << key_type->getName() << "," << value_type->getName() << ")";
return s.str();
}
static const IColumn & extractNestedColumn(const IColumn & column)
{
return assert_cast<const ColumnMap &>(column).getNestedColumn();
}
static IColumn & extractNestedColumn(IColumn & column)
{
return assert_cast<ColumnMap &>(column).getNestedColumn();
}
void DataTypeMap::serializeBinary(const Field & field, WriteBuffer & ostr) const
{
const auto & map = get<const Map &>(field);
writeVarUInt(map.size(), ostr);
for (const auto & elem : map)
{
const auto & tuple = elem.safeGet<const Tuple>();
assert(tuple.size() == 2);
key_type->serializeBinary(tuple[0], ostr);
value_type->serializeBinary(tuple[1], ostr);
}
}
void DataTypeMap::deserializeBinary(Field & field, ReadBuffer & istr) const
{
size_t size;
readVarUInt(size, istr);
field = Map(size);
for (auto & elem : field.get<Map &>())
{
Tuple tuple(2);
key_type->deserializeBinary(tuple[0], istr);
value_type->deserializeBinary(tuple[1], istr);
elem = std::move(tuple);
}
}
void DataTypeMap::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
{
nested->serializeBinary(extractNestedColumn(column), row_num, ostr);
}
void DataTypeMap::deserializeBinary(IColumn & column, ReadBuffer & istr) const
{
nested->deserializeBinary(extractNestedColumn(column), istr);
}
template <typename Writer>
void DataTypeMap::serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, Writer && writer) const
{
const auto & column_map = assert_cast<const ColumnMap &>(column);
const auto & nested_array = column_map.getNestedColumn();
const auto & nested_tuple = column_map.getNestedData();
const auto & offsets = nested_array.getOffsets();
size_t offset = offsets[row_num - 1];
size_t next_offset = offsets[row_num];
writeChar('{', ostr);
for (size_t i = offset; i < next_offset; ++i)
{
if (i != offset)
writeChar(',', ostr);
writer(key_type, nested_tuple.getColumn(0), i);
writeChar(':', ostr);
writer(value_type, nested_tuple.getColumn(1), i);
}
writeChar('}', ostr);
}
template <typename Reader>
void DataTypeMap::deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && reader) const
{
auto & column_map = assert_cast<ColumnMap &>(column);
auto & nested_array = column_map.getNestedColumn();
auto & nested_tuple = column_map.getNestedData();
auto & offsets = nested_array.getOffsets();
auto & key_column = nested_tuple.getColumn(0);
auto & value_column = nested_tuple.getColumn(1);
size_t size = 0;
assertChar('{', istr);
try
{
bool first = true;
while (!istr.eof() && *istr.position() != '}')
{
if (!first)
{
if (*istr.position() == ',')
++istr.position();
else
throw Exception("Cannot read Map from text", ErrorCodes::CANNOT_READ_MAP_FROM_TEXT);
}
first = false;
skipWhitespaceIfAny(istr);
if (*istr.position() == '}')
break;
if (need_safe_get_int_key)
{
ReadBuffer::Position tmp = istr.position();
while (*tmp != ':' && *tmp != '}')
++tmp;
*tmp = ' ';
reader(key_type, key_column);
}
else
{
reader(key_type, key_column);
skipWhitespaceIfAny(istr);
assertChar(':', istr);
}
++size;
skipWhitespaceIfAny(istr);
reader(value_type, value_column);
skipWhitespaceIfAny(istr);
}
offsets.push_back(offsets.back() + size);
assertChar('}', istr);
}
catch (...)
{
throw;
}
}
void DataTypeMap::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
serializeTextImpl(column, row_num, ostr,
[&](const DataTypePtr & subcolumn_type, const IColumn & subcolumn, size_t pos)
{
subcolumn_type->serializeAsTextQuoted(subcolumn, pos, ostr, settings);
});
}
void DataTypeMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
// need_safe_get_int_key is set for Interger to prevent to readIntTextUnsafe
bool need_safe_get_int_key = isInteger(key_type);
deserializeTextImpl(column, istr, need_safe_get_int_key,
[&](const DataTypePtr & subcolumn_type, IColumn & subcolumn)
{
subcolumn_type->deserializeAsTextQuoted(subcolumn, istr, settings);
});
}
void DataTypeMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
serializeTextImpl(column, row_num, ostr,
[&](const DataTypePtr & subcolumn_type, const IColumn & subcolumn, size_t pos)
{
subcolumn_type->serializeAsTextJSON(subcolumn, pos, ostr, settings);
});
}
void DataTypeMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
// need_safe_get_int_key is set for Interger to prevent to readIntTextUnsafe
bool need_safe_get_int_key = isInteger(key_type);
deserializeTextImpl(column, istr, need_safe_get_int_key,
[&](const DataTypePtr & subcolumn_type, IColumn & subcolumn)
{
subcolumn_type->deserializeAsTextJSON(subcolumn, istr, settings);
});
}
void DataTypeMap::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
const auto & column_map = assert_cast<const ColumnMap &>(column);
const auto & offsets = column_map.getNestedColumn().getOffsets();
size_t offset = offsets[row_num - 1];
size_t next_offset = offsets[row_num];
const auto & nested_data = column_map.getNestedData();
writeCString("<map>", ostr);
for (size_t i = offset; i < next_offset; ++i)
{
writeCString("<elem>", ostr);
writeCString("<key>", ostr);
key_type->serializeAsTextXML(nested_data.getColumn(0), i, ostr, settings);
writeCString("</key>", ostr);
writeCString("<value>", ostr);
value_type->serializeAsTextXML(nested_data.getColumn(1), i, ostr, settings);
writeCString("</value>", ostr);
writeCString("</elem>", ostr);
}
writeCString("</map>", ostr);
}
void DataTypeMap::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
WriteBufferFromOwnString wb;
serializeText(column, row_num, wb, settings);
writeCSV(wb.str(), ostr);
}
void DataTypeMap::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
String s;
readCSV(s, istr, settings.csv);
ReadBufferFromString rb(s);
deserializeText(column, rb, settings);
}
void DataTypeMap::enumerateStreams(const StreamCallback & callback, SubstreamPath & path) const
{
nested->enumerateStreams(callback, path);
}
void DataTypeMap::serializeBinaryBulkStatePrefix(
SerializeBinaryBulkSettings & settings,
SerializeBinaryBulkStatePtr & state) const
{
nested->serializeBinaryBulkStatePrefix(settings, state);
}
void DataTypeMap::serializeBinaryBulkStateSuffix(
SerializeBinaryBulkSettings & settings,
SerializeBinaryBulkStatePtr & state) const
{
nested->serializeBinaryBulkStateSuffix(settings, state);
}
void DataTypeMap::deserializeBinaryBulkStatePrefix(
DeserializeBinaryBulkSettings & settings,
DeserializeBinaryBulkStatePtr & state) const
{
nested->deserializeBinaryBulkStatePrefix(settings, state);
}
void DataTypeMap::serializeBinaryBulkWithMultipleStreams(
const IColumn & column,
size_t offset,
size_t limit,
SerializeBinaryBulkSettings & settings,
SerializeBinaryBulkStatePtr & state) const
{
nested->serializeBinaryBulkWithMultipleStreams(extractNestedColumn(column), offset, limit, settings, state);
}
void DataTypeMap::deserializeBinaryBulkWithMultipleStreams(
IColumn & column,
size_t limit,
DeserializeBinaryBulkSettings & settings,
DeserializeBinaryBulkStatePtr & state) const
{
nested->deserializeBinaryBulkWithMultipleStreams(extractNestedColumn(column), limit, settings, state);
}
void DataTypeMap::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const
{
nested->serializeProtobuf(extractNestedColumn(column), row_num, protobuf, value_index);
}
void DataTypeMap::deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const
{
nested->deserializeProtobuf(extractNestedColumn(column), protobuf, allow_add_row, row_added);
}
MutableColumnPtr DataTypeMap::createColumn() const
{
return ColumnMap::create(nested->createColumn());
}
Field DataTypeMap::getDefault() const
{
return Map();
}
bool DataTypeMap::equals(const IDataType & rhs) const
{
if (typeid(rhs) != typeid(*this))
return false;
const DataTypeMap & rhs_map = static_cast<const DataTypeMap &>(rhs);
return nested->equals(*rhs_map.nested);
}
static DataTypePtr create(const ASTPtr & arguments)
{
if (!arguments || arguments->children.size() != 2)
throw Exception("Map data type family must have two arguments: key and value types", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
DataTypes nested_types;
nested_types.reserve(arguments->children.size());
for (const ASTPtr & child : arguments->children)
nested_types.emplace_back(DataTypeFactory::instance().get(child));
return std::make_shared<DataTypeMap>(nested_types);
}
void registerDataTypeMap(DataTypeFactory & factory)
{
factory.registerDataType("Map", create);
}
}

103
src/DataTypes/DataTypeMap.h Normal file
View File

@ -0,0 +1,103 @@
#pragma once
#include <DataTypes/DataTypeWithSimpleSerialization.h>
namespace DB
{
/** Map data type.
* Map is implemented as two arrays of keys and values.
* Serialization of type 'Map(K, V)' is similar to serialization.
* of 'Array(Tuple(keys K, values V))' or in other words of 'Nested(keys K, valuev V)'.
*/
class DataTypeMap final : public DataTypeWithSimpleSerialization
{
private:
DataTypePtr key_type;
DataTypePtr value_type;
/// 'nested' is an Array(Tuple(key_type, value_type))
DataTypePtr nested;
public:
static constexpr bool is_parametric = true;
DataTypeMap(const DataTypes & elems);
DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & value_type_);
TypeIndex getTypeId() const override { return TypeIndex::Map; }
std::string doGetName() const override;
const char * getFamilyName() const override { return "Map"; }
bool canBeInsideNullable() const override { return false; }
void serializeBinary(const Field & field, WriteBuffer & ostr) const override;
void deserializeBinary(Field & field, ReadBuffer & istr) const override;
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
/** Each sub-column in a map is serialized in separate stream.
*/
void enumerateStreams(const StreamCallback & callback, SubstreamPath & path) const override;
void serializeBinaryBulkStatePrefix(
SerializeBinaryBulkSettings & settings,
SerializeBinaryBulkStatePtr & state) const override;
void serializeBinaryBulkStateSuffix(
SerializeBinaryBulkSettings & settings,
SerializeBinaryBulkStatePtr & state) const override;
void deserializeBinaryBulkStatePrefix(
DeserializeBinaryBulkSettings & settings,
DeserializeBinaryBulkStatePtr & state) const override;
void serializeBinaryBulkWithMultipleStreams(
const IColumn & column,
size_t offset,
size_t limit,
SerializeBinaryBulkSettings & settings,
SerializeBinaryBulkStatePtr & state) const override;
void deserializeBinaryBulkWithMultipleStreams(
IColumn & column,
size_t limit,
DeserializeBinaryBulkSettings & settings,
DeserializeBinaryBulkStatePtr & state) const override;
void serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const override;
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override;
Field getDefault() const override;
bool equals(const IDataType & rhs) const override;
bool isComparable() const override { return key_type->isComparable() && value_type->isComparable(); }
bool isParametric() const override { return true; }
bool haveSubtypes() const override { return true; }
const DataTypePtr & getKeyType() const { return key_type; }
const DataTypePtr & getValueType() const { return value_type; }
DataTypes getKeyValueTypes() const { return {key_type, value_type}; }
private:
template <typename Writer>
void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, Writer && writer) const;
template <typename Reader>
void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && reader) const;
};
}

View File

@ -96,22 +96,29 @@ inline UInt32 getDecimalScale(const DataTypeDecimal<T> & data_type)
return data_type.getScale();
}
template <typename FromDataType, typename ToDataType>
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>, typename ToDataType::FieldType>
convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to)
template <typename FromDataType, typename ToDataType, typename ReturnType = void>
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>, ReturnType>
convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType& result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>;
using MaxNativeType = typename MaxFieldType::NativeType;
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
MaxNativeType converted_value;
if (scale_to > scale_from)
{
converted_value = DecimalUtils::scaleMultiplier<MaxNativeType>(scale_to - scale_from);
if (common::mulOverflow(static_cast<MaxNativeType>(value.value), converted_value, converted_value))
throw Exception(std::string(ToDataType::family_name) + " convert overflow",
ErrorCodes::DECIMAL_OVERFLOW);
{
if constexpr (throw_exception)
throw Exception(std::string(ToDataType::family_name) + " convert overflow",
ErrorCodes::DECIMAL_OVERFLOW);
else
return ReturnType(false);
}
}
else
converted_value = value.value / DecimalUtils::scaleMultiplier<MaxNativeType>(scale_from - scale_to);
@ -120,35 +127,87 @@ convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_fro
{
if (converted_value < std::numeric_limits<typename ToFieldType::NativeType>::min() ||
converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
throw Exception(std::string(ToDataType::family_name) + " convert overflow",
ErrorCodes::DECIMAL_OVERFLOW);
{
if constexpr (throw_exception)
throw Exception(std::string(ToDataType::family_name) + " convert overflow",
ErrorCodes::DECIMAL_OVERFLOW);
else
return ReturnType(false);
}
}
return static_cast<typename ToFieldType::NativeType>(converted_value);
result = static_cast<typename ToFieldType::NativeType>(converted_value);
return ReturnType(true);
}
template <typename FromDataType, typename ToDataType>
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>, typename ToDataType::FieldType>
convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to)
{
using ToFieldType = typename ToDataType::FieldType;
ToFieldType result;
convertDecimalsImpl<FromDataType, ToDataType, void>(value, scale_from, scale_to, result);
return result;
}
template <typename FromDataType, typename ToDataType>
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>, bool>
tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType& result)
{
return convertDecimalsImpl<FromDataType, ToDataType, bool>(value, scale_from, scale_to, result);
}
template <typename FromDataType, typename ToDataType, typename ReturnType>
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsNumber<typename ToDataType::FieldType>, ReturnType>
convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
return DecimalUtils::convertToImpl<ToFieldType, FromFieldType, ReturnType>(value, scale, result);
}
template <typename FromDataType, typename ToDataType>
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsNumber<typename ToDataType::FieldType>, typename ToDataType::FieldType>
convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
{
using ToFieldType = typename ToDataType::FieldType;
typename ToDataType::FieldType result;
return DecimalUtils::convertTo<ToFieldType>(value, scale);
convertFromDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
return result;
}
template <typename FromDataType, typename ToDataType>
inline std::enable_if_t<IsNumber<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>, typename ToDataType::FieldType>
convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsNumber<typename ToDataType::FieldType>, bool>
tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
return convertFromDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
}
template <typename FromDataType, typename ToDataType, typename ReturnType>
inline std::enable_if_t<IsNumber<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>, ReturnType>
convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
using FromFieldType = typename FromDataType::FieldType;
using ToFieldType = typename ToDataType::FieldType;
using ToNativeType = typename ToFieldType::NativeType;
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
if constexpr (std::is_floating_point_v<FromFieldType>)
{
if (!std::isfinite(value))
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Cannot convert infinity or NaN to decimal",
ErrorCodes::DECIMAL_OVERFLOW);
{
if constexpr (throw_exception)
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Cannot convert infinity or NaN to decimal",
ErrorCodes::DECIMAL_OVERFLOW);
else
return false;
}
auto out = value * static_cast<FromFieldType>(DecimalUtils::scaleMultiplier<ToNativeType>(scale));
if constexpr (std::is_same_v<ToNativeType, Int128>)
@ -157,29 +216,60 @@ convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
static constexpr Int128 max_int128 = maxInt128();
if (out <= static_cast<ToNativeType>(min_int128) || out >= static_cast<ToNativeType>(max_int128))
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range",
ErrorCodes::DECIMAL_OVERFLOW);
{
if constexpr (throw_exception)
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range",
ErrorCodes::DECIMAL_OVERFLOW);
else
return ReturnType(false);
}
}
else
{
if (out <= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::min()) ||
out >= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::max()))
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range",
ErrorCodes::DECIMAL_OVERFLOW);
{
if constexpr (throw_exception)
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range",
ErrorCodes::DECIMAL_OVERFLOW);
else
return ReturnType(false);
}
}
return static_cast<ToNativeType>(out);
result = static_cast<ToNativeType>(out);
return ReturnType(true);
}
else
{
if constexpr (is_big_int_v<FromFieldType>)
return convertDecimals<DataTypeDecimal<Decimal256>, ToDataType>(static_cast<Int256>(value), 0, scale);
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal256>, ToDataType, ReturnType>(static_cast<Int256>(value), 0, scale, result));
else if constexpr (std::is_same_v<FromFieldType, UInt64>)
return convertDecimals<DataTypeDecimal<Decimal128>, ToDataType>(value, 0, scale);
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal128>, ToDataType, ReturnType>(value, 0, scale, result));
else
return convertDecimals<DataTypeDecimal<Decimal64>, ToDataType>(value, 0, scale);
return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal64>, ToDataType, ReturnType>(value, 0, scale, result));
}
}
template <typename FromDataType, typename ToDataType>
inline std::enable_if_t<IsNumber<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>, typename ToDataType::FieldType>
convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
{
typename ToDataType::FieldType result;
convertToDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
return result;
}
template <typename FromDataType, typename ToDataType>
inline std::enable_if_t<IsNumber<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>, bool>
tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
{
return convertToDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
}
template <typename T>
inline DataTypePtr createDecimalMaxPrecision(UInt64 scale)
{

View File

@ -1,6 +1,7 @@
#include <Common/FieldVisitors.h>
#include <DataTypes/FieldToDataType.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeMap.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypeString.h>
@ -118,6 +119,24 @@ DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const
return std::make_shared<DataTypeTuple>(element_types);
}
DataTypePtr FieldToDataType::operator() (const Map & map) const
{
DataTypes key_types;
DataTypes value_types;
key_types.reserve(map.size());
value_types.reserve(map.size());
for (const auto & elem : map)
{
const auto & tuple = elem.safeGet<const Tuple &>();
assert(tuple.size() == 2);
key_types.push_back(applyVisitor(FieldToDataType(), tuple[0]));
value_types.push_back(applyVisitor(FieldToDataType(), tuple[1]));
}
return std::make_shared<DataTypeMap>(getLeastSupertype(key_types), getLeastSupertype(value_types));
}
DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) const
{
const auto & name = static_cast<const AggregateFunctionStateData &>(x).name;

View File

@ -26,6 +26,7 @@ public:
DataTypePtr operator() (const String & x) const;
DataTypePtr operator() (const Array & x) const;
DataTypePtr operator() (const Tuple & tuple) const;
DataTypePtr operator() (const Map & map) const;
DataTypePtr operator() (const DecimalField<Decimal32> & x) const;
DataTypePtr operator() (const DecimalField<Decimal64> & x) const;
DataTypePtr operator() (const DecimalField<Decimal128> & x) const;

View File

@ -91,6 +91,8 @@ public:
TupleElement,
MapElement,
DictionaryKeys,
DictionaryIndexes,
};
@ -449,6 +451,7 @@ public:
static bool isSpecialCompressionAllowed(const SubstreamPath & path);
private:
friend class DataTypeFactory;
friend class AggregateFunctionSimpleState;
/// Customize this DataType
void setCustomization(DataTypeCustomDescPtr custom_desc_) const;
@ -517,6 +520,7 @@ struct WhichDataType
constexpr bool isUUID() const { return idx == TypeIndex::UUID; }
constexpr bool isArray() const { return idx == TypeIndex::Array; }
constexpr bool isTuple() const { return idx == TypeIndex::Tuple; }
constexpr bool isMap() const {return idx == TypeIndex::Map; }
constexpr bool isSet() const { return idx == TypeIndex::Set; }
constexpr bool isInterval() const { return idx == TypeIndex::Interval; }

View File

@ -27,6 +27,7 @@ SRCS(
DataTypeInterval.cpp
DataTypeLowCardinality.cpp
DataTypeLowCardinalityHelpers.cpp
DataTypeMap.cpp
DataTypeNothing.cpp
DataTypeNullable.cpp
DataTypeNumberBase.cpp

View File

@ -217,6 +217,9 @@ void DatabaseAtomic::renameTable(const Context & context, const String & table_n
if (is_dictionary && !inside_database)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot move dictionary to other database");
if (!exchange)
other_db.checkMetadataFilenameAvailabilityUnlocked(to_table_name, inside_database ? db_lock : other_db_lock);
StoragePtr table = getTableUnlocked(table_name, db_lock);
table->checkTableCanBeRenamed();
assert_can_move_mat_view(table);

View File

@ -42,6 +42,14 @@ void DatabaseLazy::loadStoredObjects(
iterateMetadataFiles(context, [this](const String & file_name)
{
const std::string table_name = file_name.substr(0, file_name.size() - 4);
auto detached_permanently_flag = Poco::File(getMetadataPath() + "/" + file_name + detached_suffix);
if (detached_permanently_flag.exists())
{
LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name));
return;
}
attachTable(table_name, nullptr, {});
});
}

View File

@ -164,20 +164,38 @@ void DatabaseOnDisk::createTable(
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (isDictionaryExist(table_name))
throw Exception("Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.",
ErrorCodes::DICTIONARY_ALREADY_EXISTS);
throw Exception(ErrorCodes::DICTIONARY_ALREADY_EXISTS, "Dictionary {}.{} already exists", backQuote(getDatabaseName()), backQuote(table_name));
if (isTableExist(table_name, global_context))
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists", backQuote(getDatabaseName()), backQuote(table_name));
String table_metadata_path = getObjectMetadataPath(table_name);
if (create.attach_short_syntax)
{
/// Metadata already exists, table was detached
attachTable(table_name, table, getTableDataPath(create));
removeDetachedPermanentlyFlag(table_name, table_metadata_path);
return;
}
String table_metadata_path = getObjectMetadataPath(table_name);
if (!create.attach)
checkMetadataFilenameAvailability(table_name);
if (create.attach && Poco::File(table_metadata_path).exists())
{
ASTPtr ast_detached = parseQueryFromMetadata(log, context, table_metadata_path);
auto & create_detached = ast_detached->as<ASTCreateQuery &>();
// either both should be Nil, either values should be equal
if (create.uuid != create_detached.uuid)
throw Exception(
ErrorCodes::TABLE_ALREADY_EXISTS,
"Table {}.{} already exist (detached permanently). To attach it back "
"you need to use short ATTACH syntax or a full statement with the same UUID",
backQuote(getDatabaseName()), backQuote(table_name));
}
String table_metadata_tmp_path = table_metadata_path + create_suffix;
String statement;
@ -194,6 +212,26 @@ void DatabaseOnDisk::createTable(
}
commitCreateTable(create, table, table_metadata_tmp_path, table_metadata_path);
removeDetachedPermanentlyFlag(table_name, table_metadata_path);
}
/// If the table was detached permanently we will have a flag file with
/// .sql.detached extension, is not needed anymore since we attached the table back
void DatabaseOnDisk::removeDetachedPermanentlyFlag(const String & table_name, const String & table_metadata_path) const
{
try
{
auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix);
if (detached_permanently_flag.exists())
detached_permanently_flag.remove();
}
catch (Exception & e)
{
e.addMessage("while trying to remove permanenty detached flag. Table {}.{} may still be marked as permanently detached, and will not be reattached during server restart.", backQuote(getDatabaseName()), backQuote(table_name));
throw;
}
}
void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table,
@ -215,6 +253,22 @@ void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const Stora
}
}
void DatabaseOnDisk::detachTablePermanently(const String & table_name)
{
auto table = detachTable(table_name);
Poco::File detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix);
try
{
detached_permanently_flag.createFile();
}
catch (Exception & e)
{
e.addMessage("while trying to set permanenty detached flag. Table {}.{} may be reattached during server restart.", backQuote(getDatabaseName()), backQuote(table_name));
throw;
}
}
void DatabaseOnDisk::dropTable(const Context & context, const String & table_name, bool /*no_delay*/)
{
String table_metadata_path = getObjectMetadataPath(table_name);
@ -253,6 +307,27 @@ void DatabaseOnDisk::dropTable(const Context & context, const String & table_nam
Poco::File(table_metadata_path_drop).remove();
}
void DatabaseOnDisk::checkMetadataFilenameAvailability(const String & to_table_name) const
{
std::unique_lock lock(mutex);
checkMetadataFilenameAvailabilityUnlocked(to_table_name, lock);
}
void DatabaseOnDisk::checkMetadataFilenameAvailabilityUnlocked(const String & to_table_name, std::unique_lock<std::mutex> &) const
{
String table_metadata_path = getObjectMetadataPath(to_table_name);
if (Poco::File(table_metadata_path).exists())
{
auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix);
if (detached_permanently_flag.exists())
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached permanently)", backQuote(database_name), backQuote(to_table_name));
else
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached)", backQuote(database_name), backQuote(to_table_name));
}
}
void DatabaseOnDisk::renameTable(
const Context & context,
const String & table_name,
@ -299,6 +374,9 @@ void DatabaseOnDisk::renameTable(
if (from_atomic_to_ordinary)
create.uuid = UUIDHelpers::Nil;
if (auto * target_db = dynamic_cast<DatabaseOnDisk *>(&to_database))
target_db->checkMetadataFilenameAvailability(to_table_name);
/// Notify the table that it is renamed. It will move data to new path (if it stores data on disk) and update StorageID
table->rename(to_database.getTableDataPath(create), StorageID(create));
}
@ -328,6 +406,8 @@ void DatabaseOnDisk::renameTable(
}
}
/// It returns create table statement (even if table is detached)
ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const String & table_name, const Context &, bool throw_on_error) const
{
ASTPtr ast;
@ -430,8 +510,11 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati
if (endsWith(dir_it.name(), ".sql.bak"))
continue;
static const char * tmp_drop_ext = ".sql.tmp_drop";
if (endsWith(dir_it.name(), tmp_drop_ext))
/// Permanently detached table flag
if (endsWith(dir_it.name(), ".sql.detached"))
continue;
if (endsWith(dir_it.name(), ".sql.tmp_drop"))
{
/// There are files that we tried to delete previously
metadata_files.emplace(dir_it.name(), false);

Some files were not shown because too many files have changed in this diff Show More