mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 15:42:02 +00:00
Merge branch 'master' into database_atomic_improvements
This commit is contained in:
Alexander Tokmakov
commit
09abe71b60
@ -9,12 +9,12 @@ ClickHouse is an open-source column-oriented database management system that all
|
||||
* [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information.
|
||||
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
|
||||
* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
|
||||
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
|
||||
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
|
||||
* [Yandex.Messenger channel](https://yandex.ru/chat/#/join/20e380d9-c7be-4123-ab06-e95fb946975e) shares announcements and useful links in Russian.
|
||||
* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
|
||||
* You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
|
||||
|
||||
## Upcoming Events
|
||||
|
||||
* [ClickHouse for genetic data (in Russian)](https://cloud.yandex.ru/events/152) on July 14, 2020.
|
||||
* [ClickHouse virtual office hours](https://www.eventbrite.com/e/clickhouse-july-virtual-meetup-tickets-111199787558) on July 15, 2020.
|
||||
* [ClickHouse at ByteDance (in Chinese)](https://mp.weixin.qq.com/s/Em-HjPylO8D7WPui4RREAQ) on July 17, 2020.
|
||||
|
||||
@ -686,12 +686,17 @@ public:
|
||||
inline time_t makeDateTime(UInt16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const
|
||||
{
|
||||
size_t index = makeDayNum(year, month, day_of_month);
|
||||
time_t time_offset = hour * 3600 + minute * 60 + second;
|
||||
UInt32 time_offset = hour * 3600 + minute * 60 + second;
|
||||
|
||||
if (time_offset >= lut[index].time_at_offset_change)
|
||||
time_offset -= lut[index].amount_of_offset_change;
|
||||
|
||||
return lut[index].date + time_offset;
|
||||
UInt32 res = lut[index].date + time_offset;
|
||||
|
||||
if (unlikely(res > DATE_LUT_MAX))
|
||||
return 0;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
inline const Values & getValues(DayNum d) const { return lut[d]; }
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
# This strings autochanged from release_lib.sh:
|
||||
SET(VERSION_REVISION 54436)
|
||||
SET(VERSION_REVISION 54437)
|
||||
SET(VERSION_MAJOR 20)
|
||||
SET(VERSION_MINOR 6)
|
||||
SET(VERSION_MINOR 7)
|
||||
SET(VERSION_PATCH 1)
|
||||
SET(VERSION_GITHASH efc57fb063b3fb4df968d916720ec4d4ced4642e)
|
||||
SET(VERSION_DESCRIBE v20.6.1.1-prestable)
|
||||
SET(VERSION_STRING 20.6.1.1)
|
||||
SET(VERSION_GITHASH d64e51d1a78c1b53c33915ca0f75c97b2333844f)
|
||||
SET(VERSION_DESCRIBE v20.7.1.1-prestable)
|
||||
SET(VERSION_STRING 20.7.1.1)
|
||||
# end of autochange
|
||||
|
||||
@ -24,7 +24,7 @@ set (SRCS
|
||||
add_library(amqp-cpp ${SRCS})
|
||||
|
||||
target_compile_options (amqp-cpp
|
||||
PUBLIC
|
||||
PRIVATE
|
||||
-Wno-old-style-cast
|
||||
-Wno-inconsistent-missing-destructor-override
|
||||
-Wno-deprecated
|
||||
@ -38,7 +38,7 @@ target_compile_options (amqp-cpp
|
||||
-w
|
||||
)
|
||||
|
||||
target_include_directories (amqp-cpp PUBLIC ${LIBRARY_DIR}/include)
|
||||
target_include_directories (amqp-cpp SYSTEM PUBLIC ${LIBRARY_DIR}/include)
|
||||
|
||||
target_link_libraries (amqp-cpp PUBLIC ssl)
|
||||
|
||||
|
||||
2
contrib/fmtlib
vendored
2
contrib/fmtlib
vendored
@ -1 +1 @@
|
||||
Subproject commit 297c3b2ed551a4989826fc8c4780bf533e964bd9
|
||||
Subproject commit c108ee1d590089ccf642fc85652b845924067af2
|
||||
4
debian/changelog
vendored
4
debian/changelog
vendored
@ -1,5 +1,5 @@
|
||||
clickhouse (20.6.1.1) unstable; urgency=low
|
||||
clickhouse (20.7.1.1) unstable; urgency=low
|
||||
|
||||
* Modified source code
|
||||
|
||||
-- clickhouse-release <clickhouse-release@yandex-team.ru> Mon, 22 Jun 2020 20:40:23 +0300
|
||||
-- clickhouse-release <clickhouse-release@yandex-team.ru> Mon, 13 Jul 2020 18:25:58 +0300
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=20.6.1.*
|
||||
ARG version=20.7.1.*
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=20.6.1.*
|
||||
ARG version=20.7.1.*
|
||||
ARG gosu_ver=1.10
|
||||
|
||||
RUN apt-get update \
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=20.6.1.*
|
||||
ARG version=20.7.1.*
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y apt-transport-https dirmngr && \
|
||||
|
||||
7
docker/test/fuzzer/query-fuzzer-tweaks-users.xml
Normal file
7
docker/test/fuzzer/query-fuzzer-tweaks-users.xml
Normal file
@ -0,0 +1,7 @@
|
||||
<yandex>
|
||||
<profiles>
|
||||
<default>
|
||||
<max_execution_time>10</max_execution_time>
|
||||
</default>
|
||||
</profiles>
|
||||
</yandex>
|
||||
@ -46,6 +46,7 @@ function configure
|
||||
cp -av "$repo_dir"/programs/server/config* db
|
||||
cp -av "$repo_dir"/programs/server/user* db
|
||||
cp -av "$repo_dir"/tests/config db/config.d
|
||||
cp -av "$script_dir"/query-fuzzer-tweaks-users.xml db/users.d
|
||||
}
|
||||
|
||||
function watchdog
|
||||
@ -53,6 +54,7 @@ function watchdog
|
||||
sleep 3600
|
||||
|
||||
echo "Fuzzing run has timed out"
|
||||
./clickhouse client --query "select elapsed, query from system.processes" ||:
|
||||
killall -9 clickhouse clickhouse-server clickhouse-client ||:
|
||||
}
|
||||
|
||||
@ -74,6 +76,7 @@ function fuzz
|
||||
|| fuzzer_exit_code=$?
|
||||
|
||||
echo "Fuzzer exit code is $fuzzer_exit_code"
|
||||
./clickhouse client --query "select elapsed, query from system.processes" ||:
|
||||
kill -9 $server_pid ||:
|
||||
return $fuzzer_exit_code
|
||||
}
|
||||
|
||||
@ -36,6 +36,9 @@ Action required for every item -- these are errors that must be fixed. The error
|
||||
#### Slow on client
|
||||
Action required for every item -- these are errors that must be fixed. This table shows queries that take significantly longer to process on the client than on the server. A possible reason might be sending too much data to the client, e.g., a forgotten `format Null`.
|
||||
|
||||
#### Short queries not marked as short
|
||||
Action required for every item -- these are errors that must be fixed. This table shows queries that are "short" but not explicitly marked as such. "Short" queries are too fast to meaningfully compare performance, because the changes are drowned by the noise. We consider all queries that run faster than 0.02 s to be "short", and only check the performance if they became slower than this threshold. Probably this mode is not what you want, so you have to increase the query run time to be between 1 and 0.1 s, so that the performance can be compared. You do want this "short" mode for queries that complete "immediately", such as some varieties of `select count(*)`. You have to mark them as "short" explicitly by writing `<query short="1">...`. The value of "short" attribute is evaluated as a python expression, and substitutions are performed, so you can write something like `<query short="{column1} = {column2}">select count(*) from table where {column1} > {column2}</query>`, to mark only a particular combination of variables as short.
|
||||
|
||||
#### Partial queries
|
||||
Action required for the cells marked in red. Shows the queries we are unable to run on an old server -- probably because they contain a new function. You should see this table when you add a new function and a performance test for it. Check that the run time and variance are acceptable (run time between 0.1 and 1 seconds, variance below 10%). If not, they will be highlighted in red.
|
||||
|
||||
|
||||
@ -2,11 +2,13 @@
|
||||
toc_priority: 6
|
||||
toc_title: RabbitMQ
|
||||
---
|
||||
# RabbitMQ Engine
|
||||
|
||||
# RabbitMQ Engine {#rabbitmq-engine}
|
||||
|
||||
This engine allows integrating ClickHouse with [RabbitMQ](https://www.rabbitmq.com).
|
||||
|
||||
RabbitMQ lets you:
|
||||
|
||||
- Publish or subscribe to data flows.
|
||||
- Process streams as they become available.
|
||||
|
||||
@ -43,7 +45,7 @@ Optional parameters:
|
||||
- `rabbitmq_row_delimiter` – Delimiter character, which ends the message.
|
||||
- `rabbitmq_num_consumers` – The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient.
|
||||
- `rabbitmq_num_queues` – The number of queues per consumer. Default: `1`. Specify more queues if the capacity of one queue per consumer is insufficient. Single queue can contain up to 50K messages at the same time.
|
||||
- `rabbitmq_transactional_channel` – Wrap insert queries in transactions. Default: `0`.
|
||||
- `rabbitmq_transactional_channel` – Wrap insert queries in transactions. Default: `0`.
|
||||
|
||||
Required configuration:
|
||||
|
||||
@ -83,6 +85,7 @@ Data can be channeled based on `rabbitmq_exchange_type` and the specified `rabbi
|
||||
There can be no more than one exchange per table. One exchange can be shared between multiple tables - it enables routing into multiple tables at the same time.
|
||||
|
||||
Exchange type options:
|
||||
|
||||
- `direct` - Routing is based on exact matching of keys. Example table key list: `key1,key2,key3,key4,key5`, message key can eqaul any of them.
|
||||
- `fanout` - Routing to all tables (where exchange name is the same) regardless of the keys.
|
||||
- `topic` - Routing is based on patterns with dot-separated keys. Examples: `*.logs`, `records.*.*.2020`, `*.2018,*.2019,*.2020`.
|
||||
@ -92,6 +95,7 @@ Exchange type options:
|
||||
If exchange type is not specified, then default is `fanout` and routing keys for data publishing must be randomized in range `[1, num_consumers]` for every message/batch (or in range `[1, num_consumers * num_queues]` if `rabbitmq_num_queues` is set). This table configuration works quicker then any other, especially when `rabbitmq_num_consumers` and/or `rabbitmq_num_queues` parameters are set.
|
||||
|
||||
If `rabbitmq_num_consumers` and/or `rabbitmq_num_queues` parameters are specified along with `rabbitmq_exchange_type`, then:
|
||||
|
||||
- `rabbitmq-consistent-hash-exchange` plugin must be enabled.
|
||||
- `message_id` property of the published messages must be specified (unique for each message/batch).
|
||||
|
||||
@ -116,4 +120,3 @@ Example:
|
||||
|
||||
SELECT key, value FROM daily ORDER BY key;
|
||||
```
|
||||
|
||||
|
||||
@ -26,7 +26,7 @@ Engines:
|
||||
|
||||
During `INSERT` queries, the table is locked, and other queries for reading and writing data both wait for the table to unlock. If there are no data writing queries, any number of data reading queries can be performed concurrently.
|
||||
|
||||
- Do not support [mutation](../../../sql-reference/statements/alter.md#alter-mutations) operations.
|
||||
- Do not support [mutations](../../../sql-reference/statements/alter/index.md#alter-mutations).
|
||||
|
||||
- Do not support indexes.
|
||||
|
||||
|
||||
@ -116,10 +116,10 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached
|
||||
|
||||
The folders ‘201901\_1\_1\_0’, ‘201901\_1\_7\_1’ and so on are the directories of the parts. Each part relates to a corresponding partition and contains data just for a certain month (the table in this example has partitioning by month).
|
||||
|
||||
The `detached` directory contains parts that were detached from the table using the [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition) query. The corrupted parts are also moved to this directory, instead of being deleted. The server does not use the parts from the `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../../sql-reference/statements/alter.md#alter_attach-partition) query.
|
||||
The `detached` directory contains parts that were detached from the table using the [DETACH](../../../sql-reference/statements/alter/partition.md#alter_detach-partition) query. The corrupted parts are also moved to this directory, instead of being deleted. The server does not use the parts from the `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../../sql-reference/statements/alter/partition.md#alter_attach-partition) query.
|
||||
|
||||
Note that on the operating server, you cannot manually change the set of parts or their data on the file system, since the server will not know about it. For non-replicated tables, you can do this when the server is stopped, but it isn’t recommended. For replicated tables, the set of parts cannot be changed in any case.
|
||||
|
||||
ClickHouse allows you to perform operations with the partitions: delete them, copy from one table to another, or create a backup. See the list of all operations in the section [Manipulations With Partitions and Parts](../../../sql-reference/statements/alter.md#alter_manipulations-with-partitions).
|
||||
ClickHouse allows you to perform operations with the partitions: delete them, copy from one table to another, or create a backup. See the list of all operations in the section [Manipulations With Partitions and Parts](../../../sql-reference/statements/alter/partition.md#alter_manipulations-with-partitions).
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/table_engines/custom_partitioning_key/) <!--hide-->
|
||||
|
||||
@ -212,7 +212,7 @@ This feature is helpful when using the [SummingMergeTree](../../../engines/table
|
||||
|
||||
In this case it makes sense to leave only a few columns in the primary key that will provide efficient range scans and add the remaining dimension columns to the sorting key tuple.
|
||||
|
||||
[ALTER](../../../sql-reference/statements/alter.md) of the sorting key is a lightweight operation because when a new column is simultaneously added to the table and to the sorting key, existing data parts don’t need to be changed. Since the old sorting key is a prefix of the new sorting key and there is no data in the newly added column, the data is sorted by both the old and new sorting keys at the moment of table modification.
|
||||
[ALTER](../../../sql-reference/statements/alter/index.md) of the sorting key is a lightweight operation because when a new column is simultaneously added to the table and to the sorting key, existing data parts don’t need to be changed. Since the old sorting key is a prefix of the new sorting key and there is no data in the newly added column, the data is sorted by both the old and new sorting keys at the moment of table modification.
|
||||
|
||||
### Use of Indexes and Partitions in Queries {#use-of-indexes-and-partitions-in-queries}
|
||||
|
||||
@ -490,7 +490,7 @@ If you perform the `SELECT` query between merges, you may get expired data. To a
|
||||
|
||||
`MergeTree` family table engines can store data on multiple block devices. For example, it can be useful when the data of a certain table are implicitly split into “hot” and “cold”. The most recent data is regularly requested but requires only a small amount of space. On the contrary, the fat-tailed historical data is requested rarely. If several disks are available, the “hot” data may be located on fast disks (for example, NVMe SSDs or in memory), while the “cold” data - on relatively slow ones (for example, HDD).
|
||||
|
||||
Data part is the minimum movable unit for `MergeTree`-engine tables. The data belonging to one part are stored on one disk. Data parts can be moved between disks in the background (according to user settings) as well as by means of the [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) queries.
|
||||
Data part is the minimum movable unit for `MergeTree`-engine tables. The data belonging to one part are stored on one disk. Data parts can be moved between disks in the background (according to user settings) as well as by means of the [ALTER](../../../sql-reference/statements/alter/partition.md#alter_move-partition) queries.
|
||||
|
||||
### Terms {#terms}
|
||||
|
||||
@ -636,9 +636,9 @@ The number of threads performing background moves of data parts can be changed b
|
||||
In the case of `MergeTree` tables, data is getting to disk in different ways:
|
||||
|
||||
- As a result of an insert (`INSERT` query).
|
||||
- During background merges and [mutations](../../../sql-reference/statements/alter.md#alter-mutations).
|
||||
- During background merges and [mutations](../../../sql-reference/statements/alter/index.md#alter-mutations).
|
||||
- When downloading from another replica.
|
||||
- As a result of partition freezing [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter.md#alter_freeze-partition).
|
||||
- As a result of partition freezing [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter/partition.md#alter_freeze-partition).
|
||||
|
||||
In all these cases except for mutations and partition freezing, a part is stored on a volume and a disk according to the given storage policy:
|
||||
|
||||
@ -650,7 +650,7 @@ Under the hood, mutations and partition freezing make use of [hard links](https:
|
||||
In the background, parts are moved between volumes on the basis of the amount of free space (`move_factor` parameter) according to the order the volumes are declared in the configuration file.
|
||||
Data is never transferred from the last one and into the first one. One may use system tables [system.part\_log](../../../operations/system-tables/part_log.md#system_tables-part-log) (field `type = MOVE_PART`) and [system.parts](../../../operations/system-tables/parts.md#system_tables-parts) (fields `path` and `disk`) to monitor background moves. Also, the detailed information can be found in server logs.
|
||||
|
||||
User can force moving a part or a partition from one volume to another using the query [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition), all the restrictions for background operations are taken into account. The query initiates a move on its own and does not wait for background operations to be completed. User will get an error message if not enough free space is available or if any of the required conditions are not met.
|
||||
User can force moving a part or a partition from one volume to another using the query [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter/partition.md#alter_move-partition), all the restrictions for background operations are taken into account. The query initiates a move on its own and does not wait for background operations to be completed. User will get an error message if not enough free space is available or if any of the required conditions are not met.
|
||||
|
||||
Moving data does not interfere with data replication. Therefore, different storage policies can be specified for the same table on different replicas.
|
||||
|
||||
|
||||
@ -19,7 +19,7 @@ Replication works at the level of an individual table, not the entire server. A
|
||||
|
||||
Replication does not depend on sharding. Each shard has its own independent replication.
|
||||
|
||||
Compressed data for `INSERT` and `ALTER` queries is replicated (for more information, see the documentation for [ALTER](../../../sql-reference/statements/alter.md#query_language_queries_alter)).
|
||||
Compressed data for `INSERT` and `ALTER` queries is replicated (for more information, see the documentation for [ALTER](../../../sql-reference/statements/alter/index.md#query_language_queries_alter)).
|
||||
|
||||
`CREATE`, `DROP`, `ATTACH`, `DETACH` and `RENAME` queries are executed on a single server and are not replicated:
|
||||
|
||||
|
||||
@ -27,16 +27,16 @@ ClickHouse doesn’t have real-time point deletes like in [OLTP](https://en.wiki
|
||||
|
||||
This is the most common approach to make your system based on ClickHouse [GDPR](https://gdpr-info.eu)-compliant.
|
||||
|
||||
More details on [mutations](../../sql-reference/statements/alter.md#alter-mutations).
|
||||
More details on [mutations](../../sql-reference/statements/alter/index.md#alter-mutations).
|
||||
|
||||
## DROP PARTITION {#drop-partition}
|
||||
|
||||
`ALTER TABLE ... DROP PARTITION` provides a cost-efficient way to drop a whole partition. It’s not that flexible and needs proper partitioning scheme configured on table creation, but still covers most common cases. Like mutations need to be executed from an external system for regular use.
|
||||
|
||||
More details on [manipulating partitions](../../sql-reference/statements/alter.md#alter_drop-partition).
|
||||
More details on [manipulating partitions](../../sql-reference/statements/alter/partition.md#alter_drop-partition).
|
||||
|
||||
## TRUNCATE {#truncate}
|
||||
|
||||
It’s rather radical to drop all data from a table, but in some cases it might be exactly what you need.
|
||||
|
||||
More details on [table truncation](../../sql-reference/statements/alter.md#alter_drop-partition).
|
||||
More details on [table truncation](../../sql-reference/statements/alter/partition.md#alter_drop-partition).
|
||||
|
||||
@ -13,7 +13,7 @@ The supported formats are:
|
||||
| Format | Input | Output |
|
||||
|-----------------------------------------------------------------|-------|--------|
|
||||
| [TabSeparated](#tabseparated) | ✔ | ✔ |
|
||||
| [TabSeparatedRaw](#tabseparatedraw) | ✗ | ✔ |
|
||||
| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Template](#format-template) | ✔ | ✔ |
|
||||
@ -143,7 +143,7 @@ SELECT * FROM nestedt FORMAT TSV
|
||||
## TabSeparatedRaw {#tabseparatedraw}
|
||||
|
||||
Differs from `TabSeparated` format in that the rows are written without escaping.
|
||||
This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table).
|
||||
When parsing with this format, tabs or linefeeds are not allowed in each field.
|
||||
|
||||
This format is also available under the name `TSVRaw`.
|
||||
|
||||
|
||||
@ -59,7 +59,7 @@ Privileges can be granted to a user account by the [GRANT](../sql-reference/stat
|
||||
Management queries:
|
||||
|
||||
- [CREATE USER](../sql-reference/statements/create/user.md)
|
||||
- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement)
|
||||
- [ALTER USER](../sql-reference/statements/alter/user.md#alter-user-statement)
|
||||
- [DROP USER](../sql-reference/statements/drop.md)
|
||||
- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement)
|
||||
|
||||
@ -85,7 +85,7 @@ Role contains:
|
||||
Management queries:
|
||||
|
||||
- [CREATE ROLE](../sql-reference/statements/create/role.md)
|
||||
- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement)
|
||||
- [ALTER ROLE](../sql-reference/statements/alter/role.md#alter-role-statement)
|
||||
- [DROP ROLE](../sql-reference/statements/drop.md)
|
||||
- [SET ROLE](../sql-reference/statements/set-role.md)
|
||||
- [SET DEFAULT ROLE](../sql-reference/statements/set-role.md#set-default-role-statement)
|
||||
@ -100,7 +100,7 @@ Row policy is a filter that defines which of the rows are available to a user or
|
||||
Management queries:
|
||||
|
||||
- [CREATE ROW POLICY](../sql-reference/statements/create/row-policy.md)
|
||||
- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement)
|
||||
- [ALTER ROW POLICY](../sql-reference/statements/alter/row-policy.md#alter-row-policy-statement)
|
||||
- [DROP ROW POLICY](../sql-reference/statements/drop.md#drop-row-policy-statement)
|
||||
- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement)
|
||||
|
||||
@ -111,7 +111,7 @@ Settings profile is a collection of [settings](../operations/settings/index.md).
|
||||
Management queries:
|
||||
|
||||
- [CREATE SETTINGS PROFILE](../sql-reference/statements/create/settings-profile.md#create-settings-profile-statement)
|
||||
- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement)
|
||||
- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter/settings-profile.md#alter-settings-profile-statement)
|
||||
- [DROP SETTINGS PROFILE](../sql-reference/statements/drop.md#drop-settings-profile-statement)
|
||||
- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement)
|
||||
|
||||
@ -124,7 +124,7 @@ Quota contains a set of limits for some durations, as well as a list of roles an
|
||||
Management queries:
|
||||
|
||||
- [CREATE QUOTA](../sql-reference/statements/create/quota.md)
|
||||
- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement)
|
||||
- [ALTER QUOTA](../sql-reference/statements/alter/quota.md#alter-quota-statement)
|
||||
- [DROP QUOTA](../sql-reference/statements/drop.md#drop-quota-statement)
|
||||
- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement)
|
||||
|
||||
|
||||
@ -33,7 +33,7 @@ For smaller volumes of data, a simple `INSERT INTO ... SELECT ...` to remote tab
|
||||
ClickHouse allows using the `ALTER TABLE ... FREEZE PARTITION ...` query to create a local copy of table partitions. This is implemented using hardlinks to the `/var/lib/clickhouse/shadow/` folder, so it usually does not consume extra disk space for old data. The created copies of files are not handled by ClickHouse server, so you can just leave them there: you will have a simple backup that doesn’t require any additional external system, but it will still be prone to hardware issues. For this reason, it’s better to remotely copy them to another location and then remove the local copies. Distributed filesystems and object stores are still a good options for this, but normal attached file servers with a large enough capacity might work as well (in this case the transfer will occur via the network filesystem or maybe [rsync](https://en.wikipedia.org/wiki/Rsync)).
|
||||
Data can be restored from backup using the `ALTER TABLE ... ATTACH PARTITION ...`
|
||||
|
||||
For more information about queries related to partition manipulations, see the [ALTER documentation](../sql-reference/statements/alter.md#alter_manipulations-with-partitions).
|
||||
For more information about queries related to partition manipulations, see the [ALTER documentation](../sql-reference/statements/alter/partition.md#alter_manipulations-with-partitions).
|
||||
|
||||
A third-party tool is available to automate this approach: [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup).
|
||||
|
||||
|
||||
@ -1599,13 +1599,13 @@ Default value: 268435456.
|
||||
|
||||
- [min\_insert\_block\_size\_bytes](#min-insert-block-size-bytes)
|
||||
|
||||
## output_format_pretty_grid_charset {#output-format-pretty-grid-charset}
|
||||
## output\_format\_pretty\_grid\_charset {#output-format-pretty-grid-charset}
|
||||
|
||||
Allows to change a charset which is used for printing grids borders. Available charsets are following: UTF-8, ASCII.
|
||||
|
||||
**Example**
|
||||
**Example**
|
||||
|
||||
```text
|
||||
``` text
|
||||
SET output_format_pretty_grid_charset = 'UTF-8';
|
||||
SELECT * FROM a;
|
||||
┌─a─┐
|
||||
@ -1619,5 +1619,4 @@ SELECT * FROM a;
|
||||
+---+
|
||||
```
|
||||
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) <!-- hide -->
|
||||
|
||||
@ -2,8 +2,8 @@
|
||||
|
||||
Contains information about detached parts of [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables. The `reason` column specifies why the part was detached.
|
||||
|
||||
For user-detached parts, the reason is empty. Such parts can be attached with [ALTER TABLE ATTACH PARTITION\|PART](../../sql-reference/statements/alter.md#alter_attach-partition) command.
|
||||
For user-detached parts, the reason is empty. Such parts can be attached with [ALTER TABLE ATTACH PARTITION\|PART](../../sql-reference/statements/alter/partition.md#alter_attach-partition) command.
|
||||
|
||||
For the description of other columns, see [system.parts](../../operations/system-tables/parts.md#system_tables-parts).
|
||||
|
||||
If part name is invalid, values of some columns may be `NULL`. Such parts can be deleted with [ALTER TABLE DROP DETACHED PART](../../sql-reference/statements/alter.md#alter_drop-detached).
|
||||
If part name is invalid, values of some columns may be `NULL`. Such parts can be deleted with [ALTER TABLE DROP DETACHED PART](../../sql-reference/statements/alter/partition.md#alter_drop-detached).
|
||||
|
||||
@ -22,7 +22,7 @@ Most of system tables store their data in RAM. A ClickHouse server creates such
|
||||
|
||||
Unlike other system tables, the system tables [metric\_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query\_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query\_thread\_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace\_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one.
|
||||
|
||||
By default, table growth is unlimited. To control a size of a table, you can use [TTL](../../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables.
|
||||
By default, table growth is unlimited. To control a size of a table, you can use [TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables.
|
||||
|
||||
## Sources of System Metrics {#system-tables-sources-of-system-metrics}
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# system.mutations {#system_tables-mutations}
|
||||
|
||||
The table contains information about [mutations](../../sql-reference/statements/alter.md#alter-mutations) of MergeTree tables and their progress. Each mutation command is represented by a single row. The table has the following columns:
|
||||
The table contains information about [mutations](../../sql-reference/statements/alter/index.md#alter-mutations) of MergeTree tables and their progress. Each mutation command is represented by a single row. The table has the following columns:
|
||||
|
||||
**database**, **table** - The name of the database and table to which the mutation was applied.
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@ The `system.part_log` table contains the following columns:
|
||||
- `NEW_PART` — Inserting of a new data part.
|
||||
- `MERGE_PARTS` — Merging of data parts.
|
||||
- `DOWNLOAD_PART` — Downloading a data part.
|
||||
- `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter.md#alter_detach-partition).
|
||||
- `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
|
||||
- `MUTATE_PART` — Mutating of a data part.
|
||||
- `MOVE_PART` — Moving the data part from the one disk to another one.
|
||||
- `event_date` (Date) — Event date.
|
||||
|
||||
@ -6,7 +6,7 @@ Each row describes one data part.
|
||||
|
||||
Columns:
|
||||
|
||||
- `partition` (String) – The partition name. To learn what a partition is, see the description of the [ALTER](../../sql-reference/statements/alter.md#query_language_queries_alter) query.
|
||||
- `partition` (String) – The partition name. To learn what a partition is, see the description of the [ALTER](../../sql-reference/statements/alter/index.md#query_language_queries_alter) query.
|
||||
|
||||
Formats:
|
||||
|
||||
@ -57,7 +57,7 @@ Columns:
|
||||
|
||||
- `primary_key_bytes_in_memory_allocated` (`UInt64`) – The amount of memory (in bytes) reserved for primary key values.
|
||||
|
||||
- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn’t exist. For more details, see [FREEZE PARTITION](../../sql-reference/statements/alter.md#alter_freeze-partition)
|
||||
- `is_frozen` (`UInt8`) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn’t exist. For more details, see [FREEZE PARTITION](../../sql-reference/statements/alter/partition.md#alter_freeze-partition)
|
||||
|
||||
- `database` (`String`) – Name of the database.
|
||||
|
||||
|
||||
17
docs/en/operations/system-tables/quota_limits.md
Normal file
17
docs/en/operations/system-tables/quota_limits.md
Normal file
@ -0,0 +1,17 @@
|
||||
# system.quota\_limits {#system_tables-quota_limits}
|
||||
|
||||
Contains information about maximums for all intervals of all quotas. Any number of rows or zero can correspond to one quota.
|
||||
|
||||
Columns:
|
||||
- `quota_name` ([String](../../sql-reference/data-types/string.md)) — Quota name.
|
||||
- `duration` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Length of the time interval for calculating resource consumption, in seconds.
|
||||
- `is_randomized_interval` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Logical value. It shows whether the interval is randomized. Interval always starts at the same time if it is not randomized. For example, an interval of 1 minute always starts at an integer number of minutes (i.e. it can start at 11:20:00, but it never starts at 11:20:01), an interval of one day always starts at midnight UTC. If interval is randomized, the very first interval starts at random time, and subsequent intervals starts one by one. Values:
|
||||
- `0` — Interval is not randomized.
|
||||
- `1` — Interval is randomized.
|
||||
- `max_queries` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of queries.
|
||||
- `max_errors` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of errors.
|
||||
- `max_result_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of result rows.
|
||||
- `max_result_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of RAM volume in bytes used to store a queries result.
|
||||
- `max_read_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of rows read from all tables and table functions participated in queries.
|
||||
- `max_read_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of bytes read from all tables and table functions participated in queries.
|
||||
- `max_execution_time` ([Nullable](../../sql-reference/data-types/nullable.md)([Float64](../../sql-reference/data-types/float.md))) — Maximum of the query execution time, in seconds.
|
||||
24
docs/en/operations/system-tables/quota_usage.md
Normal file
24
docs/en/operations/system-tables/quota_usage.md
Normal file
@ -0,0 +1,24 @@
|
||||
# system.quota\_usage {#system_tables-quota_usage}
|
||||
|
||||
Quota usage by the current user: how much is used and how much is left.
|
||||
|
||||
Columns:
|
||||
- `quota_name` ([String](../../sql-reference/data-types/string.md)) — Quota name.
|
||||
- `quota_key`([String](../../sql-reference/data-types/string.md)) — Key value. For example, if keys = \[`ip address`\], `quota_key` may have a value ‘192.168.1.1’.
|
||||
- `start_time`([Nullable](../../sql-reference/data-types/nullable.md)([DateTime](../../sql-reference/data-types/datetime.md))) — Start time for calculating resource consumption.
|
||||
- `end_time`([Nullable](../../sql-reference/data-types/nullable.md)([DateTime](../../sql-reference/data-types/datetime.md))) — End time for calculating resource consumption.
|
||||
- `duration` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Length of the time interval for calculating resource consumption, in seconds.
|
||||
- `queries` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The total number of requests on this interval.
|
||||
- `max_queries` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of requests.
|
||||
- `errors` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The number of queries that threw an exception.
|
||||
- `max_errors` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of errors.
|
||||
- `result_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The total number of rows given as a result.
|
||||
- `max_result_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of result rows.
|
||||
- `result_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — RAM volume in bytes used to store a queries result.
|
||||
- `max_result_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum RAM volume used to store a queries result, in bytes.
|
||||
- `read_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The total number of source rows read from tables for running the query on all remote servers.
|
||||
- `max_read_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of rows read from all tables and table functions participated in queries.
|
||||
- `read_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The total number of bytes read from all tables and table functions participated in queries.
|
||||
- `max_read_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum of bytes read from all tables and table functions.
|
||||
- `execution_time` ([Nullable](../../sql-reference/data-types/nullable.md)([Float64](../../sql-reference/data-types/float.md))) — The total query execution time, in seconds (wall time).
|
||||
- `max_execution_time` ([Nullable](../../sql-reference/data-types/nullable.md)([Float64](../../sql-reference/data-types/float.md))) — Maximum of query execution time.
|
||||
21
docs/en/operations/system-tables/quotas.md
Normal file
21
docs/en/operations/system-tables/quotas.md
Normal file
@ -0,0 +1,21 @@
|
||||
# system.quotas {#system_tables-quotas}
|
||||
|
||||
Contains information about [quotas](../../operations/system-tables/quotas.md).
|
||||
|
||||
Columns:
|
||||
- `name` ([String](../../sql-reference/data-types/string.md)) — Quota name.
|
||||
- `id` ([UUID](../../sql-reference/data-types/uuid.md)) — Quota ID.
|
||||
- `storage`([String](../../sql-reference/data-types/string.md)) — Storage of quotas. Possible value: “users.xml” if a quota configured in the users.xml file, “disk” if a quota configured by an SQL-query.
|
||||
- `keys` ([Array](../../sql-reference/data-types/array.md)([Enum8](../../sql-reference/data-types/enum.md))) — Key specifies how the quota should be shared. If two connections use the same quota and key, they share the same amounts of resources. Values:
|
||||
- `[]` — All users share the same quota.
|
||||
- `['user_name']` — Connections with the same user name share the same quota.
|
||||
- `['ip_address']` — Connections from the same IP share the same quota.
|
||||
- `['client_key']` — Connections with the same key share the same quota. A key must be explicitly provided by a client. When using [clickhouse-client](../../interfaces/cli.md), pass a key value in the `--quota-key` parameter, or use the `quota_key` parameter in the client configuration file. When using HTTP interface, use the `X-ClickHouse-Quota` header.
|
||||
- `['user_name', 'client_key']` — Connections with the same `client_key` share the same quota. If a key isn’t provided by a client, the qouta is tracked for `user_name`.
|
||||
- `['client_key', 'ip_address']` — Connections with the same `client_key` share the same quota. If a key isn’t provided by a client, the qouta is tracked for `ip_address`.
|
||||
- `durations` ([Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Time interval lengths in seconds.
|
||||
- `apply_to_all` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Logical value. It shows which users the quota is applied to. Values:
|
||||
- `0` — The quota applies to users specify in the `apply_to_list`.
|
||||
- `1` — The quota applies to all users except those listed in `apply_to_except`.
|
||||
- `apply_to_list` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — List of user names/[roles](../../operations/access-rights.md#role-management) that the quota should be applied to.
|
||||
- `apply_to_except` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — List of user names/roles that the quota should not apply to.
|
||||
25
docs/en/operations/system-tables/quotas_usage.md
Normal file
25
docs/en/operations/system-tables/quotas_usage.md
Normal file
@ -0,0 +1,25 @@
|
||||
# system.quotas\_usage {#system_tables-quotas_usage}
|
||||
|
||||
Quota usage by all users.
|
||||
|
||||
Columns:
|
||||
- `quota_name` ([String](../../sql-reference/data-types/string.md)) — Quota name.
|
||||
- `quota_key` ([String](../../sql-reference/data-types/string.md)) — Key value.
|
||||
- `is_current` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Quota usage for current user.
|
||||
- `start_time` ([Nullable](../../sql-reference/data-types/nullable.md)([DateTime](../../sql-reference/data-types/datetime.md)))) — Start time for calculating resource consumption.
|
||||
- `end_time` ([Nullable](../../sql-reference/data-types/nullable.md)([DateTime](../../sql-reference/data-types/datetime.md)))) — End time for calculating resource consumption.
|
||||
- `duration` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt32](../../sql-reference/data-types/int-uint.md))) — Length of the time interval for calculating resource consumption, in seconds.
|
||||
- `queries` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The total number of requests in this interval.
|
||||
- `max_queries` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of requests.
|
||||
- `errors` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The number of queries that threw an exception.
|
||||
- `max_errors` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of errors.
|
||||
- `result_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The total number of rows given as a result.
|
||||
- `max_result_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum of source rows read from tables.
|
||||
- `result_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — RAM volume in bytes used to store a queries result.
|
||||
- `max_result_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum RAM volume used to store a queries result, in bytes.
|
||||
- `read_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md)))) — The total number of source rows read from tables for running the query on all remote servers.
|
||||
- `max_read_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum number of rows read from all tables and table functions participated in queries.
|
||||
- `read_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The total number of bytes read from all tables and table functions participated in queries.
|
||||
- `max_read_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Maximum of bytes read from all tables and table functions.
|
||||
- `execution_time` ([Nullable](../../sql-reference/data-types/nullable.md)([Float64](../../sql-reference/data-types/float.md))) — The total query execution time, in seconds (wall time).
|
||||
- `max_execution_time` ([Nullable](../../sql-reference/data-types/nullable.md)([Float64](../../sql-reference/data-types/float.md))) — Maximum of query execution time.
|
||||
@ -12,7 +12,7 @@ ClickHouse supports the following types of queries:
|
||||
- [SELECT](../sql-reference/statements/select/index.md)
|
||||
- [INSERT INTO](../sql-reference/statements/insert-into.md)
|
||||
- [CREATE](../sql-reference/statements/create/index.md)
|
||||
- [ALTER](../sql-reference/statements/alter.md#query_language_queries_alter)
|
||||
- [ALTER](../sql-reference/statements/alter/index.md)
|
||||
- [Other types of queries](../sql-reference/statements/index.md)
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/sql-reference/) <!--hide-->
|
||||
|
||||
@ -1,600 +0,0 @@
|
||||
---
|
||||
toc_priority: 36
|
||||
toc_title: ALTER
|
||||
---
|
||||
|
||||
## ALTER Statement {#query_language_queries_alter}
|
||||
|
||||
The `ALTER` query is only supported for `*MergeTree` tables, as well as `Merge`and`Distributed`. The query has several variations.
|
||||
|
||||
### Column Manipulations {#column-manipulations}
|
||||
|
||||
Changing the table structure.
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ...
|
||||
```
|
||||
|
||||
In the query, specify a list of one or more comma-separated actions.
|
||||
Each action is an operation on a column.
|
||||
|
||||
The following actions are supported:
|
||||
|
||||
- [ADD COLUMN](#alter_add-column) — Adds a new column to the table.
|
||||
- [DROP COLUMN](#alter_drop-column) — Deletes the column.
|
||||
- [CLEAR COLUMN](#alter_clear-column) — Resets column values.
|
||||
- [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column.
|
||||
- [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL.
|
||||
|
||||
These actions are described in detail below.
|
||||
|
||||
#### ADD COLUMN {#alter_add-column}
|
||||
|
||||
``` sql
|
||||
ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after]
|
||||
```
|
||||
|
||||
Adds a new column to the table with the specified `name`, `type`, [`codec`](../../sql-reference/statements/create/table.md#codecs) and `default_expr` (see the section [Default expressions](../../sql-reference/statements/create/table.md#create-default-values)).
|
||||
|
||||
If the `IF NOT EXISTS` clause is included, the query won’t return an error if the column already exists. If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. Otherwise, the column is added to the end of the table. Note that there is no way to add a column to the beginning of a table. For a chain of actions, `name_after` can be the name of a column that is added in one of the previous actions.
|
||||
|
||||
Adding a column just changes the table structure, without performing any actions with data. The data doesn’t appear on the disk after `ALTER`. If the data is missing for a column when reading from the table, it is filled in with default values (by performing the default expression if there is one, or using zeros or empty strings). The column appears on the disk after merging data parts (see [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)).
|
||||
|
||||
This approach allows us to complete the `ALTER` query instantly, without increasing the volume of old data.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits ADD COLUMN browser String AFTER user_id
|
||||
```
|
||||
|
||||
#### DROP COLUMN {#alter_drop-column}
|
||||
|
||||
``` sql
|
||||
DROP COLUMN [IF EXISTS] name
|
||||
```
|
||||
|
||||
Deletes the column with the name `name`. If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist.
|
||||
|
||||
Deletes data from the file system. Since this deletes entire files, the query is completed almost instantly.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits DROP COLUMN browser
|
||||
```
|
||||
|
||||
#### CLEAR COLUMN {#alter_clear-column}
|
||||
|
||||
``` sql
|
||||
CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name
|
||||
```
|
||||
|
||||
Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits CLEAR COLUMN browser IN PARTITION tuple()
|
||||
```
|
||||
|
||||
#### COMMENT COLUMN {#alter_comment-column}
|
||||
|
||||
``` sql
|
||||
COMMENT COLUMN [IF EXISTS] name 'comment'
|
||||
```
|
||||
|
||||
Adds a comment to the column. If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist.
|
||||
|
||||
Each column can have one comment. If a comment already exists for the column, a new comment overwrites the previous comment.
|
||||
|
||||
Comments are stored in the `comment_expression` column returned by the [DESCRIBE TABLE](../../sql-reference/statements/misc.md#misc-describe-table) query.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits COMMENT COLUMN browser 'The table shows the browser used for accessing the site.'
|
||||
```
|
||||
|
||||
#### MODIFY COLUMN {#alter_modify-column}
|
||||
|
||||
``` sql
|
||||
MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL]
|
||||
```
|
||||
|
||||
This query changes the `name` column properties:
|
||||
|
||||
- Type
|
||||
|
||||
- Default expression
|
||||
|
||||
- TTL
|
||||
|
||||
For examples of columns TTL modifying, see [Column TTL](../engines/table_engines/mergetree_family/mergetree.md#mergetree-column-ttl).
|
||||
|
||||
If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist.
|
||||
|
||||
When changing the type, values are converted as if the [toType](../../sql-reference/functions/type-conversion-functions.md) functions were applied to them. If only the default expression is changed, the query doesn’t do anything complex, and is completed almost instantly.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits MODIFY COLUMN browser Array(String)
|
||||
```
|
||||
|
||||
Changing the column type is the only complex action – it changes the contents of files with data. For large tables, this may take a long time.
|
||||
|
||||
There are several processing stages:
|
||||
|
||||
- Preparing temporary (new) files with modified data.
|
||||
- Renaming old files.
|
||||
- Renaming the temporary (new) files to the old names.
|
||||
- Deleting the old files.
|
||||
|
||||
Only the first stage takes time. If there is a failure at this stage, the data is not changed.
|
||||
If there is a failure during one of the successive stages, data can be restored manually. The exception is if the old files were deleted from the file system but the data for the new files did not get written to the disk and was lost.
|
||||
|
||||
The `ALTER` query for changing columns is replicated. The instructions are saved in ZooKeeper, then each replica applies them. All `ALTER` queries are run in the same order. The query waits for the appropriate actions to be completed on the other replicas. However, a query to change columns in a replicated table can be interrupted, and all actions will be performed asynchronously.
|
||||
|
||||
#### ALTER Query Limitations {#alter-query-limitations}
|
||||
|
||||
The `ALTER` query lets you create and delete separate elements (columns) in nested data structures, but not whole nested data structures. To add a nested data structure, you can add columns with a name like `name.nested_name` and the type `Array(T)`. A nested data structure is equivalent to multiple array columns with a name that has the same prefix before the dot.
|
||||
|
||||
There is no support for deleting columns in the primary key or the sampling key (columns that are used in the `ENGINE` expression). Changing the type for columns that are included in the primary key is only possible if this change does not cause the data to be modified (for example, you are allowed to add values to an Enum or to change a type from `DateTime` to `UInt32`).
|
||||
|
||||
If the `ALTER` query is not sufficient to make the table changes you need, you can create a new table, copy the data to it using the [INSERT SELECT](../../sql-reference/statements/insert-into.md#insert_query_insert-select) query, then switch the tables using the [RENAME](../../sql-reference/statements/misc.md#misc_operations-rename) query and delete the old table. You can use the [clickhouse-copier](../../operations/utilities/clickhouse-copier.md) as an alternative to the `INSERT SELECT` query.
|
||||
|
||||
The `ALTER` query blocks all reads and writes for the table. In other words, if a long `SELECT` is running at the time of the `ALTER` query, the `ALTER` query will wait for it to complete. At the same time, all new queries to the same table will wait while this `ALTER` is running.
|
||||
|
||||
For tables that don’t store data themselves (such as `Merge` and `Distributed`), `ALTER` just changes the table structure, and does not change the structure of subordinate tables. For example, when running ALTER for a `Distributed` table, you will also need to run `ALTER` for the tables on all remote servers.
|
||||
|
||||
### Manipulations with Key Expressions {#manipulations-with-key-expressions}
|
||||
|
||||
The following command is supported:
|
||||
|
||||
``` sql
|
||||
MODIFY ORDER BY new_expression
|
||||
```
|
||||
|
||||
It only works for tables in the [`MergeTree`](../../engines/table-engines/mergetree-family/mergetree.md) family (including
|
||||
[replicated](../../engines/table-engines/mergetree-family/replication.md) tables). The command changes the
|
||||
[sorting key](../../engines/table-engines/mergetree-family/mergetree.md) of the table
|
||||
to `new_expression` (an expression or a tuple of expressions). Primary key remains the same.
|
||||
|
||||
The command is lightweight in a sense that it only changes metadata. To keep the property that data part
|
||||
rows are ordered by the sorting key expression you cannot add expressions containing existing columns
|
||||
to the sorting key (only columns added by the `ADD COLUMN` command in the same `ALTER` query).
|
||||
|
||||
### Manipulations with Data Skipping Indices {#manipulations-with-data-skipping-indices}
|
||||
|
||||
It only works for tables in the [`*MergeTree`](../../engines/table-engines/mergetree-family/mergetree.md) family (including
|
||||
[replicated](../../engines/table-engines/mergetree-family/replication.md) tables). The following operations
|
||||
are available:
|
||||
|
||||
- `ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value AFTER name [AFTER name2]` - Adds index description to tables metadata.
|
||||
|
||||
- `ALTER TABLE [db].name DROP INDEX name` - Removes index description from tables metadata and deletes index files from disk.
|
||||
|
||||
These commands are lightweight in a sense that they only change metadata or remove files.
|
||||
Also, they are replicated (syncing indices metadata through ZooKeeper).
|
||||
|
||||
### Manipulations with Constraints {#manipulations-with-constraints}
|
||||
|
||||
See more on [constraints](../../sql-reference/statements/create/table.md#constraints)
|
||||
|
||||
Constraints could be added or deleted using following syntax:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression;
|
||||
ALTER TABLE [db].name DROP CONSTRAINT constraint_name;
|
||||
```
|
||||
|
||||
Queries will add or remove metadata about constraints from table so they are processed immediately.
|
||||
|
||||
Constraint check *will not be executed* on existing data if it was added.
|
||||
|
||||
All changes on replicated tables are broadcasting to ZooKeeper so will be applied on other replicas.
|
||||
|
||||
### Manipulations with Partitions and Parts {#alter_manipulations-with-partitions}
|
||||
|
||||
The following operations with [partitions](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) are available:
|
||||
|
||||
- [DETACH PARTITION](#alter_detach-partition) — Moves a partition to the `detached` directory and forget it.
|
||||
- [DROP PARTITION](#alter_drop-partition) — Deletes a partition.
|
||||
- [ATTACH PART\|PARTITION](#alter_attach-partition) — Adds a part or partition from the `detached` directory to the table.
|
||||
- [ATTACH PARTITION FROM](#alter_attach-partition-from) — Copies the data partition from one table to another and adds.
|
||||
- [REPLACE PARTITION](#alter_replace-partition) — Copies the data partition from one table to another and replaces.
|
||||
- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) — Moves the data partition from one table to another.
|
||||
- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition.
|
||||
- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition.
|
||||
- [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition.
|
||||
- [FETCH PARTITION](#alter_fetch-partition) — Downloads a partition from another server.
|
||||
- [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume.
|
||||
|
||||
<!-- -->
|
||||
|
||||
#### DETACH PARTITION {#alter_detach-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name DETACH PARTITION partition_expr
|
||||
```
|
||||
|
||||
Moves all data for the specified partition to the `detached` directory. The server forgets about the detached data partition as if it does not exist. The server will not know about this data until you make the [ATTACH](#alter_attach-partition) query.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits DETACH PARTITION 201901
|
||||
```
|
||||
|
||||
Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
After the query is executed, you can do whatever you want with the data in the `detached` directory — delete it from the file system, or just leave it.
|
||||
|
||||
This query is replicated – it moves the data to the `detached` directory on all replicas. Note that you can execute this query only on a leader replica. To find out if a replica is a leader, perform the `SELECT` query to the [system.replicas](../../operations/system-tables/replicas.md#system_tables-replicas) table. Alternatively, it is easier to make a `DETACH` query on all replicas - all the replicas throw an exception, except the leader replica.
|
||||
|
||||
#### DROP PARTITION {#alter_drop-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name DROP PARTITION partition_expr
|
||||
```
|
||||
|
||||
Deletes the specified partition from the table. This query tags the partition as inactive and deletes data completely, approximately in 10 minutes.
|
||||
|
||||
Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
The query is replicated – it deletes data on all replicas.
|
||||
|
||||
#### DROP DETACHED PARTITION\|PART {#alter_drop-detached}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr
|
||||
```
|
||||
|
||||
Removes the specified part or all parts of the specified partition from `detached`.
|
||||
Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
#### ATTACH PARTITION\|PART {#alter_attach-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name ATTACH PARTITION|PART partition_expr
|
||||
```
|
||||
|
||||
Adds data to the table from the `detached` directory. It is possible to add data for an entire partition or for a separate part. Examples:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits ATTACH PARTITION 201901;
|
||||
ALTER TABLE visits ATTACH PART 201901_2_2_0;
|
||||
```
|
||||
|
||||
Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
This query is replicated. The replica-initiator checks whether there is data in the `detached` directory. If data exists, the query checks its integrity. If everything is correct, the query adds the data to the table. All other replicas download the data from the replica-initiator.
|
||||
|
||||
So you can put data to the `detached` directory on one replica, and use the `ALTER ... ATTACH` query to add it to the table on all replicas.
|
||||
|
||||
#### ATTACH PARTITION FROM {#alter_attach-partition-from}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1
|
||||
```
|
||||
|
||||
This query copies the data partition from the `table1` to `table2` adds data to exsisting in the `table2`. Note that data won’t be deleted from `table1`.
|
||||
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
|
||||
#### REPLACE PARTITION {#alter_replace-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1
|
||||
```
|
||||
|
||||
This query copies the data partition from the `table1` to `table2` and replaces existing partition in the `table2`. Note that data won’t be deleted from `table1`.
|
||||
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
|
||||
#### MOVE PARTITION TO TABLE {#alter_move_to_table-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest
|
||||
```
|
||||
|
||||
This query moves the data partition from the `table_source` to `table_dest` with deleting the data from `table_source`.
|
||||
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
- Both tables must be the same engine family (replicated or non-replicated).
|
||||
- Both tables must have the same storage policy.
|
||||
|
||||
#### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name CLEAR COLUMN column_name IN PARTITION partition_expr
|
||||
```
|
||||
|
||||
Resets all values in the specified column in a partition. If the `DEFAULT` clause was determined when creating a table, this query sets the column value to a specified default value.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902
|
||||
```
|
||||
|
||||
#### FREEZE PARTITION {#alter_freeze-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name FREEZE [PARTITION partition_expr]
|
||||
```
|
||||
|
||||
This query creates a local backup of a specified partition. If the `PARTITION` clause is omitted, the query creates the backup of all partitions at once.
|
||||
|
||||
!!! note "Note"
|
||||
The entire backup process is performed without stopping the server.
|
||||
|
||||
Note that for old-styled tables you can specify the prefix of the partition name (for example, ‘2019’) - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
At the time of execution, for a data snapshot, the query creates hardlinks to a table data. Hardlinks are placed in the directory `/var/lib/clickhouse/shadow/N/...`, where:
|
||||
|
||||
- `/var/lib/clickhouse/` is the working ClickHouse directory specified in the config.
|
||||
- `N` is the incremental number of the backup.
|
||||
|
||||
!!! note "Note"
|
||||
If you use [a set of disks for data storage in a table](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes), the `shadow/N` directory appears on every disk, storing data parts that matched by the `PARTITION` expression.
|
||||
|
||||
The same structure of directories is created inside the backup as inside `/var/lib/clickhouse/`. The query performs ‘chmod’ for all files, forbidding writing into them.
|
||||
|
||||
After creating the backup, you can copy the data from `/var/lib/clickhouse/shadow/` to the remote server and then delete it from the local server. Note that the `ALTER t FREEZE PARTITION` query is not replicated. It creates a local backup only on the local server.
|
||||
|
||||
The query creates backup almost instantly (but first it waits for the current queries to the corresponding table to finish running).
|
||||
|
||||
`ALTER TABLE t FREEZE PARTITION` copies only the data, not table metadata. To make a backup of table metadata, copy the file `/var/lib/clickhouse/metadata/database/table.sql`
|
||||
|
||||
To restore data from a backup, do the following:
|
||||
|
||||
1. Create the table if it does not exist. To view the query, use the .sql file (replace `ATTACH` in it with `CREATE`).
|
||||
2. Copy the data from the `data/database/table/` directory inside the backup to the `/var/lib/clickhouse/data/database/table/detached/` directory.
|
||||
3. Run `ALTER TABLE t ATTACH PARTITION` queries to add the data to a table.
|
||||
|
||||
Restoring from a backup doesn’t require stopping the server.
|
||||
|
||||
For more information about backups and restoring data, see the [Data Backup](../../operations/backup.md) section.
|
||||
|
||||
#### CLEAR INDEX IN PARTITION {#alter_clear-index-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr
|
||||
```
|
||||
|
||||
The query works similar to `CLEAR COLUMN`, but it resets an index instead of a column data.
|
||||
|
||||
#### FETCH PARTITION {#alter_fetch-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper'
|
||||
```
|
||||
|
||||
Downloads a partition from another server. This query only works for the replicated tables.
|
||||
|
||||
The query does the following:
|
||||
|
||||
1. Downloads the partition from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper.
|
||||
2. Then the query puts the downloaded data to the `detached` directory of the `table_name` table. Use the [ATTACH PARTITION\|PART](#alter_attach-partition) query to add the data to the table.
|
||||
|
||||
For example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits';
|
||||
ALTER TABLE users ATTACH PARTITION 201902;
|
||||
```
|
||||
|
||||
Note that:
|
||||
|
||||
- The `ALTER ... FETCH PARTITION` query isn’t replicated. It places the partition to the `detached` directory only on the local server.
|
||||
- The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas.
|
||||
|
||||
Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas.
|
||||
|
||||
Although the query is called `ALTER TABLE`, it does not change the table structure and does not immediately change the data available in the table.
|
||||
|
||||
#### MOVE PARTITION\|PART {#alter_move-partition}
|
||||
|
||||
Moves partitions or data parts to another volume or disk for `MergeTree`-engine tables. See [Using Multiple Block Devices for Data Storage](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes).
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'
|
||||
```
|
||||
|
||||
The `ALTER TABLE t MOVE` query:
|
||||
|
||||
- Not replicated, because different replicas can have different storage policies.
|
||||
- Returns an error if the specified disk or volume is not configured. Query also returns an error if conditions of data moving, that specified in the storage policy, can’t be applied.
|
||||
- Can return an error in the case, when data to be moved is already moved by a background process, concurrent `ALTER TABLE t MOVE` query or as a result of background data merging. A user shouldn’t perform any additional actions in this case.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow'
|
||||
ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd'
|
||||
```
|
||||
|
||||
#### How to Set Partition Expression {#alter-how-to-specify-part-expr}
|
||||
|
||||
You can specify the partition expression in `ALTER ... PARTITION` queries in different ways:
|
||||
|
||||
- As a value from the `partition` column of the `system.parts` table. For example, `ALTER TABLE visits DETACH PARTITION 201901`.
|
||||
- As the expression from the table column. Constants and constant expressions are supported. For example, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`.
|
||||
- Using the partition ID. Partition ID is a string identifier of the partition (human-readable, if possible) that is used as the names of partitions in the file system and in ZooKeeper. The partition ID must be specified in the `PARTITION ID` clause, in a single quotes. For example, `ALTER TABLE visits DETACH PARTITION ID '201901'`.
|
||||
- In the [ALTER ATTACH PART](#alter_attach-partition) and [DROP DETACHED PART](#alter_drop-detached) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached\_parts](../../operations/system-tables/detached_parts.md#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`.
|
||||
|
||||
Usage of quotes when specifying the partition depends on the type of partition expression. For example, for the `String` type, you have to specify its name in quotes (`'`). For the `Date` and `Int*` types no quotes are needed.
|
||||
|
||||
For old-style tables, you can specify the partition either as a number `201901` or a string `'201901'`. The syntax for the new-style tables is stricter with types (similar to the parser for the VALUES input format).
|
||||
|
||||
All the rules above are also true for the [OPTIMIZE](../../sql-reference/statements/misc.md#misc_operations-optimize) query. If you need to specify the only partition when optimizing a non-partitioned table, set the expression `PARTITION tuple()`. For example:
|
||||
|
||||
``` sql
|
||||
OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL;
|
||||
```
|
||||
|
||||
The examples of `ALTER ... PARTITION` queries are demonstrated in the tests [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
|
||||
|
||||
### Manipulations with Table TTL {#manipulations-with-table-ttl}
|
||||
|
||||
You can change [table TTL](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) with a request of the following form:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table-name MODIFY TTL ttl-expression
|
||||
```
|
||||
|
||||
### Synchronicity of ALTER Queries {#synchronicity-of-alter-queries}
|
||||
|
||||
For non-replicatable tables, all `ALTER` queries are performed synchronously. For replicatable tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas.
|
||||
|
||||
For `ALTER ... ATTACH|DETACH|DROP` queries, you can use the `replication_alter_partitions_sync` setting to set up waiting.
|
||||
Possible values: `0` – do not wait; `1` – only wait for own execution (default); `2` – wait for all.
|
||||
|
||||
### Mutations {#alter-mutations}
|
||||
|
||||
Mutations are an ALTER query variant that allows changing or deleting rows in a table. In contrast to standard `UPDATE` and `DELETE` queries that are intended for point data changes, mutations are intended for heavy operations that change a lot of rows in a table. Supported for the `MergeTree` family of table engines including the engines with replication support.
|
||||
|
||||
Existing tables are ready for mutations as-is (no conversion necessary), but after the first mutation is applied to a table, its metadata format becomes incompatible with previous server versions and falling back to a previous version becomes impossible.
|
||||
|
||||
Currently available commands:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table DELETE WHERE filter_expr
|
||||
```
|
||||
|
||||
The `filter_expr` must be of type `UInt8`. The query deletes rows in the table for which this expression takes a non-zero value.
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr
|
||||
```
|
||||
|
||||
The `filter_expr` must be of type `UInt8`. This query updates values of specified columns to the values of corresponding expressions in rows for which the `filter_expr` takes a non-zero value. Values are casted to the column type using the `CAST` operator. Updating columns that are used in the calculation of the primary or the partition key is not supported.
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name
|
||||
```
|
||||
|
||||
The query rebuilds the secondary index `name` in the partition `partition_name`.
|
||||
|
||||
One query can contain several commands separated by commas.
|
||||
|
||||
For \*MergeTree tables mutations execute by rewriting whole data parts. There is no atomicity - parts are substituted for mutated parts as soon as they are ready and a `SELECT` query that started executing during a mutation will see data from parts that have already been mutated along with data from parts that have not been mutated yet.
|
||||
|
||||
Mutations are totally ordered by their creation order and are applied to each part in that order. Mutations are also partially ordered with INSERTs - data that was inserted into the table before the mutation was submitted will be mutated and data that was inserted after that will not be mutated. Note that mutations do not block INSERTs in any way.
|
||||
|
||||
A mutation query returns immediately after the mutation entry is added (in case of replicated tables to ZooKeeper, for nonreplicated tables - to the filesystem). The mutation itself executes asynchronously using the system profile settings. To track the progress of mutations you can use the [`system.mutations`](../../operations/system-tables/mutations.md#system_tables-mutations) table. A mutation that was successfully submitted will continue to execute even if ClickHouse servers are restarted. There is no way to roll back the mutation once it is submitted, but if the mutation is stuck for some reason it can be cancelled with the [`KILL MUTATION`](../../sql-reference/statements/misc.md#kill-mutation) query.
|
||||
|
||||
Entries for finished mutations are not deleted right away (the number of preserved entries is determined by the `finished_mutations_to_keep` storage engine parameter). Older mutation entries are deleted.
|
||||
|
||||
## ALTER USER {#alter-user-statement}
|
||||
|
||||
Changes ClickHouse user accounts.
|
||||
|
||||
### Syntax {#alter-user-syntax}
|
||||
|
||||
``` sql
|
||||
ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
[IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
|
||||
[[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
### Description {#alter-user-dscr}
|
||||
|
||||
To use `ALTER USER` you must have the [ALTER USER](../../sql-reference/statements/grant.md#grant-access-management) privilege.
|
||||
|
||||
### Examples {#alter-user-examples}
|
||||
|
||||
Set assigned roles as default:
|
||||
|
||||
``` sql
|
||||
ALTER USER user DEFAULT ROLE role1, role2
|
||||
```
|
||||
|
||||
If roles aren’t previously assigned to a user, ClickHouse throws an exception.
|
||||
|
||||
Set all the assigned roles to default:
|
||||
|
||||
``` sql
|
||||
ALTER USER user DEFAULT ROLE ALL
|
||||
```
|
||||
|
||||
If a role is assigned to a user in the future, it will become default automatically.
|
||||
|
||||
Set all the assigned roles to default, excepting `role1` and `role2`:
|
||||
|
||||
``` sql
|
||||
ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2
|
||||
```
|
||||
|
||||
## ALTER ROLE {#alter-role-statement}
|
||||
|
||||
Changes roles.
|
||||
|
||||
### Syntax {#alter-role-syntax}
|
||||
|
||||
``` sql
|
||||
ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
## ALTER ROW POLICY {#alter-row-policy-statement}
|
||||
|
||||
Changes row policy.
|
||||
|
||||
### Syntax {#alter-row-policy-syntax}
|
||||
|
||||
``` sql
|
||||
ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table
|
||||
[RENAME TO new_name]
|
||||
[AS {PERMISSIVE | RESTRICTIVE}]
|
||||
[FOR SELECT]
|
||||
[USING {condition | NONE}][,...]
|
||||
[TO {role [,...] | ALL | ALL EXCEPT role [,...]}]
|
||||
```
|
||||
|
||||
## ALTER QUOTA {#alter-quota-statement}
|
||||
|
||||
Changes quotas.
|
||||
|
||||
### Syntax {#alter-quota-syntax}
|
||||
|
||||
``` sql
|
||||
ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
[KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}]
|
||||
[FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY}
|
||||
{MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] |
|
||||
NO LIMITS | TRACKING ONLY} [,...]]
|
||||
[TO {role [,...] | ALL | ALL EXCEPT role [,...]}]
|
||||
```
|
||||
|
||||
## ALTER SETTINGS PROFILE {#alter-settings-profile-statement}
|
||||
|
||||
Changes settings profiles.
|
||||
|
||||
### Syntax {#alter-settings-profile-syntax}
|
||||
|
||||
``` sql
|
||||
ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/query_language/alter/) <!--hide-->
|
||||
149
docs/en/sql-reference/statements/alter/column.md
Normal file
149
docs/en/sql-reference/statements/alter/column.md
Normal file
@ -0,0 +1,149 @@
|
||||
---
|
||||
toc_priority: 37
|
||||
toc_title: COLUMN
|
||||
---
|
||||
|
||||
# Column Manipulations {#column-manipulations}
|
||||
|
||||
A set of queries that allow changing the table structure.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ...
|
||||
```
|
||||
|
||||
In the query, specify a list of one or more comma-separated actions.
|
||||
Each action is an operation on a column.
|
||||
|
||||
The following actions are supported:
|
||||
|
||||
- [ADD COLUMN](#alter_add-column) — Adds a new column to the table.
|
||||
- [DROP COLUMN](#alter_drop-column) — Deletes the column.
|
||||
- [CLEAR COLUMN](#alter_clear-column) — Resets column values.
|
||||
- [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column.
|
||||
- [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL.
|
||||
|
||||
These actions are described in detail below.
|
||||
|
||||
## ADD COLUMN {#alter_add-column}
|
||||
|
||||
``` sql
|
||||
ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after]
|
||||
```
|
||||
|
||||
Adds a new column to the table with the specified `name`, `type`, [`codec`](../../../sql-reference/statements/create/table.md#codecs) and `default_expr` (see the section [Default expressions](../../../sql-reference/statements/create/table.md#create-default-values)).
|
||||
|
||||
If the `IF NOT EXISTS` clause is included, the query won’t return an error if the column already exists. If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. Otherwise, the column is added to the end of the table. Note that there is no way to add a column to the beginning of a table. For a chain of actions, `name_after` can be the name of a column that is added in one of the previous actions.
|
||||
|
||||
Adding a column just changes the table structure, without performing any actions with data. The data doesn’t appear on the disk after `ALTER`. If the data is missing for a column when reading from the table, it is filled in with default values (by performing the default expression if there is one, or using zeros or empty strings). The column appears on the disk after merging data parts (see [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md)).
|
||||
|
||||
This approach allows us to complete the `ALTER` query instantly, without increasing the volume of old data.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits ADD COLUMN browser String AFTER user_id
|
||||
```
|
||||
|
||||
## DROP COLUMN {#alter_drop-column}
|
||||
|
||||
``` sql
|
||||
DROP COLUMN [IF EXISTS] name
|
||||
```
|
||||
|
||||
Deletes the column with the name `name`. If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist.
|
||||
|
||||
Deletes data from the file system. Since this deletes entire files, the query is completed almost instantly.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits DROP COLUMN browser
|
||||
```
|
||||
|
||||
## CLEAR COLUMN {#alter_clear-column}
|
||||
|
||||
``` sql
|
||||
CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name
|
||||
```
|
||||
|
||||
Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits CLEAR COLUMN browser IN PARTITION tuple()
|
||||
```
|
||||
|
||||
## COMMENT COLUMN {#alter_comment-column}
|
||||
|
||||
``` sql
|
||||
COMMENT COLUMN [IF EXISTS] name 'comment'
|
||||
```
|
||||
|
||||
Adds a comment to the column. If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist.
|
||||
|
||||
Each column can have one comment. If a comment already exists for the column, a new comment overwrites the previous comment.
|
||||
|
||||
Comments are stored in the `comment_expression` column returned by the [DESCRIBE TABLE](../../../sql-reference/statements/misc.md#misc-describe-table) query.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits COMMENT COLUMN browser 'The table shows the browser used for accessing the site.'
|
||||
```
|
||||
|
||||
## MODIFY COLUMN {#alter_modify-column}
|
||||
|
||||
``` sql
|
||||
MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL]
|
||||
```
|
||||
|
||||
This query changes the `name` column properties:
|
||||
|
||||
- Type
|
||||
|
||||
- Default expression
|
||||
|
||||
- TTL
|
||||
|
||||
For examples of columns TTL modifying, see [Column TTL](../../engines/table_engines/mergetree_family/mergetree.md#mergetree-column-ttl).
|
||||
|
||||
If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist.
|
||||
|
||||
When changing the type, values are converted as if the [toType](../../../sql-reference/functions/type-conversion-functions.md) functions were applied to them. If only the default expression is changed, the query doesn’t do anything complex, and is completed almost instantly.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits MODIFY COLUMN browser Array(String)
|
||||
```
|
||||
|
||||
Changing the column type is the only complex action – it changes the contents of files with data. For large tables, this may take a long time.
|
||||
|
||||
There are several processing stages:
|
||||
|
||||
- Preparing temporary (new) files with modified data.
|
||||
- Renaming old files.
|
||||
- Renaming the temporary (new) files to the old names.
|
||||
- Deleting the old files.
|
||||
|
||||
Only the first stage takes time. If there is a failure at this stage, the data is not changed.
|
||||
If there is a failure during one of the successive stages, data can be restored manually. The exception is if the old files were deleted from the file system but the data for the new files did not get written to the disk and was lost.
|
||||
|
||||
The `ALTER` query for changing columns is replicated. The instructions are saved in ZooKeeper, then each replica applies them. All `ALTER` queries are run in the same order. The query waits for the appropriate actions to be completed on the other replicas. However, a query to change columns in a replicated table can be interrupted, and all actions will be performed asynchronously.
|
||||
|
||||
## Limitations {#alter-query-limitations}
|
||||
|
||||
The `ALTER` query lets you create and delete separate elements (columns) in nested data structures, but not whole nested data structures. To add a nested data structure, you can add columns with a name like `name.nested_name` and the type `Array(T)`. A nested data structure is equivalent to multiple array columns with a name that has the same prefix before the dot.
|
||||
|
||||
There is no support for deleting columns in the primary key or the sampling key (columns that are used in the `ENGINE` expression). Changing the type for columns that are included in the primary key is only possible if this change does not cause the data to be modified (for example, you are allowed to add values to an Enum or to change a type from `DateTime` to `UInt32`).
|
||||
|
||||
If the `ALTER` query is not sufficient to make the table changes you need, you can create a new table, copy the data to it using the [INSERT SELECT](../../../sql-reference/statements/insert-into.md#insert_query_insert-select) query, then switch the tables using the [RENAME](../../../sql-reference/statements/misc.md#misc_operations-rename) query and delete the old table. You can use the [clickhouse-copier](../../../operations/utilities/clickhouse-copier.md) as an alternative to the `INSERT SELECT` query.
|
||||
|
||||
The `ALTER` query blocks all reads and writes for the table. In other words, if a long `SELECT` is running at the time of the `ALTER` query, the `ALTER` query will wait for it to complete. At the same time, all new queries to the same table will wait while this `ALTER` is running.
|
||||
|
||||
For tables that don’t store data themselves (such as `Merge` and `Distributed`), `ALTER` just changes the table structure, and does not change the structure of subordinate tables. For example, when running ALTER for a `Distributed` table, you will also need to run `ALTER` for the tables on all remote servers.
|
||||
22
docs/en/sql-reference/statements/alter/constraint.md
Normal file
22
docs/en/sql-reference/statements/alter/constraint.md
Normal file
@ -0,0 +1,22 @@
|
||||
---
|
||||
toc_priority: 43
|
||||
toc_title: CONSTRAINT
|
||||
---
|
||||
|
||||
# Manipulating Constraints {#manipulations-with-constraints}
|
||||
|
||||
Constraints could be added or deleted using following syntax:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression;
|
||||
ALTER TABLE [db].name DROP CONSTRAINT constraint_name;
|
||||
```
|
||||
|
||||
See more on [constraints](../../../sql-reference/statements/create/table.md#constraints).
|
||||
|
||||
Queries will add or remove metadata about constraints from table so they are processed immediately.
|
||||
|
||||
!!! warning "Warning"
|
||||
Constraint check **will not be executed** on existing data if it was added.
|
||||
|
||||
All changes on replicated tables are broadcasted to ZooKeeper and will be applied on other replicas as well.
|
||||
19
docs/en/sql-reference/statements/alter/delete.md
Normal file
19
docs/en/sql-reference/statements/alter/delete.md
Normal file
@ -0,0 +1,19 @@
|
||||
---
|
||||
toc_priority: 39
|
||||
toc_title: DELETE
|
||||
---
|
||||
|
||||
# ALTER TABLE … DELETE Statement {#alter-mutations}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr
|
||||
```
|
||||
|
||||
Allows to asynchronously delete data matching the specified filtering expression. Implemented as a [mutation](../../../sql-reference/statements/index.md#mutations).
|
||||
|
||||
!!! note "Note"
|
||||
The `ALTER TABLE` prefix makes this syntax different from most other systems supporting SQL. It is intended to signify that unlike similar queries in OLTP databases this is a heavy operation not designed for frequent use.
|
||||
|
||||
The `filter_expr` must be of type `UInt8`. The query deletes rows in the table for which this expression takes a non-zero value.
|
||||
|
||||
One query can contain several commands separated by commas.
|
||||
48
docs/en/sql-reference/statements/alter/index.md
Normal file
48
docs/en/sql-reference/statements/alter/index.md
Normal file
@ -0,0 +1,48 @@
|
||||
---
|
||||
toc_priority: 36
|
||||
toc_title: ALTER
|
||||
---
|
||||
|
||||
## ALTER {#query_language_queries_alter}
|
||||
|
||||
Most `ALTER` queries modify table settings or data:
|
||||
|
||||
- [COLUMN](../../../sql-reference/statements/alter/column.md)
|
||||
- [PARTITION](../../../sql-reference/statements/alter/partition.md)
|
||||
- [DELETE](../../../sql-reference/statements/alter/delete.md)
|
||||
- [UPDATE](../../../sql-reference/statements/alter/update.md)
|
||||
- [ORDER BY](../../../sql-reference/statements/alter/order-by.md)
|
||||
- [INDEX](../../../sql-reference/statements/alter/index/index.md)
|
||||
- [CONSTRAINT](../../../sql-reference/statements/alter/constraint.md)
|
||||
- [TTL](../../../sql-reference/statements/alter/ttl.md)
|
||||
|
||||
!!! note "Note"
|
||||
Most `ALTER` queries are supported only for [\*MergeTree](../../../engines/table-engines/mergetree-family/index.md) tables, as well as [Merge](../../../engines/table-engines/special/merge.md) and [Distributed](../../../engines/table-engines/special/distributed.md).
|
||||
|
||||
While these `ALTER` settings modify entities related to role-based access control:
|
||||
|
||||
- [USER](../../../sql-reference/statements/alter/user.md)
|
||||
- [ROLE](../../../sql-reference/statements/alter/role.md)
|
||||
- [QUOTA](../../../sql-reference/statements/alter/quota.md)
|
||||
- [ROW POLICY](../../../sql-reference/statements/alter/row-policy.md)
|
||||
- [SETTINGS PROFILE](../../../sql-reference/statements/alter/settings-profile.md)
|
||||
|
||||
## Synchronicity of ALTER Queries {#synchronicity-of-alter-queries}
|
||||
|
||||
For non-replicated tables, all `ALTER` queries are performed synchronously. For replicated tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas.
|
||||
|
||||
For `ALTER ... ATTACH|DETACH|DROP` queries, you can use the `replication_alter_partitions_sync` setting to set up waiting. Possible values: `0` – do not wait; `1` – only wait for own execution (default); `2` – wait for all.
|
||||
|
||||
## Mutations {#mutations}
|
||||
|
||||
`ALTER` queries that are intended to manipulate table data are implemented with a mechanism called “mutations”, most notably [ALTER TABLE … DELETE](../../../sql-reference/statements/alter/delete.md) and [ALTER TABLE … UPDATE](../../../sql-reference/statements/alter/update.md). They are asynchronous background processes similar to merges in [MergeTree](../../../engines/table-engines/mergetree-family/index.md) tables that to produce new “mutated” versions of parts.
|
||||
|
||||
For `*MergeTree` tables mutations execute by **rewriting whole data parts**. There is no atomicity - parts are substituted for mutated parts as soon as they are ready and a `SELECT` query that started executing during a mutation will see data from parts that have already been mutated along with data from parts that have not been mutated yet.
|
||||
|
||||
Mutations are totally ordered by their creation order and are applied to each part in that order. Mutations are also partially ordered with `INSERT INTO` queries: data that was inserted into the table before the mutation was submitted will be mutated and data that was inserted after that will not be mutated. Note that mutations do not block inserts in any way.
|
||||
|
||||
A mutation query returns immediately after the mutation entry is added (in case of replicated tables to ZooKeeper, for non-replicated tables - to the filesystem). The mutation itself executes asynchronously using the system profile settings. To track the progress of mutations you can use the [`system.mutations`](../../../operations/system-tables/mutations.md#system_tables-mutations) table. A mutation that was successfully submitted will continue to execute even if ClickHouse servers are restarted. There is no way to roll back the mutation once it is submitted, but if the mutation is stuck for some reason it can be cancelled with the [`KILL MUTATION`](../../../sql-reference/statements/misc.md#kill-mutation) query.
|
||||
|
||||
Entries for finished mutations are not deleted right away (the number of preserved entries is determined by the `finished_mutations_to_keep` storage engine parameter). Older mutation entries are deleted.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/query_language/alter/) <!--hide-->
|
||||
23
docs/en/sql-reference/statements/alter/index/index.md
Normal file
23
docs/en/sql-reference/statements/alter/index/index.md
Normal file
@ -0,0 +1,23 @@
|
||||
---
|
||||
toc_hidden_folder: true
|
||||
toc_priority: 42
|
||||
toc_title: INDEX
|
||||
---
|
||||
|
||||
# Manipulating Data Skipping Indices {#manipulations-with-data-skipping-indices}
|
||||
|
||||
The following operations are available:
|
||||
|
||||
- `ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value AFTER name [AFTER name2]` - Adds index description to tables metadata.
|
||||
|
||||
- `ALTER TABLE [db].name DROP INDEX name` - Removes index description from tables metadata and deletes index files from disk.
|
||||
|
||||
- `ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name` - The query rebuilds the secondary index `name` in the partition `partition_name`. Implemented as a [mutation](../../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
The first two commands areare lightweight in a sense that they only change metadata or remove files.
|
||||
|
||||
Also, they are replicated, syncing indices metadata via ZooKeeper.
|
||||
|
||||
!!! note "Note"
|
||||
Index manipulation is supported only for tables with [`*MergeTree`](../../../../engines/table-engines/mergetree-family/mergetree.md) engine (including
|
||||
[replicated](../../../../engines/table-engines/mergetree-family/replication.md) variants).
|
||||
18
docs/en/sql-reference/statements/alter/order-by.md
Normal file
18
docs/en/sql-reference/statements/alter/order-by.md
Normal file
@ -0,0 +1,18 @@
|
||||
---
|
||||
toc_priority: 41
|
||||
toc_title: ORDER BY
|
||||
---
|
||||
|
||||
# Manipulating Key Expressions {#manipulations-with-key-expressions}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY ORDER BY new_expression
|
||||
```
|
||||
|
||||
The command changes the [sorting key](../../../engines/table-engines/mergetree-family/mergetree.md) of the table to `new_expression` (an expression or a tuple of expressions). Primary key remains the same.
|
||||
|
||||
The command is lightweight in a sense that it only changes metadata. To keep the property that data part rows are ordered by the sorting key expression you cannot add expressions containing existing columns to the sorting key (only columns added by the `ADD COLUMN` command in the same `ALTER` query).
|
||||
|
||||
!!! note "Note"
|
||||
It only works for tables in the [`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) family (including
|
||||
[replicated](../../../engines/table-engines/mergetree-family/replication.md) tables).
|
||||
253
docs/en/sql-reference/statements/alter/partition.md
Normal file
253
docs/en/sql-reference/statements/alter/partition.md
Normal file
@ -0,0 +1,253 @@
|
||||
---
|
||||
toc_priority: 38
|
||||
toc_title: PARTITION
|
||||
---
|
||||
|
||||
# Manipulating Partitions and Parts {#alter_manipulations-with-partitions}
|
||||
|
||||
The following operations with [partitions](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md) are available:
|
||||
|
||||
- [DETACH PARTITION](#alter_detach-partition) — Moves a partition to the `detached` directory and forget it.
|
||||
- [DROP PARTITION](#alter_drop-partition) — Deletes a partition.
|
||||
- [ATTACH PART\|PARTITION](#alter_attach-partition) — Adds a part or partition from the `detached` directory to the table.
|
||||
- [ATTACH PARTITION FROM](#alter_attach-partition-from) — Copies the data partition from one table to another and adds.
|
||||
- [REPLACE PARTITION](#alter_replace-partition) — Copies the data partition from one table to another and replaces.
|
||||
- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) — Moves the data partition from one table to another.
|
||||
- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition.
|
||||
- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition.
|
||||
- [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition.
|
||||
- [FETCH PARTITION](#alter_fetch-partition) — Downloads a partition from another server.
|
||||
- [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume.
|
||||
|
||||
<!-- -->
|
||||
|
||||
## DETACH PARTITION {#alter_detach-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name DETACH PARTITION partition_expr
|
||||
```
|
||||
|
||||
Moves all data for the specified partition to the `detached` directory. The server forgets about the detached data partition as if it does not exist. The server will not know about this data until you make the [ATTACH](#alter_attach-partition) query.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits DETACH PARTITION 201901
|
||||
```
|
||||
|
||||
Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
After the query is executed, you can do whatever you want with the data in the `detached` directory — delete it from the file system, or just leave it.
|
||||
|
||||
This query is replicated – it moves the data to the `detached` directory on all replicas. Note that you can execute this query only on a leader replica. To find out if a replica is a leader, perform the `SELECT` query to the [system.replicas](../../../operations/system-tables/replicas.md#system_tables-replicas) table. Alternatively, it is easier to make a `DETACH` query on all replicas - all the replicas throw an exception, except the leader replica.
|
||||
|
||||
## DROP PARTITION {#alter_drop-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name DROP PARTITION partition_expr
|
||||
```
|
||||
|
||||
Deletes the specified partition from the table. This query tags the partition as inactive and deletes data completely, approximately in 10 minutes.
|
||||
|
||||
Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
The query is replicated – it deletes data on all replicas.
|
||||
|
||||
## DROP DETACHED PARTITION\|PART {#alter_drop-detached}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr
|
||||
```
|
||||
|
||||
Removes the specified part or all parts of the specified partition from `detached`.
|
||||
Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
## ATTACH PARTITION\|PART {#alter_attach-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name ATTACH PARTITION|PART partition_expr
|
||||
```
|
||||
|
||||
Adds data to the table from the `detached` directory. It is possible to add data for an entire partition or for a separate part. Examples:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits ATTACH PARTITION 201901;
|
||||
ALTER TABLE visits ATTACH PART 201901_2_2_0;
|
||||
```
|
||||
|
||||
Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
This query is replicated. The replica-initiator checks whether there is data in the `detached` directory. If data exists, the query checks its integrity. If everything is correct, the query adds the data to the table. All other replicas download the data from the replica-initiator.
|
||||
|
||||
So you can put data to the `detached` directory on one replica, and use the `ALTER ... ATTACH` query to add it to the table on all replicas.
|
||||
|
||||
## ATTACH PARTITION FROM {#alter_attach-partition-from}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1
|
||||
```
|
||||
|
||||
This query copies the data partition from the `table1` to `table2` adds data to exsisting in the `table2`. Note that data won’t be deleted from `table1`.
|
||||
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
|
||||
## REPLACE PARTITION {#alter_replace-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1
|
||||
```
|
||||
|
||||
This query copies the data partition from the `table1` to `table2` and replaces existing partition in the `table2`. Note that data won’t be deleted from `table1`.
|
||||
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
|
||||
## MOVE PARTITION TO TABLE {#alter_move_to_table-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest
|
||||
```
|
||||
|
||||
This query moves the data partition from the `table_source` to `table_dest` with deleting the data from `table_source`.
|
||||
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
- Both tables must be the same engine family (replicated or non-replicated).
|
||||
- Both tables must have the same storage policy.
|
||||
|
||||
## CLEAR COLUMN IN PARTITION {#alter_clear-column-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name CLEAR COLUMN column_name IN PARTITION partition_expr
|
||||
```
|
||||
|
||||
Resets all values in the specified column in a partition. If the `DEFAULT` clause was determined when creating a table, this query sets the column value to a specified default value.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902
|
||||
```
|
||||
|
||||
## FREEZE PARTITION {#alter_freeze-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name FREEZE [PARTITION partition_expr]
|
||||
```
|
||||
|
||||
This query creates a local backup of a specified partition. If the `PARTITION` clause is omitted, the query creates the backup of all partitions at once.
|
||||
|
||||
!!! note "Note"
|
||||
The entire backup process is performed without stopping the server.
|
||||
|
||||
Note that for old-styled tables you can specify the prefix of the partition name (for example, ‘2019’) - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
|
||||
|
||||
At the time of execution, for a data snapshot, the query creates hardlinks to a table data. Hardlinks are placed in the directory `/var/lib/clickhouse/shadow/N/...`, where:
|
||||
|
||||
- `/var/lib/clickhouse/` is the working ClickHouse directory specified in the config.
|
||||
- `N` is the incremental number of the backup.
|
||||
|
||||
!!! note "Note"
|
||||
If you use [a set of disks for data storage in a table](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes), the `shadow/N` directory appears on every disk, storing data parts that matched by the `PARTITION` expression.
|
||||
|
||||
The same structure of directories is created inside the backup as inside `/var/lib/clickhouse/`. The query performs ‘chmod’ for all files, forbidding writing into them.
|
||||
|
||||
After creating the backup, you can copy the data from `/var/lib/clickhouse/shadow/` to the remote server and then delete it from the local server. Note that the `ALTER t FREEZE PARTITION` query is not replicated. It creates a local backup only on the local server.
|
||||
|
||||
The query creates backup almost instantly (but first it waits for the current queries to the corresponding table to finish running).
|
||||
|
||||
`ALTER TABLE t FREEZE PARTITION` copies only the data, not table metadata. To make a backup of table metadata, copy the file `/var/lib/clickhouse/metadata/database/table.sql`
|
||||
|
||||
To restore data from a backup, do the following:
|
||||
|
||||
1. Create the table if it does not exist. To view the query, use the .sql file (replace `ATTACH` in it with `CREATE`).
|
||||
2. Copy the data from the `data/database/table/` directory inside the backup to the `/var/lib/clickhouse/data/database/table/detached/` directory.
|
||||
3. Run `ALTER TABLE t ATTACH PARTITION` queries to add the data to a table.
|
||||
|
||||
Restoring from a backup doesn’t require stopping the server.
|
||||
|
||||
For more information about backups and restoring data, see the [Data Backup](../../../operations/backup.md) section.
|
||||
|
||||
## CLEAR INDEX IN PARTITION {#alter_clear-index-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr
|
||||
```
|
||||
|
||||
The query works similar to `CLEAR COLUMN`, but it resets an index instead of a column data.
|
||||
|
||||
## FETCH PARTITION {#alter_fetch-partition}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper'
|
||||
```
|
||||
|
||||
Downloads a partition from another server. This query only works for the replicated tables.
|
||||
|
||||
The query does the following:
|
||||
|
||||
1. Downloads the partition from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper.
|
||||
2. Then the query puts the downloaded data to the `detached` directory of the `table_name` table. Use the [ATTACH PARTITION\|PART](#alter_attach-partition) query to add the data to the table.
|
||||
|
||||
For example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits';
|
||||
ALTER TABLE users ATTACH PARTITION 201902;
|
||||
```
|
||||
|
||||
Note that:
|
||||
|
||||
- The `ALTER ... FETCH PARTITION` query isn’t replicated. It places the partition to the `detached` directory only on the local server.
|
||||
- The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas.
|
||||
|
||||
Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas.
|
||||
|
||||
Although the query is called `ALTER TABLE`, it does not change the table structure and does not immediately change the data available in the table.
|
||||
|
||||
## MOVE PARTITION\|PART {#alter_move-partition}
|
||||
|
||||
Moves partitions or data parts to another volume or disk for `MergeTree`-engine tables. See [Using Multiple Block Devices for Data Storage](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes).
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'
|
||||
```
|
||||
|
||||
The `ALTER TABLE t MOVE` query:
|
||||
|
||||
- Not replicated, because different replicas can have different storage policies.
|
||||
- Returns an error if the specified disk or volume is not configured. Query also returns an error if conditions of data moving, that specified in the storage policy, can’t be applied.
|
||||
- Can return an error in the case, when data to be moved is already moved by a background process, concurrent `ALTER TABLE t MOVE` query or as a result of background data merging. A user shouldn’t perform any additional actions in this case.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow'
|
||||
ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd'
|
||||
```
|
||||
|
||||
## How to Set Partition Expression {#alter-how-to-specify-part-expr}
|
||||
|
||||
You can specify the partition expression in `ALTER ... PARTITION` queries in different ways:
|
||||
|
||||
- As a value from the `partition` column of the `system.parts` table. For example, `ALTER TABLE visits DETACH PARTITION 201901`.
|
||||
- As the expression from the table column. Constants and constant expressions are supported. For example, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`.
|
||||
- Using the partition ID. Partition ID is a string identifier of the partition (human-readable, if possible) that is used as the names of partitions in the file system and in ZooKeeper. The partition ID must be specified in the `PARTITION ID` clause, in a single quotes. For example, `ALTER TABLE visits DETACH PARTITION ID '201901'`.
|
||||
- In the [ALTER ATTACH PART](#alter_attach-partition) and [DROP DETACHED PART](#alter_drop-detached) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached\_parts](../../../operations/system-tables/detached_parts.md#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`.
|
||||
|
||||
Usage of quotes when specifying the partition depends on the type of partition expression. For example, for the `String` type, you have to specify its name in quotes (`'`). For the `Date` and `Int*` types no quotes are needed.
|
||||
|
||||
All the rules above are also true for the [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) query. If you need to specify the only partition when optimizing a non-partitioned table, set the expression `PARTITION tuple()`. For example:
|
||||
|
||||
``` sql
|
||||
OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL;
|
||||
```
|
||||
|
||||
The examples of `ALTER ... PARTITION` queries are demonstrated in the tests [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
|
||||
20
docs/en/sql-reference/statements/alter/quota.md
Normal file
20
docs/en/sql-reference/statements/alter/quota.md
Normal file
@ -0,0 +1,20 @@
|
||||
---
|
||||
toc_priority: 46
|
||||
toc_title: QUOTA
|
||||
---
|
||||
|
||||
# ALTER QUOTA {#alter-quota-statement}
|
||||
|
||||
Changes quotas.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
[KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}]
|
||||
[FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY}
|
||||
{MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] |
|
||||
NO LIMITS | TRACKING ONLY} [,...]]
|
||||
[TO {role [,...] | ALL | ALL EXCEPT role [,...]}]
|
||||
```
|
||||
16
docs/en/sql-reference/statements/alter/role.md
Normal file
16
docs/en/sql-reference/statements/alter/role.md
Normal file
@ -0,0 +1,16 @@
|
||||
---
|
||||
toc_priority: 46
|
||||
toc_title: ROLE
|
||||
---
|
||||
|
||||
## ALTER ROLE {#alter-role-statement}
|
||||
|
||||
Changes roles.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
19
docs/en/sql-reference/statements/alter/row-policy.md
Normal file
19
docs/en/sql-reference/statements/alter/row-policy.md
Normal file
@ -0,0 +1,19 @@
|
||||
---
|
||||
toc_priority: 47
|
||||
toc_title: ROW POLICY
|
||||
---
|
||||
|
||||
# ALTER ROW POLICY {#alter-row-policy-statement}
|
||||
|
||||
Changes row policy.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table
|
||||
[RENAME TO new_name]
|
||||
[AS {PERMISSIVE | RESTRICTIVE}]
|
||||
[FOR SELECT]
|
||||
[USING {condition | NONE}][,...]
|
||||
[TO {role [,...] | ALL | ALL EXCEPT role [,...]}]
|
||||
```
|
||||
16
docs/en/sql-reference/statements/alter/settings-profile.md
Normal file
16
docs/en/sql-reference/statements/alter/settings-profile.md
Normal file
@ -0,0 +1,16 @@
|
||||
---
|
||||
toc_priority: 48
|
||||
toc_title: SETTINGS PROFILE
|
||||
---
|
||||
|
||||
## ALTER SETTINGS PROFILE {#alter-settings-profile-statement}
|
||||
|
||||
Changes settings profiles.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...]
|
||||
```
|
||||
12
docs/en/sql-reference/statements/alter/ttl.md
Normal file
12
docs/en/sql-reference/statements/alter/ttl.md
Normal file
@ -0,0 +1,12 @@
|
||||
---
|
||||
toc_priority: 44
|
||||
toc_title: TTL
|
||||
---
|
||||
|
||||
### Manipulations with Table TTL {#manipulations-with-table-ttl}
|
||||
|
||||
You can change [table TTL](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) with a request of the following form:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE table-name MODIFY TTL ttl-expression
|
||||
```
|
||||
19
docs/en/sql-reference/statements/alter/update.md
Normal file
19
docs/en/sql-reference/statements/alter/update.md
Normal file
@ -0,0 +1,19 @@
|
||||
---
|
||||
toc_priority: 40
|
||||
toc_title: UPDATE
|
||||
---
|
||||
|
||||
# ALTER TABLE … UPDATE Statements {#alter-table-update-statements}
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr
|
||||
```
|
||||
|
||||
Allows to asynchronously manipulate data matching the specified filtering expression. Implemented as a [mutation](../../../sql-reference/statements/index.md#mutations).
|
||||
|
||||
!!! note "Note"
|
||||
The `ALTER TABLE` prefix makes this syntax different from most other systems supporting SQL. It is intended to signify that unlike similar queries in OLTP databases this is a heavy operation not designed for frequent use.
|
||||
|
||||
The `filter_expr` must be of type `UInt8`. This query updates values of specified columns to the values of corresponding expressions in rows for which the `filter_expr` takes a non-zero value. Values are casted to the column type using the `CAST` operator. Updating columns that are used in the calculation of the primary or the partition key is not supported.
|
||||
|
||||
One query can contain several commands separated by commas.
|
||||
45
docs/en/sql-reference/statements/alter/user.md
Normal file
45
docs/en/sql-reference/statements/alter/user.md
Normal file
@ -0,0 +1,45 @@
|
||||
---
|
||||
toc_priority: 45
|
||||
toc_title: USER
|
||||
---
|
||||
|
||||
# ALTER USER {#alter-user-statement}
|
||||
|
||||
Changes ClickHouse user accounts.
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name]
|
||||
[RENAME TO new_name]
|
||||
[IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
|
||||
[[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
To use `ALTER USER` you must have the [ALTER USER](../../../sql-reference/statements/grant.md#grant-access-management) privilege.
|
||||
|
||||
## Examples {#alter-user-examples}
|
||||
|
||||
Set assigned roles as default:
|
||||
|
||||
``` sql
|
||||
ALTER USER user DEFAULT ROLE role1, role2
|
||||
```
|
||||
|
||||
If roles aren’t previously assigned to a user, ClickHouse throws an exception.
|
||||
|
||||
Set all the assigned roles to default:
|
||||
|
||||
``` sql
|
||||
ALTER USER user DEFAULT ROLE ALL
|
||||
```
|
||||
|
||||
If a role is assigned to a user in the future, it will become default automatically.
|
||||
|
||||
Set all the assigned roles to default, excepting `role1` and `role2`:
|
||||
|
||||
``` sql
|
||||
ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2
|
||||
```
|
||||
@ -18,7 +18,7 @@ CREATE ROLE [IF NOT EXISTS | OR REPLACE] name
|
||||
|
||||
A user can be assigned multiple roles. Users can apply their assigned roles in arbitrary combinations by the [SET ROLE](../../../sql-reference/statements/set-role.md) statement. The final scope of privileges is a combined set of all the privileges of all the applied roles. If a user has privileges granted directly to it’s user account, they are also combined with the privileges granted by roles.
|
||||
|
||||
User can have default roles which apply at user login. To set default roles, use the [SET DEFAULT ROLE](../../../sql-reference/statements/set-role.md#set-default-role-statement) statement or the [ALTER USER](../../../sql-reference/statements/alter.md#alter-user-statement) statement.
|
||||
User can have default roles which apply at user login. To set default roles, use the [SET DEFAULT ROLE](../../../sql-reference/statements/set-role.md#set-default-role-statement) statement or the [ALTER USER](../../../sql-reference/statements/alter/user.md#alter-user-statement) statement.
|
||||
|
||||
To revoke a role, use the [REVOKE](../../../sql-reference/statements/revoke.md) statement.
|
||||
|
||||
|
||||
@ -52,7 +52,7 @@ If you specify `POPULATE`, the existing table data is inserted in the view when
|
||||
|
||||
A `SELECT` query can contain `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`… Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` is set, data is aggregated during insertion, but only within a single packet of inserted data. The data won’t be further aggregated. The exception is when using an `ENGINE` that independently performs data aggregation, such as `SummingMergeTree`.
|
||||
|
||||
The execution of [ALTER](../../../sql-reference/statements/alter.md) queries on materialized views has limitations, so they might be inconvenient. If the materialized view uses the construction `TO [db.]name`, you can `DETACH` the view, run `ALTER` for the target table, and then `ATTACH` the previously detached (`DETACH`) view.
|
||||
The execution of [ALTER](../../../sql-reference/statements/alter/index.md) queries on materialized views has limitations, so they might be inconvenient. If the materialized view uses the construction `TO [db.]name`, you can `DETACH` the view, run `ALTER` for the target table, and then `ATTACH` the previously detached (`DETACH`) view.
|
||||
|
||||
Views look the same as normal tables. For example, they are listed in the result of the `SHOW TABLES` query.
|
||||
|
||||
|
||||
@ -251,7 +251,7 @@ The granted privilege allows `john` to insert data to the `x` and/or `y` columns
|
||||
|
||||
### ALTER {#grant-alter}
|
||||
|
||||
Allows executing [ALTER](../../sql-reference/statements/alter.md) queries according to the following hierarchy of privileges:
|
||||
Allows executing [ALTER](../../sql-reference/statements/alter/index.md) queries according to the following hierarchy of privileges:
|
||||
|
||||
- `ALTER`. Level: `COLUMN`.
|
||||
- `ALTER TABLE`. Level: `GROUP`
|
||||
|
||||
@ -11,7 +11,7 @@ Statements represent various kinds of action you can perform using SQL queries.
|
||||
- [SELECT](../../sql-reference/statements/select/index.md)
|
||||
- [INSERT INTO](../../sql-reference/statements/insert-into.md)
|
||||
- [CREATE](../../sql-reference/statements/create/index.md)
|
||||
- [ALTER](../../sql-reference/statements/alter.md)
|
||||
- [ALTER](../../sql-reference/statements/alter/index.md)
|
||||
- [SYSTEM](../../sql-reference/statements/system.md)
|
||||
- [SHOW](../../sql-reference/statements/show.md)
|
||||
- [GRANT](../../sql-reference/statements/grant.md)
|
||||
|
||||
@ -50,7 +50,7 @@ KILL MUTATION [ON CLUSTER cluster]
|
||||
[FORMAT format]
|
||||
```
|
||||
|
||||
Tries to cancel and remove [mutations](../../sql-reference/statements/alter.md#alter-mutations) that are currently executing. Mutations to cancel are selected from the [`system.mutations`](../../operations/system-tables/mutations.md#system_tables-mutations) table using the filter specified by the `WHERE` clause of the `KILL` query.
|
||||
Tries to cancel and remove [mutations](../../sql-reference/statements/alter/index.md#alter-mutations) that are currently executing. Mutations to cancel are selected from the [`system.mutations`](../../operations/system-tables/mutations.md#system_tables-mutations) table using the filter specified by the `WHERE` clause of the `KILL` query.
|
||||
|
||||
A test query (`TEST`) only checks the user’s rights and displays a list of queries to stop.
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@ The `OPTMIZE` query is also supported for the [MaterializedView](../../engines/t
|
||||
When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all nodes (if the `replication_alter_partitions_sync` setting is enabled).
|
||||
|
||||
- If `OPTIMIZE` doesn’t perform a merge for any reason, it doesn’t notify the client. To enable notifications, use the [optimize\_throw\_if\_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting.
|
||||
- If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter.md#alter-how-to-specify-part-expr).
|
||||
- If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter/index.md#alter-how-to-specify-part-expr).
|
||||
- If you specify `FINAL`, optimization is performed even when all the data is already in one part.
|
||||
- If you specify `DEDUPLICATE`, then completely identical rows will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine.
|
||||
|
||||
|
||||
@ -22,9 +22,9 @@ SELECT [DISTINCT] expr_list
|
||||
[WHERE expr]
|
||||
[GROUP BY expr_list] [WITH TOTALS]
|
||||
[HAVING expr]
|
||||
[ORDER BY expr_list]
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[UNION ALL ...]
|
||||
[INTO OUTFILE filename]
|
||||
[FORMAT format]
|
||||
|
||||
@ -11,3 +11,52 @@ toc_title: LIMIT
|
||||
`n` and `m` must be non-negative integers.
|
||||
|
||||
If there is no [ORDER BY](../../../sql-reference/statements/select/order-by.md) clause that explicitly sorts results, the choice of rows for the result may be arbitrary and non-deterministic.
|
||||
|
||||
## LIMIT … WITH TIES Modifier {#limit-with-ties}
|
||||
|
||||
When you set `WITH TIES` modifier for `LIMIT n[,m]` and specify `ORDER BY expr_list`, you will get in result first `n` or `n,m` rows and all rows with same `ORDER BY` fields values equal to row at position `n` for `LIMIT n` and `m` for `LIMIT n,m`.
|
||||
|
||||
This modifier also can be combined with [ORDER BY … WITH FILL modifier](../../../sql-reference/statements/select/order-by.md#orderby-with-fill).
|
||||
|
||||
For example, the following query
|
||||
|
||||
``` sql
|
||||
SELECT * FROM (
|
||||
SELECT number%50 AS n FROM numbers(100)
|
||||
) ORDER BY n LIMIT 0,5
|
||||
```
|
||||
|
||||
returns
|
||||
|
||||
``` text
|
||||
┌─n─┐
|
||||
│ 0 │
|
||||
│ 0 │
|
||||
│ 1 │
|
||||
│ 1 │
|
||||
│ 2 │
|
||||
└───┘
|
||||
```
|
||||
|
||||
but after apply `WITH TIES` modifier
|
||||
|
||||
``` sql
|
||||
SELECT * FROM (
|
||||
SELECT number%50 AS n FROM numbers(100)
|
||||
) ORDER BY n LIMIT 0,5 WITH TIES
|
||||
```
|
||||
|
||||
it returns another rows set
|
||||
|
||||
``` text
|
||||
┌─n─┐
|
||||
│ 0 │
|
||||
│ 0 │
|
||||
│ 1 │
|
||||
│ 1 │
|
||||
│ 2 │
|
||||
│ 2 │
|
||||
└───┘
|
||||
```
|
||||
|
||||
cause row number 6 have same value “2” for field `n` as row number 5
|
||||
|
||||
@ -69,3 +69,136 @@ If there is not enough RAM, it is possible to perform sorting in external memory
|
||||
Running a query may use more memory than `max_bytes_before_external_sort`. For this reason, this setting must have a value significantly smaller than `max_memory_usage`. As an example, if your server has 128 GB of RAM and you need to run a single query, set `max_memory_usage` to 100 GB, and `max_bytes_before_external_sort` to 80 GB.
|
||||
|
||||
External sorting works much less effectively than sorting in RAM.
|
||||
|
||||
## ORDER BY Expr WITH FILL Modifier {#orderby-with-fill}
|
||||
|
||||
This modifier also can be combined with [LIMIT … WITH TIES modifier](../../../sql-reference/statements/select/limit.md#limit-with-ties).
|
||||
|
||||
`WITH FILL` modifier can be set after `ORDER BY expr` with optional `FROM expr`, `TO expr` and `STEP expr` parameters.
|
||||
All missed values of `expr` column will be filled sequentially and other columns will be filled as defaults.
|
||||
|
||||
Use following syntax for filling multiple columns add `WITH FILL` modifier with optional parameters after each field name in `ORDER BY` section.
|
||||
|
||||
``` sql
|
||||
ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr]
|
||||
```
|
||||
|
||||
`WITH FILL` can be applied only for fields with Numeric (all kind of float, decimal, int) or Date/DateTime types.
|
||||
When `FROM const_expr` not defined sequence of filling use minimal `expr` field value from `ORDER BY`.
|
||||
When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`.
|
||||
When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types as `days` for Date type and as `seconds` for DateTime type.
|
||||
When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type.
|
||||
|
||||
For example, the following query
|
||||
|
||||
``` sql
|
||||
SELECT n, source FROM (
|
||||
SELECT toFloat32(number % 10) AS n, 'original' AS source
|
||||
FROM numbers(10) WHERE number % 3 = 1
|
||||
) ORDER BY n
|
||||
```
|
||||
|
||||
returns
|
||||
|
||||
``` text
|
||||
┌─n─┬─source───┐
|
||||
│ 1 │ original │
|
||||
│ 4 │ original │
|
||||
│ 7 │ original │
|
||||
└───┴──────────┘
|
||||
```
|
||||
|
||||
but after apply `WITH FILL` modifier
|
||||
|
||||
``` sql
|
||||
SELECT n, source FROM (
|
||||
SELECT toFloat32(number % 10) AS n, 'original' AS source
|
||||
FROM numbers(10) WHERE number % 3 = 1
|
||||
) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5
|
||||
```
|
||||
|
||||
returns
|
||||
|
||||
``` text
|
||||
┌───n─┬─source───┐
|
||||
│ 0 │ │
|
||||
│ 0.5 │ │
|
||||
│ 1 │ original │
|
||||
│ 1.5 │ │
|
||||
│ 2 │ │
|
||||
│ 2.5 │ │
|
||||
│ 3 │ │
|
||||
│ 3.5 │ │
|
||||
│ 4 │ original │
|
||||
│ 4.5 │ │
|
||||
│ 5 │ │
|
||||
│ 5.5 │ │
|
||||
│ 7 │ original │
|
||||
└─────┴──────────┘
|
||||
```
|
||||
|
||||
For the case when we have multiple fields `ORDER BY field2 WITH FILL, field1 WITH FILL` order of filling will follow the order of fields in `ORDER BY` clause.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
toDate((number * 10) * 86400) AS d1,
|
||||
toDate(number * 86400) AS d2,
|
||||
'original' AS source
|
||||
FROM numbers(10)
|
||||
WHERE (number % 3) = 1
|
||||
ORDER BY
|
||||
d2 WITH FILL,
|
||||
d1 WITH FILL STEP 5;
|
||||
```
|
||||
|
||||
returns
|
||||
|
||||
``` text
|
||||
┌───d1───────┬───d2───────┬─source───┐
|
||||
│ 1970-01-11 │ 1970-01-02 │ original │
|
||||
│ 0000-00-00 │ 1970-01-03 │ │
|
||||
│ 0000-00-00 │ 1970-01-04 │ │
|
||||
│ 1970-02-10 │ 1970-01-05 │ original │
|
||||
│ 0000-00-00 │ 1970-01-06 │ │
|
||||
│ 0000-00-00 │ 1970-01-07 │ │
|
||||
│ 1970-03-12 │ 1970-01-08 │ original │
|
||||
└────────────┴────────────┴──────────┘
|
||||
```
|
||||
|
||||
Field `d1` doesn’t fill and use default value cause we don’t have repeated values for `d2` value, and sequence for `d1` can’t be properly calculated.
|
||||
|
||||
The following query with a changed field in `ORDER BY`
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
toDate((number * 10) * 86400) AS d1,
|
||||
toDate(number * 86400) AS d2,
|
||||
'original' AS source
|
||||
FROM numbers(10)
|
||||
WHERE (number % 3) = 1
|
||||
ORDER BY
|
||||
d1 WITH FILL STEP 5,
|
||||
d2 WITH FILL;
|
||||
```
|
||||
|
||||
returns
|
||||
|
||||
``` text
|
||||
┌───d1───────┬───d2───────┬─source───┐
|
||||
│ 1970-01-11 │ 1970-01-02 │ original │
|
||||
│ 1970-01-16 │ 0000-00-00 │ │
|
||||
│ 1970-01-21 │ 0000-00-00 │ │
|
||||
│ 1970-01-26 │ 0000-00-00 │ │
|
||||
│ 1970-01-31 │ 0000-00-00 │ │
|
||||
│ 1970-02-05 │ 0000-00-00 │ │
|
||||
│ 1970-02-10 │ 1970-01-05 │ original │
|
||||
│ 1970-02-15 │ 0000-00-00 │ │
|
||||
│ 1970-02-20 │ 0000-00-00 │ │
|
||||
│ 1970-02-25 │ 0000-00-00 │ │
|
||||
│ 1970-03-02 │ 0000-00-00 │ │
|
||||
│ 1970-03-07 │ 0000-00-00 │ │
|
||||
│ 1970-03-12 │ 1970-01-08 │ original │
|
||||
└────────────┴────────────┴──────────┘
|
||||
```
|
||||
|
||||
@ -20,9 +20,9 @@ SELECT [DISTINCT] expr_list
|
||||
[WHERE expr]
|
||||
[GROUP BY expr_list] [WITH TOTALS]
|
||||
[HAVING expr]
|
||||
[ORDER BY expr_list]
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[UNION ALL ...]
|
||||
[INTO OUTFILE filename]
|
||||
[FORMAT format]
|
||||
|
||||
@ -20,9 +20,9 @@ SELECT [DISTINCT] expr_list
|
||||
[WHERE expr]
|
||||
[GROUP BY expr_list] [WITH TOTALS]
|
||||
[HAVING expr]
|
||||
[ORDER BY expr_list]
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[UNION ALL ...]
|
||||
[INTO OUTFILE filename]
|
||||
[FORMAT format]
|
||||
|
||||
@ -20,9 +20,9 @@ SELECT [DISTINCT] expr_list
|
||||
[WHERE expr]
|
||||
[GROUP BY expr_list] [WITH TOTALS]
|
||||
[HAVING expr]
|
||||
[ORDER BY expr_list]
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[UNION ALL ...]
|
||||
[INTO OUTFILE filename]
|
||||
[FORMAT format]
|
||||
|
||||
@ -194,4 +194,4 @@ ClickHouse имеет сильную типизацию, поэтому нет
|
||||
!!! note "Note"
|
||||
Кластер ClickHouse состоит из независимых шардов, а каждый шард состоит из реплик. Кластер **не является эластичным** (not elastic), поэтому после добавления нового шарда данные не будут автоматически распределены между ними. Вместо этого нужно изменить настройки, чтобы выровнять нагрузку на кластер. Эта реализация дает вам больший контроль, и вполне приемлема для относительно небольших кластеров, таких как десятки узлов. Но для кластеров с сотнями узлов, которые мы используем в эксплуатации, такой подход становится существенным недостатком. Движки таблиц, которые охватывают весь кластер с динамически реплицируемыми областями, которые могут быть автоматически разделены и сбалансированы между кластерами, еще предстоит реализовать.
|
||||
|
||||
{## [Original article](https://clickhouse.tech/docs/ru/development/architecture/) ##}
|
||||
{## [Original article](https://clickhouse.tech/docs/ru/development/architecture/) ##}
|
||||
|
||||
@ -1261,4 +1261,91 @@ Cодержит информацию о дисках, заданных в [ко
|
||||
|
||||
Если политика хранения содержит несколько томов, то каждому тому соответствует отдельная запись в таблице.
|
||||
|
||||
## system.quotas {#system_tables-quotas}
|
||||
Содержит информацию о [квотах](quotas.md).
|
||||
|
||||
Столбцы:
|
||||
- `name` ([String](../sql-reference/data-types/string.md)) — Имя квоты.
|
||||
- `id` ([UUID](../sql-reference/data-types/uuid.md)) — ID квоты.
|
||||
- `storage`([String](../sql-reference/data-types/string.md)) — Хранилище квот. Возможные значения: "users.xml", если квота задана в файле users.xml, "disk" — если квота задана в SQL-запросе.
|
||||
- `keys` ([Array](../sql-reference/data-types/array.md)([Enum8](../sql-reference/data-types/enum.md))) — Ключ определяет совместное использование квоты. Если два соединения используют одну и ту же квоту, они совместно используют один и тот же объем ресурсов. Значения:
|
||||
- `[]` — Все пользователи используют одну и ту же квоту.
|
||||
- `['user_name']` — Соединения с одинаковым именем пользователя используют одну и ту же квоту.
|
||||
- `['ip_address']` — Соединения с одинаковым IP-адресом используют одну и ту же квоту.
|
||||
- `['client_key']` — Соединения с одинаковым ключом используют одну и ту же квоту. Ключ может быть явно задан клиентом. При использовании [clickhouse-client](../interfaces/cli.md), передайте ключевое значение в параметре `--quota-key`, или используйте параметр `quota_key` файле настроек клиента. В случае использования HTTP интерфейса, используйте заголовок `X-ClickHouse-Quota`.
|
||||
- `['user_name', 'client_key']` — Соединения с одинаковым ключом используют одну и ту же квоту. Если ключ не предоставлен клиентом, то квота отслеживается для `user_name`.
|
||||
- `['client_key', 'ip_address']` — Соединения с одинаковым ключом используют одну и ту же квоту. Если ключ не предоставлен клиентом, то квота отслеживается для `ip_address`.
|
||||
- `durations` ([Array](../sql-reference/data-types/array.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Длины временных интервалов для расчета потребления ресурсов, в секундах.
|
||||
- `apply_to_all` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Логическое значение. Он показывает, к каким пользователям применяется квота. Значения:
|
||||
- `0` — Квота применяется к пользователям, перечисленным в списке `apply_to_list`.
|
||||
- `1` — Квота применяется к пользователям, за исключением тех, что перечислены в списке `apply_to_except`.
|
||||
- `apply_to_list` ([Array](../sql-reference/data-types/array.md)([String](../sql-reference/data-types/string.md))) — Список имен пользователей/[ролей](../operations/access-rights.md#role-management) к которым применяется квота.
|
||||
- `apply_to_except` ([Array](../sql-reference/data-types/array.md)([String](../sql-reference/data-types/string.md))) — Список имен пользователей/ролей к которым квота применяться не должна.
|
||||
|
||||
## system.quota_limits {#system_tables-quota_limits}
|
||||
Содержит информацию о максимумах для всех интервалов всех квот. Одной квоте могут соответствовать любое количество строк или ноль.
|
||||
|
||||
Столбцы:
|
||||
- `quota_name` ([String](../sql-reference/data-types/string.md)) — Имя квоты.
|
||||
- `duration` ([UInt32](../sql-reference/data-types/int-uint.md)) — Длина временного интервала для расчета потребления ресурсов, в секундах.
|
||||
- `is_randomized_interval` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Логическое значение. Оно показывает, является ли интервал рандомизированным. Интервал всегда начинается в одно и то же время, если он не рандомизирован. Например, интервал в 1 минуту всегда начинается с целого числа минут (то есть он может начинаться в 11:20:00, но никогда не начинается в 11:20:01), интервал в один день всегда начинается в полночь UTC. Если интервал рандомизирован, то самый первый интервал начинается в произвольное время, а последующие интервалы начинаются один за другим. Значения:
|
||||
- `0` — Интервал рандомизирован.
|
||||
- `1` — Интервал не рандомизирован.
|
||||
- `max_queries` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное число запросов.
|
||||
- `max_errors` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество ошибок.
|
||||
- `max_result_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество строк результата.
|
||||
- `max_result_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальный объем оперативной памяти в байтах, используемый для хранения результата запроса.
|
||||
- `max_read_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество строк, считываемых из всех таблиц и табличных функций, участвующих в запросе.
|
||||
- `max_read_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество байтов, считываемых из всех таблиц и табличных функций, участвующих в запросе.
|
||||
- `max_execution_time` ([Nullable](../sql-reference/data-types/nullable.md)([Float64](../sql-reference/data-types/float.md))) — Максимальное время выполнения запроса, в секундах.
|
||||
|
||||
## system.quota_usage {#system_tables-quota_usage}
|
||||
Использование квоты текущим пользователем: сколько используется и сколько осталось.
|
||||
|
||||
Столбцы:
|
||||
- `quota_name` ([String](../sql-reference/data-types/string.md)) — Имя квоты.
|
||||
- `quota_key`([String](../sql-reference/data-types/string.md)) — Значение ключа. Например, если keys = `ip_address`, `quota_key` может иметь значение '192.168.1.1'.
|
||||
- `start_time`([Nullable](../sql-reference/data-types/nullable.md)([DateTime](../sql-reference/data-types/datetime.md))) — Время начала расчета потребления ресурсов.
|
||||
- `end_time`([Nullable](../sql-reference/data-types/nullable.md)([DateTime](../sql-reference/data-types/datetime.md))) — Время окончания расчета потребления ресурс
|
||||
- `duration` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Длина временного интервала для расчета потребления ресурсов, в секундах.
|
||||
- `queries` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Общее количество запросов на этом интервале.
|
||||
- `max_queries` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество запросов.
|
||||
- `errors` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Число запросов, вызвавших ошибки.
|
||||
- `max_errors` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное число ошибок.
|
||||
- `result_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Общее количество строк результата.
|
||||
- `max_result_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество строк результата.
|
||||
- `result_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Объем оперативной памяти в байтах, используемый для хранения результата запроса.
|
||||
- `max_result_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальный объем оперативной памяти, используемый для хранения результата запроса, в байтах.
|
||||
- `read_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Общее число исходных строк, считываемых из таблиц для выполнения запроса на всех удаленных серверах.
|
||||
- `max_read_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество строк, считываемых из всех таблиц и табличных функций, участвующих в запросах.
|
||||
- `read_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Общее количество байт, считанных из всех таблиц и табличных функций, участвующих в запросах.
|
||||
- `max_read_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество байт, считываемых из всех таблиц и табличных функций.
|
||||
- `execution_time` ([Nullable](../sql-reference/data-types/nullable.md)([Float64](../sql-reference/data-types/float.md))) — Общее время выполнения запроса, в секундах.
|
||||
- `max_execution_time` ([Nullable](../sql-reference/data-types/nullable.md)([Float64](../sql-reference/data-types/float.md))) — Максимальное время выполнения запроса.
|
||||
|
||||
## system.quotas_usage {#system_tables-quotas_usage}
|
||||
Использование квот всеми пользователями.
|
||||
|
||||
Столбцы:
|
||||
- `quota_name` ([String](../sql-reference/data-types/string.md)) — Имя квоты.
|
||||
- `quota_key` ([String](../sql-reference/data-types/string.md)) — Ключ квоты.
|
||||
- `is_current` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Квота используется для текущего пользователя.
|
||||
- `start_time` ([Nullable](../sql-reference/data-types/nullable.md)([DateTime](../sql-reference/data-types/datetime.md)))) — Время начала расчета потребления ресурсов.
|
||||
- `end_time` ([Nullable](../sql-reference/data-types/nullable.md)([DateTime](../sql-reference/data-types/datetime.md)))) — Время окончания расчета потребления ресурсов.
|
||||
- `duration` ([Nullable](../sql-reference/data-types/nullable.md)([UInt32](../sql-reference/data-types/int-uint.md))) — Длина временного интервала для расчета потребления ресурсов, в секундах.
|
||||
- `queries` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Общее количество запросов на этом интервале.
|
||||
- `max_queries` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное число запросов.
|
||||
- `errors` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Число запросов, вызвавших ошибки.
|
||||
- `max_errors` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное число ошибок.
|
||||
- `result_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — The total number of rows given as a result.
|
||||
- `max_result_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Maximum of source rows read from tables.
|
||||
- `result_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Объем оперативной памяти в байтах, используемый для хранения результата запроса.
|
||||
- `max_result_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальный объем оперативной памяти, используемый для хранения результата запроса, в байтах.
|
||||
- `read_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Общее число исходных строк, считываемых из таблиц для выполнения запроса на всех удаленных серверах.
|
||||
- `max_read_rows` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество строк, считываемых из всех таблиц и табличных функций, участвующих в запросах.
|
||||
- `read_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Общее количество байт, считанных из всех таблиц и табличных функций, участвующих в запросах.
|
||||
- `max_read_bytes` ([Nullable](../sql-reference/data-types/nullable.md)([UInt64](../sql-reference/data-types/int-uint.md))) — Максимальное количество байт, считываемых из всех таблиц и табличных функций.
|
||||
- `execution_time` ([Nullable](../sql-reference/data-types/nullable.md)([Float64](../sql-reference/data-types/float.md))) — Общее время выполнения запроса, в секундах.
|
||||
- `max_execution_time` ([Nullable](../sql-reference/data-types/nullable.md)([Float64](../sql-reference/data-types/float.md))) — Максимальное время выполнения запроса.
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/) <!--hide-->
|
||||
|
||||
@ -18,9 +18,9 @@ SELECT [DISTINCT] expr_list
|
||||
[WHERE expr]
|
||||
[GROUP BY expr_list] [WITH TOTALS]
|
||||
[HAVING expr]
|
||||
[ORDER BY expr_list]
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[UNION ALL ...]
|
||||
[INTO OUTFILE filename]
|
||||
[FORMAT format]
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
---
|
||||
toc_title: LIMIT
|
||||
---
|
||||
|
||||
# Секция LIMIT {#limit-clause}
|
||||
|
||||
`LIMIT m` позволяет выбрать из результата первые `m` строк.
|
||||
@ -7,3 +11,47 @@
|
||||
`n` и `m` должны быть неотрицательными целыми числами.
|
||||
|
||||
При отсутствии секции [ORDER BY](order-by.md), однозначно сортирующей результат, результат может быть произвольным и может являться недетерминированным.
|
||||
|
||||
## Модификатор LIMIT ... WITH TIES {#limit-with-ties}
|
||||
|
||||
Когда вы установите модификатор WITH TIES для `LIMIT n[,m]` и указываете `ORDER BY expr_list`, вы получите первые `n` или `n,m` строк и дополнительно все строки с теми же самым значениями полей указанных в `ORDER BY` равными строке на позиции `n` для `LIMIT n` или `m` для `LIMIT n,m`.
|
||||
|
||||
Этот модификатор также может быть скомбинирован с [ORDER BY ... WITH FILL модификатором](../../../sql-reference/statements/select/order-by.md#orderby-with-fill)
|
||||
|
||||
Для примера следующий запрос
|
||||
```sql
|
||||
SELECT * FROM (
|
||||
SELECT number%50 AS n FROM numbers(100)
|
||||
) ORDER BY n LIMIT 0,5
|
||||
```
|
||||
|
||||
возвращает
|
||||
```text
|
||||
┌─n─┐
|
||||
│ 0 │
|
||||
│ 0 │
|
||||
│ 1 │
|
||||
│ 1 │
|
||||
│ 2 │
|
||||
└───┘
|
||||
```
|
||||
|
||||
но после применения модификатора `WITH TIES`
|
||||
```sql
|
||||
SELECT * FROM (
|
||||
SELECT number%50 AS n FROM numbers(100)
|
||||
) ORDER BY n LIMIT 0,5 WITH TIES
|
||||
```
|
||||
|
||||
возвращает другой набор строк
|
||||
```text
|
||||
┌─n─┐
|
||||
│ 0 │
|
||||
│ 0 │
|
||||
│ 1 │
|
||||
│ 1 │
|
||||
│ 2 │
|
||||
│ 2 │
|
||||
└───┘
|
||||
```
|
||||
поскольку строка на позиции 6 имеет тоже самое значение "2" для поля `n` что и строка на позиции 5
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
---
|
||||
toc_title: ORDER BY
|
||||
---
|
||||
|
||||
# Секция ORDER BY {#select-order-by}
|
||||
|
||||
Секция `ORDER BY` содержит список выражений, к каждому из которых также может быть приписано `DESC` или `ASC` (направление сортировки). Если ничего не приписано - это аналогично приписыванию `ASC`. `ASC` - сортировка по возрастанию, `DESC` - сортировка по убыванию. Обозначение направления сортировки действует на одно выражение, а не на весь список. Пример: `ORDER BY Visits DESC, SearchPhrase`
|
||||
@ -31,7 +35,7 @@
|
||||
└───┴──────┘
|
||||
```
|
||||
|
||||
Выполнение запроса `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` получить:
|
||||
Выполните запрос `SELECT * FROM t_null_nan ORDER BY y NULLS FIRST` чтобы получить:
|
||||
|
||||
``` text
|
||||
┌─x─┬────y─┐
|
||||
@ -66,3 +70,128 @@
|
||||
|
||||
Внешняя сортировка работает существенно менее эффективно, чем сортировка в оперативке.
|
||||
|
||||
## Модификатор ORDER BY expr WITH FILL {#orderby-with-fill}
|
||||
|
||||
Этот модификатор также может быть скобинирован с модификатором [LIMIT ... WITH TIES](../../../sql-reference/statements/select/limit.md#limit-with-ties)
|
||||
|
||||
`WITH FILL` модификатор может быть установлен после `ORDER BY expr` с опциональными параметрами `FROM expr`, `TO expr` и `STEP expr`.
|
||||
Все пропущенные значнеия для колонки `expr` будут заполненые значениями соответсвующими предполагаемой последовательности значений колонки, другие колонки будут заполнены значенями по умолчанию.
|
||||
|
||||
Используйте следующую конструкцию для заполнения нескольких колонок с модификатором `WITH FILL` с необязательными параметрами после каждого имени поля в секции `ORDER BY`.
|
||||
|
||||
```sql
|
||||
ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr]
|
||||
```
|
||||
|
||||
`WITH FILL` может быть применене только к полям с числовыми (все разновидности float, int, decimal) или временными (все разновидности Date, DateTime) типами.
|
||||
Когда не определен `FROM const_expr`, последовательность заполнения использует минимальное значение поля `expr` из `ORDER BY`.
|
||||
Когда не определен `TO const_expr`, последовательность заполнения использует максимальное значение поля `expr` из `ORDER BY`.
|
||||
Когда `STEP const_numeric_expr` определен, тогда `const_numeric_expr` интерпретируется `как есть` для числовых типов, как `дни` для типа Date и как `секунды` для типа DateTime.
|
||||
Когда `STEP const_numeric_expr` не указан, тогда используется `1.0` для числовых типов, `1 день` для типа Date и `1 секунда` для типа DateTime.
|
||||
|
||||
|
||||
Для примера, следующий запрос
|
||||
```sql
|
||||
SELECT n, source FROM (
|
||||
SELECT toFloat32(number % 10) AS n, 'original' AS source
|
||||
FROM numbers(10) WHERE number % 3 = 1
|
||||
) ORDER BY n
|
||||
```
|
||||
|
||||
возвращает
|
||||
```text
|
||||
┌─n─┬─source───┐
|
||||
│ 1 │ original │
|
||||
│ 4 │ original │
|
||||
│ 7 │ original │
|
||||
└───┴──────────┘
|
||||
```
|
||||
|
||||
но после применения модификатора `WITH FILL`
|
||||
```sql
|
||||
SELECT n, source FROM (
|
||||
SELECT toFloat32(number % 10) AS n, 'original' AS source
|
||||
FROM numbers(10) WHERE number % 3 = 1
|
||||
) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5
|
||||
```
|
||||
|
||||
возвращает
|
||||
```text
|
||||
┌───n─┬─source───┐
|
||||
│ 0 │ │
|
||||
│ 0.5 │ │
|
||||
│ 1 │ original │
|
||||
│ 1.5 │ │
|
||||
│ 2 │ │
|
||||
│ 2.5 │ │
|
||||
│ 3 │ │
|
||||
│ 3.5 │ │
|
||||
│ 4 │ original │
|
||||
│ 4.5 │ │
|
||||
│ 5 │ │
|
||||
│ 5.5 │ │
|
||||
│ 7 │ original │
|
||||
└─────┴──────────┘
|
||||
```
|
||||
|
||||
Для случая когда у нас есть несколько полей `ORDER BY field2 WITH FILL, field1 WITH FILL` порядок заполнения будет следовать порядку полей в секции `ORDER BY`.
|
||||
|
||||
Пример:
|
||||
```sql
|
||||
SELECT
|
||||
toDate((number * 10) * 86400) AS d1,
|
||||
toDate(number * 86400) AS d2,
|
||||
'original' AS source
|
||||
FROM numbers(10)
|
||||
WHERE (number % 3) = 1
|
||||
ORDER BY
|
||||
d2 WITH FILL,
|
||||
d1 WITH FILL STEP 5;
|
||||
```
|
||||
|
||||
возвращает
|
||||
```text
|
||||
┌───d1───────┬───d2───────┬─source───┐
|
||||
│ 1970-01-11 │ 1970-01-02 │ original │
|
||||
│ 0000-00-00 │ 1970-01-03 │ │
|
||||
│ 0000-00-00 │ 1970-01-04 │ │
|
||||
│ 1970-02-10 │ 1970-01-05 │ original │
|
||||
│ 0000-00-00 │ 1970-01-06 │ │
|
||||
│ 0000-00-00 │ 1970-01-07 │ │
|
||||
│ 1970-03-12 │ 1970-01-08 │ original │
|
||||
└────────────┴────────────┴──────────┘
|
||||
```
|
||||
|
||||
Поле `d1` не заполняет и используется значение по умолчанию поскольку у нас нет повторяющихся значения для `d2` поэтому мы не можем правильно рассчитать последователность заполнения для`d1`.
|
||||
|
||||
Cледующий запрос (с измененым порядком в ORDER BY)
|
||||
```sql
|
||||
SELECT
|
||||
toDate((number * 10) * 86400) AS d1,
|
||||
toDate(number * 86400) AS d2,
|
||||
'original' AS source
|
||||
FROM numbers(10)
|
||||
WHERE (number % 3) = 1
|
||||
ORDER BY
|
||||
d1 WITH FILL STEP 5,
|
||||
d2 WITH FILL;
|
||||
```
|
||||
|
||||
возвращает
|
||||
```text
|
||||
┌───d1───────┬───d2───────┬─source───┐
|
||||
│ 1970-01-11 │ 1970-01-02 │ original │
|
||||
│ 1970-01-16 │ 0000-00-00 │ │
|
||||
│ 1970-01-21 │ 0000-00-00 │ │
|
||||
│ 1970-01-26 │ 0000-00-00 │ │
|
||||
│ 1970-01-31 │ 0000-00-00 │ │
|
||||
│ 1970-02-05 │ 0000-00-00 │ │
|
||||
│ 1970-02-10 │ 1970-01-05 │ original │
|
||||
│ 1970-02-15 │ 0000-00-00 │ │
|
||||
│ 1970-02-20 │ 0000-00-00 │ │
|
||||
│ 1970-02-25 │ 0000-00-00 │ │
|
||||
│ 1970-03-02 │ 0000-00-00 │ │
|
||||
│ 1970-03-07 │ 0000-00-00 │ │
|
||||
│ 1970-03-12 │ 1970-01-08 │ original │
|
||||
└────────────┴────────────┴──────────┘
|
||||
```
|
||||
|
||||
@ -33,4 +33,4 @@ clusterAllReplicas('cluster_name', db, table)
|
||||
**See Also**
|
||||
|
||||
- [skip\_unavailable\_shards](../../operations/settings/settings.md#settings-skip_unavailable_shards)
|
||||
- [load\_balancing](../../operations/settings/settings.md#settings-load_balancing)
|
||||
- [load\_balancing](../../operations/settings/settings.md#settings-load_balancing)
|
||||
|
||||
@ -62,11 +62,12 @@ Upd. Включено для системных таблиц.
|
||||
|
||||
Вместо этого предлагается разрешить кускам таблиц типа MergeTree располагать данные в разных форматах. А именно: - в оперативной памяти; - на диске со всеми столбцами в одном файле; - на диске со столбцами в отдельных файлах: в зависимости от размера куска и прошедшего времени. Для размещения кусков в оперативной памяти, придётся также реализовать опциональную поддержку write-ahead log с настраиваемыми правилами по сбросу на диск. Это позволит избавиться от проблем с мелкими вставками для MergeTree таблиц. Для ReplicatedMergeTree таблиц, это решит проблему лишь частично.
|
||||
|
||||
### 1.7. Буферизация и WAL в MergeTree {#buferizatsiia-i-wal-v-mergetree}
|
||||
### 1.7. + Буферизация и WAL в MergeTree {#buferizatsiia-i-wal-v-mergetree}
|
||||
|
||||
Требует 1.6. Антон Попов. Задача взята в работу. Q2.
|
||||
Есть pull request.
|
||||
Upd. В стадии код-ревью.
|
||||
Upd. Готово для использования начиная с версии 20.6 в экспериментальном режиме.
|
||||
|
||||
### 1.8. + Перенос между разделами по TTL {#perenos-mezhdu-razdelami-po-ttl}
|
||||
|
||||
@ -127,6 +128,8 @@ Q4.
|
||||
|
||||
Upd. Олег будет делать только часть про HDFS.
|
||||
Upd. Реализация поверх S3 является рабочей на уровне PoC.
|
||||
Upd. Реализация поверх S3 ужасно тормозит и сейчас доделывается.
|
||||
Upd. Реализацию поверх HDFS взял AmosBird для Kuaishou.
|
||||
|
||||
### 1.13. + Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final}
|
||||
|
||||
@ -379,6 +382,7 @@ Upd. Появилась вторая версия LTS - 20.3.
|
||||
### 6.8. Таблица system.crashes {#tablitsa-system-crashes}
|
||||
|
||||
Сравнительно простая задача, но только для опытных разработчиков.
|
||||
Upd. В разработке.
|
||||
|
||||
### 6.9. + Отправлять информацию клиенту, если сервер падает по сигналу {#otpravliat-informatsiiu-klientu-esli-server-padaet-po-signalu}
|
||||
|
||||
@ -525,7 +529,7 @@ https://github.com/ClickHouse/ClickHouse/issues/8027\#issuecomment-566670282
|
||||
Проверили на настоящем сервере Huawei, а также в специальном Docker контейнере, который содержит внутри qemu-user-static.
|
||||
Также можно проверить на Cavium, на Raspberry Pi а также на твоём Android телефоне.
|
||||
|
||||
### 7.20. + Автосборка для FreeBSD x86\_64 {#avtosborka-dlia-freebsd-x86-64}
|
||||
### 7.20. + Автосборка для FreeBSD x86_64 {#avtosborka-dlia-freebsd-x86-64}
|
||||
|
||||
[Иван Лежанкин](https://github.com/abyss7).
|
||||
|
||||
@ -604,6 +608,8 @@ Upd. Сделаны fuzzBits, fuzzBytes.
|
||||
|
||||
2. Генерация и выполнение случайных синтаксически корректных запросов на случайных данных.
|
||||
|
||||
Upd. Александр Кузьменков сделал фазер запросов на основе мутаций AST.
|
||||
|
||||
### 7.25. + Синхронизация релизов в Аркадию {#sinkhronizatsiia-relizov-v-arkadiiu}
|
||||
|
||||
Изначально занимался Олег Алексеенков. Сейчас он перешёл работать в дружественный отдел, но обещает продолжать синхронизацию.
|
||||
@ -626,10 +632,10 @@ Upd: Добавлены все файлы побайтово.
|
||||
Команда DevTools. Прогресс по задаче под вопросом.
|
||||
Upd. Готово (все директории кроме contrib).
|
||||
|
||||
### 7.27. Запуск автотестов в Аркадии {#zapusk-avtotestov-v-arkadii}
|
||||
### 7.27. + Запуск автотестов в Аркадии {#zapusk-avtotestov-v-arkadii}
|
||||
|
||||
Требует 7.26. Коллеги начали делать, есть результат.
|
||||
Upd. В Аркадии частично работает небольшая часть тестов.
|
||||
Upd. В Аркадии частично работает небольшая часть тестов. И этого достаточно.
|
||||
|
||||
### 7.29. Опции clickhouse install, stop, start вместо postinst, init.d, systemd скриптов {#optsii-clickhouse-install-stop-start-vmesto-postinst-init-d-systemd-skriptov}
|
||||
|
||||
@ -659,10 +665,11 @@ Upd. В Аркадии частично работает небольшая ча
|
||||
В очереди. Иван Лежанкин.
|
||||
Отсутствует прогресс.
|
||||
|
||||
### 7.34. Бэкпортировать bugfix автоматически {#bekportirovat-bugfix-avtomaticheski}
|
||||
### 7.34. + Бэкпортировать bugfix автоматически {#bekportirovat-bugfix-avtomaticheski}
|
||||
|
||||
В очереди. Иван Лежанкин.
|
||||
Присутствует прогресс.
|
||||
Сделано.
|
||||
|
||||
### 7.35. Начальные правила для авто-merge {#nachalnye-pravila-dlia-avto-merge}
|
||||
|
||||
@ -676,7 +683,7 @@ Upd. В Аркадии частично работает небольшая ча
|
||||
Сейчас добавляем некоторых доверенных контрибьюторов в ручном режиме.
|
||||
Upd. Всё ещё добавляем в ручном режиме.
|
||||
|
||||
### 7.37. Разобраться с repo.yandex.ru {#razobratsia-s-repo-yandex-ru}
|
||||
### 7.37. + Разобраться с repo.yandex.ru {#razobratsia-s-repo-yandex-ru}
|
||||
|
||||
Есть жалобы на скорость загрузки и неудобство maintenance, operations, visibility.
|
||||
|
||||
@ -733,11 +740,12 @@ Upd. Задача взята в работу.
|
||||
Артемий Бобровский, ВШЭ
|
||||
Есть pull request.
|
||||
|
||||
### 8.11. Движок таблиц для чтения из Mongo {#dvizhok-tablits-dlia-chteniia-iz-mongo}
|
||||
### 8.11. + Движок таблиц для чтения из Mongo {#dvizhok-tablits-dlia-chteniia-iz-mongo}
|
||||
|
||||
Артемий Бобровский, ВШЭ
|
||||
Есть pull request.
|
||||
Upd. В стадии код-ревью.
|
||||
Готово.
|
||||
|
||||
### 8.12. Пропуск столбцов в форматах Parquet, ORC {#propusk-stolbtsov-v-formatakh-parquet-orc}
|
||||
|
||||
@ -745,9 +753,9 @@ Upd. В стадии код-ревью.
|
||||
|
||||
### 8.13. Поддержка массивов в Parquet, ORC {#podderzhka-massivov-v-parquet-orc}
|
||||
|
||||
### 8.14. Запись данных в ORC {#zapis-dannykh-v-orc}
|
||||
### 8.14. + Запись данных в ORC {#zapis-dannykh-v-orc}
|
||||
|
||||
Павел Круглов, ВШЭ. Есть pull request.
|
||||
Павел Круглов, ВШЭ. Есть pull request. Готово.
|
||||
|
||||
### 8.15. Запись данных в CapNProto {#zapis-dannykh-v-capnproto}
|
||||
|
||||
@ -790,7 +798,7 @@ Upd. Готово.
|
||||
|
||||
Maxim Fedotov, Wargaming + Yuri Baranov, Яндекс.
|
||||
|
||||
### 8.19. Интеграция с RabbitMQ {#integratsiia-s-rabbitmq}
|
||||
### 8.19. + Интеграция с RabbitMQ {#integratsiia-s-rabbitmq}
|
||||
|
||||
Ксения Сумарокова, ВШЭ.
|
||||
|
||||
@ -800,10 +808,11 @@ Maxim Fedotov, Wargaming + Yuri Baranov, Яндекс.
|
||||
|
||||
Есть pull request в процессе разработки.
|
||||
Upd. В процессе code review.
|
||||
Upd. Готово.
|
||||
|
||||
### 8.20. Интеграция с SQS {#integratsiia-s-sqs}
|
||||
### 8.20. - Интеграция с SQS {#integratsiia-s-sqs}
|
||||
|
||||
Низкий приоритет.
|
||||
Низкий приоритет. Отменено.
|
||||
|
||||
### 8.21. Поддержка произвольного количества языков для имён регионов {#podderzhka-proizvolnogo-kolichestva-iazykov-dlia-imion-regionov}
|
||||
|
||||
@ -832,8 +841,6 @@ Upd. Сделано теми людьми, кому не запрещено ра
|
||||
|
||||
### 9.2. Преднастроенные именованные соединения к внешним БД {#prednastroennye-imenovannye-soedineniia-k-vneshnim-bd}
|
||||
|
||||
Валерий Батурин, ВШЭ.
|
||||
|
||||
ClickHouse предоставляет возможность обратиться к внешней базе данных из языка запросов. Это реализовано в виде табличных функций. В параметрах к табличной функции указывается адрес удалённой базы данных (хост, порт), а также аутентификационные данные (имя пользователя, пароль). Аутентификационные данные указываются в запросе в открытом виде и, таким образом, попадают в историю запросов и в логи, что компрометирует безопасность системы.
|
||||
|
||||
Вместо этого предлагается описывать необходимые данные в конфигурационном файле сервера или в отдельном сервисе и ссылаться на них по именам.
|
||||
@ -883,12 +890,8 @@ Upd. Ура, нашли причину и исправили.
|
||||
|
||||
### 10.7. Поддержка Nullable в словарях {#podderzhka-nullable-v-slovariakh}
|
||||
|
||||
Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ.
|
||||
|
||||
### 10.8. Поддержка массивов в словарях {#podderzhka-massivov-v-slovariakh}
|
||||
|
||||
Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ.
|
||||
|
||||
### 10.9. - Уменьшение блокировок для cache словарей за счёт одновременных запросов одного и того же {#umenshenie-blokirovok-dlia-cache-slovarei-za-schiot-odnovremennykh-zaprosov-odnogo-i-togo-zhe}
|
||||
|
||||
Заменено в пользу 10.10, 10.11.
|
||||
@ -1022,6 +1025,8 @@ Q3.
|
||||
|
||||
### 13.3. Пулы ресурсов {#puly-resursov}
|
||||
|
||||
Александр Казаков.
|
||||
|
||||
Требует 13.2 или сможем сделать более неудобную реализацию раньше.
|
||||
Обсуждается вариант неудобной реализации. Пока средний приоритет, целимся на Q1/Q2.
|
||||
Вариант реализации выбрал Александр Казаков.
|
||||
@ -1065,10 +1070,10 @@ zhang2014
|
||||
Upd. Есть pull request-ы.
|
||||
Upd. DISTINCT готов.
|
||||
|
||||
### 14.9. Поддержка запроса EXPLAIN {#podderzhka-zaprosa-explain}
|
||||
### 14.9. + Поддержка запроса EXPLAIN {#podderzhka-zaprosa-explain}
|
||||
|
||||
Требует 2.1. [Николай Кочетов](https://github.com/KochetovNicolai).
|
||||
Upd. Есть pull request.
|
||||
Upd. Есть pull request. Готово.
|
||||
|
||||
### 14.10. arrayReduce как функция высшего порядка {#arrayreduce-kak-funktsiia-vysshego-poriadka}
|
||||
|
||||
@ -1229,7 +1234,7 @@ Upd. Есть pull request.
|
||||
|
||||
Предлагается реализовать в ClickHouse статистические тесты (Analysis of Variance, тесты нормальности распределения и т. п.) в виде агрегатных функций. Пример: `welchTTest(value, sample_idx)`.
|
||||
|
||||
Сделали прототип двух тестов, есть pull request.
|
||||
Сделали прототип двух тестов, есть pull request. Также есть pull request для корелляции рангов.
|
||||
|
||||
### 18.3. Инфраструктура для тренировки моделей в ClickHouse {#infrastruktura-dlia-trenirovki-modelei-v-clickhouse}
|
||||
|
||||
@ -1346,6 +1351,8 @@ Upd. Для DISTINCT есть pull request.
|
||||
|
||||
### 21.6. Уменьшение числа потоков для SELECT в случае тривиального INSERT SELECT {#umenshenie-chisla-potokov-dlia-select-v-sluchae-trivialnogo-insert-select}
|
||||
|
||||
ucasFL, в разработке.
|
||||
|
||||
### 21.7. Кэш результатов запросов {#kesh-rezultatov-zaprosov}
|
||||
|
||||
[Achimbab](https://github.com/achimbab).
|
||||
@ -1363,6 +1370,7 @@ Upd. В обсуждении.
|
||||
Для domain-specific кэшей (как например, кэш разжатых данных) выгодно, чтобы они использовали как можно больший объём свободной памяти. Но в этом случае, памяти может не хватить для других структур данных в программе. Если аллокатор памяти знает про кэш, то выделение памяти можно было бы делать путём вытеснения данных из кэша.
|
||||
|
||||
Upd. Есть нерабочий прототип, скорее всего будет отложено.
|
||||
Upd. Отложено до осени.
|
||||
|
||||
### 21.8.1. Отдельный аллокатор для кэшей с ASLR {#otdelnyi-allokator-dlia-keshei-s-aslr}
|
||||
|
||||
@ -1390,6 +1398,7 @@ Amos Bird.
|
||||
Сделана замена цепочек if на multiIf, но внезапно оказалось, что это является не оптимизацией, а наоборот.
|
||||
Сделано ещё несколько оптимизаций.
|
||||
Upd. Все вышеперечисленные оптимизации доступны в pull requests.
|
||||
Upd. Из них почти все помержены, осталась одна.
|
||||
|
||||
### 21.12. Алгебраические оптимизации запросов {#algebraicheskie-optimizatsii-zaprosov}
|
||||
|
||||
@ -1404,6 +1413,8 @@ Upd. Все вышеперечисленные оптимизации досту
|
||||
- Удаление дублирующихся DISTINCT, ORDER BY из подзапросов.
|
||||
|
||||
Несколько оптимизаций есть в PR.
|
||||
Upd. Все оптимизации кроме "Обращение инъективных функций в сравнениях на равенство" есть в PR.
|
||||
Upd. Из них больше половины помержены, осталось ещё две.
|
||||
|
||||
### 21.13. Fusion агрегатных функций {#fusion-agregatnykh-funktsii}
|
||||
|
||||
@ -1415,6 +1426,8 @@ Constraints позволяют задать выражение, истиннос
|
||||
|
||||
Если выражение содержит равенство, то встретив в запросе одну из частей равенства, её можно заменить на другую часть равенства, если это сделает проще чтение данных или вычисление выражения. Например, задан constraint: `URLDomain = domain(URL)`. Значит, выражение `domain(URL)` можно заменить на `URLDomain`.
|
||||
|
||||
Upd. Возможно будет отложено на следующий год.
|
||||
|
||||
### 21.15. Многоступенчатое чтение данных вместо PREWHERE {#mnogostupenchatoe-chtenie-dannykh-vmesto-prewhere}
|
||||
|
||||
Требует 2.1 и 21.10.
|
||||
@ -1459,6 +1472,7 @@ Constraints позволяют задать выражение, истиннос
|
||||
### 21.20. Использование материализованных представлений для оптимизации запросов {#ispolzovanie-materializovannykh-predstavlenii-dlia-optimizatsii-zaprosov}
|
||||
|
||||
В ByteDance есть готовая реализация, но они её боятся из-за, возможно, низкого качества кода.
|
||||
Upd. Вместо этого будем делать задачу 1.16.
|
||||
|
||||
### 21.21. + Чтение больших файлов с помощью mmap {#chtenie-bolshikh-failov-s-pomoshchiu-mmap}
|
||||
|
||||
@ -1540,11 +1554,12 @@ Upd. Добавили таймауты.
|
||||
### 22.15. Нормализация коммитов в Kafka и идемпотентности операций {#normalizatsiia-kommitov-v-kafka-i-idempotentnosti-operatsii}
|
||||
|
||||
Altinity.
|
||||
Я не в курсе, какой статус.
|
||||
|
||||
### 22.16. + Исправление низкой производительности кодека DoubleDelta {#ispravlenie-nizkoi-proizvoditelnosti-kodeka-doubledelta}
|
||||
|
||||
Василий Немков, Altinity - в процессе.
|
||||
Можно считать, что сделано, хотя отсутствие SIMD оптимизаций для variable length кодеков - это ужасно.
|
||||
Upd. Готово.
|
||||
|
||||
### 22.17. Консистентно работающий POPULATE для MaterializedView {#konsistentno-rabotaiushchii-populate-dlia-materializedview}
|
||||
|
||||
@ -1602,16 +1617,16 @@ Altinity.
|
||||
|
||||
## 23. Default Festival {#default-festival}
|
||||
|
||||
### 23.1. + Включение minimalistic\_part\_header в ZooKeeper {#vkliuchenie-minimalistic-part-header-v-zookeeper}
|
||||
### 23.1. + Включение minimalistic_part_header в ZooKeeper {#vkliuchenie-minimalistic-part-header-v-zookeeper}
|
||||
|
||||
Сильно уменьшает объём данных в ZooKeeper. Уже год в продакшене в Яндекс.Метрике.
|
||||
Алексей Миловидов, ноябрь 2019.
|
||||
|
||||
### 23.2. Включение distributed\_aggregation\_memory\_efficient {#vkliuchenie-distributed-aggregation-memory-efficient}
|
||||
### 23.2. Включение distributed_aggregation_memory_efficient {#vkliuchenie-distributed-aggregation-memory-efficient}
|
||||
|
||||
Есть риски меньшей производительности лёгких запросов, хотя производительность тяжёлых запросов всегда увеличивается.
|
||||
|
||||
### 23.3. Включение min\_bytes\_to\_external\_sort и min\_bytes\_to\_external\_group\_by {#vkliuchenie-min-bytes-to-external-sort-i-min-bytes-to-external-group-by}
|
||||
### 23.3. Включение min_bytes_to_external_sort и min_bytes_to_external_group_by {#vkliuchenie-min-bytes-to-external-sort-i-min-bytes-to-external-group-by}
|
||||
|
||||
Желательно 5.2. и 13.1.
|
||||
|
||||
@ -1619,11 +1634,11 @@ Altinity.
|
||||
|
||||
Есть гипотеза, что плохо работает на очень больших кластерах.
|
||||
|
||||
### 23.5. Включение compile\_expressions {#vkliuchenie-compile-expressions}
|
||||
### 23.5. Включение compile_expressions {#vkliuchenie-compile-expressions}
|
||||
|
||||
Требует 7.2. Задачу изначально на 99% сделал Денис Скоробогатов, ВШЭ и Яндекс. Остальной процент доделывал Алексей Миловидов, а затем [Александр Сапин](https://github.com/alesapin).
|
||||
|
||||
### 23.6. Включение учёта порядка столбцов в CSV {#vkliuchenie-uchiota-poriadka-stolbtsov-v-csv}
|
||||
### 23.6. + Включение учёта порядка столбцов в CSV {#vkliuchenie-uchiota-poriadka-stolbtsov-v-csv}
|
||||
|
||||
Просто аккуратно включить.
|
||||
|
||||
@ -1668,13 +1683,11 @@ ClickHouse поддерживает LZ4 и ZSTD для сжатия данных
|
||||
|
||||
### 24.3. Экспериментальные кодеки {#eksperimentalnye-kodeki}
|
||||
|
||||
Вероника Фалчикова, Лада Торчик, ВШЭ.
|
||||
|
||||
Существуют специализированные алгоритмы кодирования числовых последовательностей: Group VarInt, MaskedVByte, PFOR. Необходимо изучить наиболее эффективные реализации этих алгоритмов. Примеры вы сможете найти на https://github.com/lemire и https://github.com/powturbo/ а также https://github.com/schizofreny/middle-out
|
||||
|
||||
Внедрить их в ClickHouse в виде кодеков и изучить их работу на тестовых датасетах.
|
||||
|
||||
Upd. Есть два pull requests в начальной стадии, возможно будет отложено.
|
||||
Upd. Есть два pull requests в начальной стадии, отложено.
|
||||
|
||||
### 24.4. Шифрование в ClickHouse на уровне VFS {#shifrovanie-v-clickhouse-na-urovne-vfs}
|
||||
|
||||
@ -1692,7 +1705,7 @@ Upd. Есть два pull requests в начальной стадии, возм
|
||||
2. Шифрование отдельных значений.
|
||||
Для этого требуется реализовать функции шифрования и расшифрования, доступные из SQL. Для шифрования реализовать возможность добавления нужного количества случайных бит для исключения одинаковых зашифрованных значений на одинаковых данных. Это позволит реализовать возможность «забывания» данных без удаления строк таблицы: можно шифровать данные разных клиентов разными ключами, и для того, чтобы забыть данные одного клиента, потребуется всего лишь удалить ключ.
|
||||
|
||||
Будет делать Василий Немков, Altinity
|
||||
Делает Василий Немков, Altinity
|
||||
|
||||
### 24.6. Userspace RAID {#userspace-raid}
|
||||
|
||||
@ -1796,6 +1809,8 @@ Upd. Прототип bitonic sort помержен, но целесообраз
|
||||
|
||||
Требует 2.1.
|
||||
|
||||
Upd. Есть два прототипа от внешних контрибьюторов.
|
||||
|
||||
### 24.15. Поддержка полуструктурированных данных {#podderzhka-polustrukturirovannykh-dannykh}
|
||||
|
||||
Требует 1.14 и 2.10.
|
||||
@ -1889,7 +1904,7 @@ ucasFL, ICT.
|
||||
Жанна Зосимова, ВШЭ.
|
||||
Upd. Пока поддержали Arrow как формат ввода-вывода.
|
||||
|
||||
### - 24.30. - ClickHouse как графовая СУБД {#clickhouse-kak-grafovaia-subd}
|
||||
### 24.30. - ClickHouse как графовая СУБД {#clickhouse-kak-grafovaia-subd}
|
||||
|
||||
Amos Bird, но его решение слишком громоздкое и пока не open-source. Отменено.
|
||||
|
||||
@ -1971,10 +1986,14 @@ Amos Bird, но его решение слишком громоздкое и п
|
||||
|
||||
Алексей Миловидов и все подготовленные докладчики.
|
||||
Upd. Есть Saint HighLoad online.
|
||||
Upd. Есть C++ Russia.
|
||||
CodeFest, DUMP, UWDC отменились.
|
||||
|
||||
### 25.15. Конференции зарубежные: Percona, DataOps, попытка попасть на более крупные {#konferentsii-zarubezhnye-percona-dataops-popytka-popast-na-bolee-krupnye}
|
||||
|
||||
Алексей Миловидов и все подготовленные докладчики
|
||||
Upd. Есть Percona.
|
||||
DataOps отменилась.
|
||||
|
||||
### 25.16. Сайт play.clickhouse {#sait-play-clickhouse}
|
||||
|
||||
@ -1995,6 +2014,7 @@ Upd. Есть Saint HighLoad online.
|
||||
|
||||
Алексей Миловидов и вся группа разработки.
|
||||
Благодаря Robert Hodges добавлен CMU.
|
||||
Upd. Взаимодействие с ВШЭ 2019/2020 успешно выполнено.
|
||||
|
||||
### 25.18. - Лекция в ШАД {#lektsiia-v-shad}
|
||||
|
||||
@ -2012,6 +2032,8 @@ Upd. Есть поползновения с TDEngine.
|
||||
|
||||
### 25.21. Повторное награждение контрибьюторов в Китае {#povtornoe-nagrazhdenie-kontribiutorov-v-kitae}
|
||||
|
||||
Upd. Ждём снятия ограничений и восстановления активности по онлайн митапам.
|
||||
|
||||
### 25.22. On-site помощь с ClickHouse компаниям в дни рядом с мероприятиями {#on-site-pomoshch-s-clickhouse-kompaniiam-v-dni-riadom-s-meropriiatiiami}
|
||||
|
||||
[Иван Блинков](https://github.com/blinkov/) - организация. Провёл мероприятие для турецкой компании.
|
||||
@ -2019,6 +2041,8 @@ Upd. On-site заменяется на Online.
|
||||
|
||||
### 25.23. Новый мерч для ClickHouse {#novyi-merch-dlia-clickhouse}
|
||||
|
||||
Upd. Старого пока хватает, раздача уменьшилась.
|
||||
|
||||
### 25.24. Конкурсы bughunter или оптимизации кода на C++ {#konkursy-bughunter-ili-optimizatsii-koda-na-c}
|
||||
|
||||
Проведение конкурсов должно начинаться для сотрудников Яндекса, пока нет согласования.
|
||||
|
||||
@ -24,9 +24,9 @@ SELECT [DISTINCT] expr_list
|
||||
[WHERE expr]
|
||||
[GROUP BY expr_list] [WITH TOTALS]
|
||||
[HAVING expr]
|
||||
[ORDER BY expr_list]
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
[UNION ALL ...]
|
||||
[INTO OUTFILE filename]
|
||||
[FORMAT format]
|
||||
|
||||
@ -1054,6 +1054,10 @@ private:
|
||||
auto base_before_fuzz = fuzz_base->formatForErrorMessage();
|
||||
|
||||
ast_to_process = fuzz_base->clone();
|
||||
|
||||
std::stringstream dump_of_cloned_ast;
|
||||
ast_to_process->dumpTree(dump_of_cloned_ast);
|
||||
|
||||
fuzzer.fuzzMain(ast_to_process);
|
||||
|
||||
auto base_after_fuzz = fuzz_base->formatForErrorMessage();
|
||||
@ -1066,6 +1070,8 @@ private:
|
||||
base_after_fuzz.c_str());
|
||||
fprintf(stderr, "dump before fuzz:\n%s\n",
|
||||
dump_before_fuzz.str().c_str());
|
||||
fprintf(stderr, "dump of cloned ast:\n%s\n",
|
||||
dump_of_cloned_ast.str().c_str());
|
||||
fprintf(stderr, "dump after fuzz:\n");
|
||||
fuzz_base->dumpTree(std::cerr);
|
||||
assert(false);
|
||||
|
||||
@ -377,7 +377,7 @@ struct Settings : public SettingsCollection<Settings>
|
||||
\
|
||||
M(SettingBool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \
|
||||
M(SettingBool, use_compact_format_in_distributed_parts_names, false, "Changes format of directories names for distributed table insert parts.", 0) \
|
||||
M(SettingUInt64, multiple_joins_rewriter_version, 1, "1 or 2. Second rewriter version knows about table columns and keep not clashed names as is.", 0) \
|
||||
M(SettingUInt64, multiple_joins_rewriter_version, 2, "1 or 2. Second rewriter version knows about table columns and keep not clashed names as is.", 0) \
|
||||
M(SettingBool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \
|
||||
M(SettingUInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \
|
||||
M(SettingSeconds, temporary_live_view_timeout, DEFAULT_TEMPORARY_LIVE_VIEW_TIMEOUT_SEC, "Timeout after which temporary live view is deleted.", 0) \
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadBufferFromS3.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/SeekAvoidingReadBuffer.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <IO/WriteBufferFromS3.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
@ -423,7 +424,8 @@ DiskS3::DiskS3(
|
||||
String s3_root_path_,
|
||||
String metadata_path_,
|
||||
size_t min_upload_part_size_,
|
||||
size_t min_multi_part_upload_size_)
|
||||
size_t min_multi_part_upload_size_,
|
||||
size_t min_bytes_for_seek_)
|
||||
: name(std::move(name_))
|
||||
, client(std::move(client_))
|
||||
, proxy_configuration(std::move(proxy_configuration_))
|
||||
@ -432,6 +434,7 @@ DiskS3::DiskS3(
|
||||
, metadata_path(std::move(metadata_path_))
|
||||
, min_upload_part_size(min_upload_part_size_)
|
||||
, min_multi_part_upload_size(min_multi_part_upload_size_)
|
||||
, min_bytes_for_seek(min_bytes_for_seek_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -537,7 +540,8 @@ std::unique_ptr<ReadBufferFromFileBase> DiskS3::readFile(const String & path, si
|
||||
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Read from file by path: {}. Existing S3 objects: {}",
|
||||
backQuote(metadata_path + path), metadata.s3_objects.size());
|
||||
|
||||
return std::make_unique<ReadIndirectBufferFromS3>(client, bucket, metadata, buf_size);
|
||||
auto reader = std::make_unique<ReadIndirectBufferFromS3>(client, bucket, metadata, buf_size);
|
||||
return std::make_unique<SeekAvoidingReadBuffer>(std::move(reader), min_bytes_for_seek);
|
||||
}
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase> DiskS3::writeFile(const String & path, size_t buf_size, WriteMode mode, size_t estimated_size, size_t)
|
||||
|
||||
@ -27,7 +27,8 @@ public:
|
||||
String s3_root_path_,
|
||||
String metadata_path_,
|
||||
size_t min_upload_part_size_,
|
||||
size_t min_multi_part_upload_size_);
|
||||
size_t min_multi_part_upload_size_,
|
||||
size_t min_bytes_for_seek_);
|
||||
|
||||
const String & getName() const override { return name; }
|
||||
|
||||
@ -111,6 +112,7 @@ private:
|
||||
const String metadata_path;
|
||||
size_t min_upload_part_size;
|
||||
size_t min_multi_part_upload_size;
|
||||
size_t min_bytes_for_seek;
|
||||
|
||||
UInt64 reserved_bytes = 0;
|
||||
UInt64 reservation_count = 0;
|
||||
|
||||
@ -140,7 +140,8 @@ void registerDiskS3(DiskFactory & factory)
|
||||
uri.key,
|
||||
metadata_path,
|
||||
context.getSettingsRef().s3_min_upload_part_size,
|
||||
config.getUInt64(config_prefix + ".min_multi_part_upload_size", 10*1024*1024));
|
||||
config.getUInt64(config_prefix + ".min_multi_part_upload_size", 10 * 1024 * 1024),
|
||||
config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024));
|
||||
|
||||
/// This code is used only to check access to the corresponding disk.
|
||||
checkWriteAccess(*s3disk);
|
||||
|
||||
@ -3,6 +3,7 @@ LIBRARY()
|
||||
PEERDIR(
|
||||
clickhouse/src/Common
|
||||
contrib/libs/protobuf
|
||||
contrib/libs/protoc
|
||||
)
|
||||
|
||||
SRCS(
|
||||
|
||||
@ -1096,8 +1096,10 @@ public:
|
||||
return;
|
||||
}
|
||||
|
||||
auto * left_generic = block.getByPosition(arguments[0]).type.get();
|
||||
auto * right_generic = block.getByPosition(arguments[1]).type.get();
|
||||
const auto & left_argument = block.getByPosition(arguments[0]);
|
||||
const auto & right_argument = block.getByPosition(arguments[1]);
|
||||
auto * left_generic = left_argument.type.get();
|
||||
auto * right_generic = right_argument.type.get();
|
||||
bool valid = castBothTypes(left_generic, right_generic, [&](const auto & left, const auto & right)
|
||||
{
|
||||
using LeftDataType = std::decay_t<decltype(left)>;
|
||||
@ -1112,8 +1114,17 @@ public:
|
||||
else
|
||||
return executeNumeric(block, arguments, result, left, right);
|
||||
});
|
||||
|
||||
if (!valid)
|
||||
throw Exception(getName() + "'s arguments do not match the expected data types", ErrorCodes::LOGICAL_ERROR);
|
||||
{
|
||||
// This is a logical error, because the types should have been checked
|
||||
// by getReturnTypeImpl().
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Arguments of '{}' have incorrect data types: '{}' of type '{}',"
|
||||
" '{}' of type '{}'", getName(),
|
||||
left_argument.name, left_argument.type->getName(),
|
||||
right_argument.name, right_argument.type->getName());
|
||||
}
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
66
src/IO/SeekAvoidingReadBuffer.cpp
Normal file
66
src/IO/SeekAvoidingReadBuffer.cpp
Normal file
@ -0,0 +1,66 @@
|
||||
#include <IO/SeekAvoidingReadBuffer.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
SeekAvoidingReadBuffer::SeekAvoidingReadBuffer(std::unique_ptr<ReadBufferFromFileBase> nested_, UInt64 min_bytes_for_seek_)
|
||||
: nested(std::move(nested_))
|
||||
, min_bytes_for_seek(min_bytes_for_seek_)
|
||||
{
|
||||
swap(*nested);
|
||||
}
|
||||
|
||||
|
||||
std::string SeekAvoidingReadBuffer::getFileName() const
|
||||
{
|
||||
return nested->getFileName();
|
||||
}
|
||||
|
||||
|
||||
off_t SeekAvoidingReadBuffer::getPosition()
|
||||
{
|
||||
swap(*nested);
|
||||
off_t position = nested->getPosition();
|
||||
swap(*nested);
|
||||
return position;
|
||||
}
|
||||
|
||||
|
||||
off_t SeekAvoidingReadBuffer::seek(off_t off, int whence)
|
||||
{
|
||||
off_t position = getPosition();
|
||||
|
||||
if (whence == SEEK_CUR)
|
||||
{
|
||||
off += position;
|
||||
whence = SEEK_SET;
|
||||
}
|
||||
|
||||
if (whence == SEEK_SET && off >= position && off < position + static_cast<off_t>(min_bytes_for_seek))
|
||||
{
|
||||
swap(*nested);
|
||||
nested->ignore(off - position);
|
||||
swap(*nested);
|
||||
position = off;
|
||||
}
|
||||
else
|
||||
{
|
||||
swap(*nested);
|
||||
position = nested->seek(off, whence);
|
||||
swap(*nested);
|
||||
}
|
||||
|
||||
return position;
|
||||
}
|
||||
|
||||
|
||||
bool SeekAvoidingReadBuffer::nextImpl()
|
||||
{
|
||||
swap(*nested);
|
||||
bool nested_result = nested->next();
|
||||
swap(*nested);
|
||||
return nested_result;
|
||||
}
|
||||
|
||||
}
|
||||
31
src/IO/SeekAvoidingReadBuffer.h
Normal file
31
src/IO/SeekAvoidingReadBuffer.h
Normal file
@ -0,0 +1,31 @@
|
||||
#pragma once
|
||||
|
||||
#include <IO/ReadBufferFromFileBase.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// `SeekAvoidingReadBuffer` prefers sequential reads over seeks within specified window.
|
||||
/// It is useful in network and spinning disk storage media when seek is relatively expensive
|
||||
/// operation.
|
||||
/// See also: `merge_tree_min_rows_for_seek`.
|
||||
class SeekAvoidingReadBuffer : public ReadBufferFromFileBase
|
||||
{
|
||||
std::unique_ptr<ReadBufferFromFileBase> nested;
|
||||
|
||||
UInt64 min_bytes_for_seek; /// Minimum positive seek offset which shall be executed using seek operation.
|
||||
|
||||
public:
|
||||
SeekAvoidingReadBuffer(std::unique_ptr<ReadBufferFromFileBase> nested_, UInt64 min_bytes_for_seek_);
|
||||
|
||||
std::string getFileName() const override;
|
||||
|
||||
off_t getPosition() override;
|
||||
|
||||
off_t seek(off_t off, int whence) override;
|
||||
|
||||
bool nextImpl() override;
|
||||
};
|
||||
|
||||
}
|
||||
@ -39,6 +39,7 @@ SRCS(
|
||||
readFloatText.cpp
|
||||
ReadHelpers.cpp
|
||||
ReadWriteBufferFromHTTP.cpp
|
||||
SeekAvoidingReadBuffer.cpp
|
||||
UseSSL.cpp
|
||||
WriteBufferAIO.cpp
|
||||
WriteBufferFromFile.cpp
|
||||
|
||||
@ -23,7 +23,6 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
extern const int INCORRECT_QUERY;
|
||||
}
|
||||
|
||||
@ -62,10 +61,6 @@ BlockIO InterpreterAlterQuery::execute()
|
||||
alter_commands.emplace_back(std::move(*alter_command));
|
||||
else if (auto partition_command = PartitionCommand::parse(command_ast))
|
||||
{
|
||||
if (partition_command->type == PartitionCommand::DROP_DETACHED_PARTITION
|
||||
&& !context.getSettingsRef().allow_drop_detached)
|
||||
throw DB::Exception("Cannot execute query: DROP DETACHED PART is disabled "
|
||||
"(see allow_drop_detached setting)", ErrorCodes::SUPPORT_IS_DISABLED);
|
||||
partition_commands.emplace_back(std::move(*partition_command));
|
||||
}
|
||||
else if (auto mut_command = MutationCommand::parse(command_ast))
|
||||
@ -90,6 +85,7 @@ BlockIO InterpreterAlterQuery::execute()
|
||||
|
||||
if (!partition_commands.empty())
|
||||
{
|
||||
table->checkAlterPartitionIsPossible(partition_commands, metadata_snapshot, context.getSettingsRef());
|
||||
table->alterPartition(query_ptr, metadata_snapshot, partition_commands, context);
|
||||
}
|
||||
|
||||
|
||||
@ -689,6 +689,9 @@ static UInt64 getLimitForSorting(const ASTSelectQuery & query, const Context & c
|
||||
if (!query.distinct && !query.limitBy() && !query.limit_with_ties && !query.arrayJoinExpressionList() && query.limitLength())
|
||||
{
|
||||
auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context);
|
||||
if (limit_length > std::numeric_limits<UInt64>::max() - limit_offset)
|
||||
return 0;
|
||||
|
||||
return limit_length + limit_offset;
|
||||
}
|
||||
return 0;
|
||||
@ -1287,6 +1290,7 @@ void InterpreterSelectQuery::executeFetchColumns(
|
||||
&& !query.limitBy()
|
||||
&& query.limitLength()
|
||||
&& !query_analyzer->hasAggregation()
|
||||
&& limit_length <= std::numeric_limits<UInt64>::max() - limit_offset
|
||||
&& limit_length + limit_offset < max_block_size)
|
||||
{
|
||||
max_block_size = std::max(UInt64(1), limit_length + limit_offset);
|
||||
@ -1649,8 +1653,9 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before
|
||||
auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, *context);
|
||||
UInt64 limit_for_distinct = 0;
|
||||
|
||||
/// If after this stage of DISTINCT ORDER BY is not executed, then you can get no more than limit_length + limit_offset of different rows.
|
||||
if (!query.orderBy() || !before_order)
|
||||
/// If after this stage of DISTINCT ORDER BY is not executed,
|
||||
/// then you can get no more than limit_length + limit_offset of different rows.
|
||||
if ((!query.orderBy() || !before_order) && limit_length <= std::numeric_limits<UInt64>::max() - limit_offset)
|
||||
limit_for_distinct = limit_length + limit_offset;
|
||||
|
||||
SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode);
|
||||
@ -1678,6 +1683,9 @@ void InterpreterSelectQuery::executePreLimit(QueryPlan & query_plan, bool do_not
|
||||
|
||||
if (do_not_skip_offset)
|
||||
{
|
||||
if (limit_length > std::numeric_limits<UInt64>::max() - limit_offset)
|
||||
return;
|
||||
|
||||
limit_length += limit_offset;
|
||||
limit_offset = 0;
|
||||
}
|
||||
|
||||
@ -302,7 +302,7 @@ bool TableJoin::needStreamWithNonJoinedRows() const
|
||||
return isRightOrFull(kind());
|
||||
}
|
||||
|
||||
bool TableJoin::allowDictJoin(const String & dict_key, const Block & sample_block, Names & names, NamesAndTypesList & result_columns) const
|
||||
bool TableJoin::allowDictJoin(const String & dict_key, const Block & sample_block, Names & src_names, NamesAndTypesList & dst_columns) const
|
||||
{
|
||||
/// Support ALL INNER, [ANY | ALL | SEMI | ANTI] LEFT
|
||||
if (!isLeft(kind()) && !(isInner(kind()) && strictness() == ASTTableJoin::Strictness::All))
|
||||
@ -312,18 +312,26 @@ bool TableJoin::allowDictJoin(const String & dict_key, const Block & sample_bloc
|
||||
if (right_keys.size() != 1)
|
||||
return false;
|
||||
|
||||
/// TODO: support 'JOIN ... ON expr(dict_key) = table_key'
|
||||
auto it_key = original_names.find(right_keys[0]);
|
||||
if (it_key == original_names.end())
|
||||
return false;
|
||||
|
||||
if (dict_key != it_key->second)
|
||||
return false; /// JOIN key != Dictionary key
|
||||
|
||||
for (const auto & col : sample_block)
|
||||
{
|
||||
String original = original_names.find(col.name)->second;
|
||||
if (col.name == right_keys[0])
|
||||
{
|
||||
if (original != dict_key)
|
||||
return false; /// JOIN key != Dictionary key
|
||||
continue; /// do not extract key column
|
||||
}
|
||||
|
||||
names.push_back(original);
|
||||
result_columns.push_back({col.name, col.type});
|
||||
auto it = original_names.find(col.name);
|
||||
if (it != original_names.end())
|
||||
{
|
||||
String original = it->second;
|
||||
src_names.push_back(original);
|
||||
dst_columns.push_back({col.name, col.type});
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@ -12,7 +12,7 @@ ASTPtr ASTNameTypePair::clone() const
|
||||
|
||||
if (type)
|
||||
{
|
||||
res->type = type;
|
||||
res->type = type->clone();
|
||||
res->children.push_back(res->type);
|
||||
}
|
||||
|
||||
|
||||
@ -7,10 +7,11 @@
|
||||
#include <IO/copyData.h>
|
||||
#include <arrow/api.h>
|
||||
#include <arrow/ipc/reader.h>
|
||||
#include <arrow/status.h>
|
||||
#include <arrow/result.h>
|
||||
#include "ArrowBufferedStreams.h"
|
||||
#include "ArrowColumnToCHColumn.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -30,13 +31,12 @@ Chunk ArrowBlockInputFormat::generate()
|
||||
{
|
||||
Chunk res;
|
||||
const Block & header = getPort().getHeader();
|
||||
std::vector<std::shared_ptr<arrow::RecordBatch>> single_batch(1);
|
||||
arrow::Status read_status;
|
||||
arrow::Result<std::shared_ptr<arrow::RecordBatch>> batch_result;
|
||||
|
||||
if (stream)
|
||||
{
|
||||
read_status = stream_reader->ReadNext(&single_batch[0]);
|
||||
if (!single_batch[0])
|
||||
batch_result = stream_reader->Next();
|
||||
if (batch_result.ok() && !(*batch_result))
|
||||
return res;
|
||||
}
|
||||
else
|
||||
@ -44,22 +44,21 @@ Chunk ArrowBlockInputFormat::generate()
|
||||
if (record_batch_current >= record_batch_total)
|
||||
return res;
|
||||
|
||||
read_status = file_reader->ReadRecordBatch(record_batch_current, &single_batch[0]);
|
||||
batch_result = file_reader->ReadRecordBatch(record_batch_current);
|
||||
}
|
||||
|
||||
if (!read_status.ok())
|
||||
throw Exception{"Error while reading batch of Arrow data: " + read_status.ToString(),
|
||||
ErrorCodes::CANNOT_READ_ALL_DATA};
|
||||
if (!batch_result.ok())
|
||||
throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA,
|
||||
"Error while reading batch of Arrow data: {}", batch_result.status().ToString());
|
||||
|
||||
std::shared_ptr<arrow::Table> table;
|
||||
arrow::Status make_status = arrow::Table::FromRecordBatches(single_batch, &table);
|
||||
if (!make_status.ok())
|
||||
throw Exception{"Error while reading table of Arrow data: " + read_status.ToString(),
|
||||
ErrorCodes::CANNOT_READ_ALL_DATA};
|
||||
auto table_result = arrow::Table::FromRecordBatches({*batch_result});
|
||||
if (!table_result.ok())
|
||||
throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA,
|
||||
"Error while reading batch of Arrow data: {}", table_result.status().ToString());
|
||||
|
||||
++record_batch_current;
|
||||
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, header, "Arrow");
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, *table_result, header, "Arrow");
|
||||
|
||||
return res;
|
||||
}
|
||||
@ -77,15 +76,22 @@ void ArrowBlockInputFormat::resetParser()
|
||||
|
||||
void ArrowBlockInputFormat::prepareReader()
|
||||
{
|
||||
arrow::Status status;
|
||||
|
||||
if (stream)
|
||||
status = arrow::ipc::RecordBatchStreamReader::Open(asArrowFile(in), &stream_reader);
|
||||
{
|
||||
auto stream_reader_status = arrow::ipc::RecordBatchStreamReader::Open(asArrowFile(in));
|
||||
if (!stream_reader_status.ok())
|
||||
throw Exception(ErrorCodes::UNKNOWN_EXCEPTION,
|
||||
"Error while opening a table: {}", stream_reader_status.status().ToString());
|
||||
stream_reader = *stream_reader_status;
|
||||
}
|
||||
else
|
||||
status = arrow::ipc::RecordBatchFileReader::Open(asArrowFile(in), &file_reader);
|
||||
|
||||
if (!status.ok())
|
||||
throw Exception{"Error while opening a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION};
|
||||
{
|
||||
auto file_reader_status = arrow::ipc::RecordBatchFileReader::Open(asArrowFile(in));
|
||||
if (!file_reader_status.ok())
|
||||
throw Exception(ErrorCodes::UNKNOWN_EXCEPTION,
|
||||
"Error while opening a table: {}", file_reader_status.status().ToString());
|
||||
file_reader = *file_reader_status;
|
||||
}
|
||||
|
||||
if (stream)
|
||||
record_batch_total = -1;
|
||||
|
||||
@ -5,9 +5,11 @@
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <arrow/ipc/writer.h>
|
||||
#include <arrow/table.h>
|
||||
#include <arrow/result.h>
|
||||
#include "ArrowBufferedStreams.h"
|
||||
#include "CHColumnToArrowColumn.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
@ -35,7 +37,8 @@ void ArrowBlockOutputFormat::consume(Chunk chunk)
|
||||
auto status = writer->WriteTable(*arrow_table, format_settings.arrow.row_group_size);
|
||||
|
||||
if (!status.ok())
|
||||
throw Exception{"Error while writing a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION};
|
||||
throw Exception(ErrorCodes::UNKNOWN_EXCEPTION,
|
||||
"Error while writing a table: {}", status.ToString());
|
||||
}
|
||||
|
||||
void ArrowBlockOutputFormat::finalize()
|
||||
@ -44,22 +47,26 @@ void ArrowBlockOutputFormat::finalize()
|
||||
{
|
||||
auto status = writer->Close();
|
||||
if (!status.ok())
|
||||
throw Exception{"Error while closing a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION};
|
||||
throw Exception(ErrorCodes::UNKNOWN_EXCEPTION,
|
||||
"Error while closing a table: {}", status.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
void ArrowBlockOutputFormat::prepareWriter(const std::shared_ptr<arrow::Schema> & schema)
|
||||
{
|
||||
arrow::Status status;
|
||||
arrow::Result<std::shared_ptr<arrow::ipc::RecordBatchWriter>> writer_status;
|
||||
|
||||
// TODO: should we use arrow::ipc::IpcOptions::alignment?
|
||||
if (stream)
|
||||
status = arrow::ipc::RecordBatchStreamWriter::Open(arrow_ostream.get(), schema, &writer);
|
||||
writer_status = arrow::ipc::NewStreamWriter(arrow_ostream.get(), schema);
|
||||
else
|
||||
status = arrow::ipc::RecordBatchFileWriter::Open(arrow_ostream.get(), schema, &writer);
|
||||
writer_status = arrow::ipc::NewFileWriter(arrow_ostream.get(), schema);
|
||||
|
||||
if (!status.ok())
|
||||
throw Exception{"Error while opening a table writer: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION};
|
||||
if (!writer_status.ok())
|
||||
throw Exception(ErrorCodes::UNKNOWN_EXCEPTION,
|
||||
"Error while opening a table writer: {}", writer_status.status().ToString());
|
||||
|
||||
writer = *writer_status;
|
||||
}
|
||||
|
||||
void registerOutputFormatProcessorArrow(FormatFactory & factory)
|
||||
|
||||
@ -7,10 +7,11 @@
|
||||
#include <IO/copyData.h>
|
||||
#include <arrow/buffer.h>
|
||||
#include <arrow/io/api.h>
|
||||
#include <arrow/status.h>
|
||||
#include <arrow/result.h>
|
||||
|
||||
#include <sys/stat.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -65,12 +66,15 @@ arrow::Result<int64_t> RandomAccessFileFromSeekableReadBuffer::Read(int64_t nbyt
|
||||
|
||||
arrow::Result<std::shared_ptr<arrow::Buffer>> RandomAccessFileFromSeekableReadBuffer::Read(int64_t nbytes)
|
||||
{
|
||||
std::shared_ptr<arrow::Buffer> buf;
|
||||
ARROW_RETURN_NOT_OK(arrow::AllocateBuffer(nbytes, &buf));
|
||||
size_t n = in.readBig(reinterpret_cast<char *>(buf->mutable_data()), nbytes);
|
||||
auto buffer_status = arrow::AllocateBuffer(nbytes);
|
||||
ARROW_RETURN_NOT_OK(buffer_status);
|
||||
|
||||
auto read_buffer = arrow::SliceBuffer(buf, 0, n);
|
||||
return arrow::Result<std::shared_ptr<arrow::Buffer>>(read_buffer);
|
||||
auto shared_buffer = std::shared_ptr<arrow::Buffer>(std::move(std::move(*buffer_status)));
|
||||
|
||||
size_t n = in.readBig(reinterpret_cast<char *>(shared_buffer->mutable_data()), nbytes);
|
||||
|
||||
auto read_buffer = arrow::SliceBuffer(shared_buffer, 0, n);
|
||||
return arrow::Result<std::shared_ptr<arrow::Buffer>>(shared_buffer);
|
||||
}
|
||||
|
||||
arrow::Status RandomAccessFileFromSeekableReadBuffer::Seek(int64_t position)
|
||||
|
||||
59
src/Processors/Formats/Impl/TabSeparatedRawRowInputFormat.h
Normal file
59
src/Processors/Formats/Impl/TabSeparatedRawRowInputFormat.h
Normal file
@ -0,0 +1,59 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Block.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <Processors/Formats/RowInputFormatWithDiagnosticInfo.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** A stream to input data in tsv format, but without escaping individual values.
|
||||
* It only supports columns without '\n' or '\t'
|
||||
*/
|
||||
class TabSeparatedRawRowInputFormat : public TabSeparatedRowInputFormat
|
||||
{
|
||||
public:
|
||||
/** with_names - the first line is the header with the names of the columns
|
||||
* with_types - on the next line header with type names
|
||||
*/
|
||||
TabSeparatedRawRowInputFormat(
|
||||
const Block & header_,
|
||||
ReadBuffer & in_,
|
||||
const Params & params_,
|
||||
bool with_names_,
|
||||
bool with_types_,
|
||||
const FormatSettings & format_settings_)
|
||||
: TabSeparatedRowInputFormat(header_, in_, params_, with_names_, with_types_, format_settings_)
|
||||
{
|
||||
}
|
||||
|
||||
String getName() const override { return "TabSeparatedRawRowInputFormat"; }
|
||||
|
||||
bool readField(IColumn & column, const DataTypePtr & type, bool) override
|
||||
{
|
||||
String tmp;
|
||||
|
||||
while (!in.eof())
|
||||
{
|
||||
char * pos = find_first_symbols<'\n', '\t'>(in.position(), in.buffer().end());
|
||||
|
||||
tmp.append(in.position(), pos - in.position());
|
||||
in.position() = pos;
|
||||
|
||||
if (pos == in.buffer().end())
|
||||
in.next();
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
ReadBufferFromString cell(tmp);
|
||||
|
||||
type->deserializeAsWholeText(column, cell, format_settings);
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@ -3,6 +3,7 @@
|
||||
#include <IO/Operators.h>
|
||||
|
||||
#include <Processors/Formats/Impl/TabSeparatedRowInputFormat.h>
|
||||
#include <Processors/Formats/Impl/TabSeparatedRawRowInputFormat.h>
|
||||
#include <Formats/verbosePrintString.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <DataTypes/DataTypeNothing.h>
|
||||
@ -360,6 +361,18 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory)
|
||||
});
|
||||
}
|
||||
|
||||
for (const auto * name : {"TabSeparatedRaw", "TSVRaw"})
|
||||
{
|
||||
factory.registerInputFormatProcessor(name, [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<TabSeparatedRawRowInputFormat>(sample, buf, params, false, false, settings);
|
||||
});
|
||||
}
|
||||
|
||||
for (const auto * name : {"TabSeparatedWithNames", "TSVWithNames"})
|
||||
{
|
||||
factory.registerInputFormatProcessor(name, [](
|
||||
|
||||
@ -28,10 +28,14 @@ public:
|
||||
|
||||
void resetParser() override;
|
||||
|
||||
private:
|
||||
protected:
|
||||
bool with_names;
|
||||
bool with_types;
|
||||
const FormatSettings format_settings;
|
||||
|
||||
virtual bool readField(IColumn & column, const DataTypePtr & type, bool is_last_file_column);
|
||||
|
||||
private:
|
||||
DataTypes data_types;
|
||||
|
||||
using IndexesMap = std::unordered_map<String, size_t>;
|
||||
@ -43,8 +47,6 @@ private:
|
||||
std::vector<UInt8> read_columns;
|
||||
std::vector<size_t> columns_to_fill_with_default_values;
|
||||
|
||||
bool readField(IColumn & column, const DataTypePtr & type, bool is_last_file_column);
|
||||
|
||||
void addInputColumn(const String & column_name);
|
||||
void setupAllColumnsByTableSchema();
|
||||
void fillUnreadColumnsWithDefaults(MutableColumns & columns, RowReadExtension & row_read_extension);
|
||||
|
||||
@ -10,7 +10,7 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
LimitTransform::LimitTransform(
|
||||
const Block & header_, size_t limit_, size_t offset_, size_t num_streams,
|
||||
const Block & header_, UInt64 limit_, UInt64 offset_, size_t num_streams,
|
||||
bool always_read_till_end_, bool with_ties_,
|
||||
SortDescription description_)
|
||||
: IProcessor(InputPorts(num_streams, header_), OutputPorts(num_streams, header_))
|
||||
@ -46,7 +46,7 @@ LimitTransform::LimitTransform(
|
||||
}
|
||||
}
|
||||
|
||||
Chunk LimitTransform::makeChunkWithPreviousRow(const Chunk & chunk, size_t row) const
|
||||
Chunk LimitTransform::makeChunkWithPreviousRow(const Chunk & chunk, UInt64 row) const
|
||||
{
|
||||
assert(row < chunk.getNumRows());
|
||||
ColumnRawPtrs current_columns = extractSortColumns(chunk.getColumns());
|
||||
@ -93,7 +93,6 @@ IProcessor::Status LimitTransform::prepare(
|
||||
throw Exception(
|
||||
"Unexpected status for LimitTransform::preparePair : " + IProcessor::statusToName(status),
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
@ -107,9 +106,12 @@ IProcessor::Status LimitTransform::prepare(
|
||||
if (num_finished_port_pairs == ports_data.size())
|
||||
return Status::Finished;
|
||||
|
||||
bool limit_is_unreachable = (limit > std::numeric_limits<UInt64>::max() - offset);
|
||||
|
||||
/// If we reached limit for some port, then close others. Otherwise some sources may infinitely read data.
|
||||
/// Example: SELECT * FROM system.numbers_mt WHERE number = 1000000 LIMIT 1
|
||||
if ((rows_read >= offset + limit) && !previous_row_chunk && !always_read_till_end)
|
||||
if ((!limit_is_unreachable && rows_read >= offset + limit)
|
||||
&& !previous_row_chunk && !always_read_till_end)
|
||||
{
|
||||
for (auto & input : inputs)
|
||||
input.close();
|
||||
@ -158,8 +160,10 @@ LimitTransform::Status LimitTransform::preparePair(PortsData & data)
|
||||
return Status::PortFull;
|
||||
}
|
||||
|
||||
bool limit_is_unreachable = (limit > std::numeric_limits<UInt64>::max() - offset);
|
||||
|
||||
/// Check if we are done with pushing.
|
||||
bool is_limit_reached = (rows_read >= offset + limit) && !previous_row_chunk;
|
||||
bool is_limit_reached = !limit_is_unreachable && rows_read >= offset + limit && !previous_row_chunk;
|
||||
if (is_limit_reached)
|
||||
{
|
||||
if (!always_read_till_end)
|
||||
@ -223,7 +227,8 @@ LimitTransform::Status LimitTransform::preparePair(PortsData & data)
|
||||
return Status::NeedData;
|
||||
}
|
||||
|
||||
if (rows_read >= offset + rows && rows_read <= offset + limit)
|
||||
if (rows <= std::numeric_limits<UInt64>::max() - offset && rows_read >= offset + rows
|
||||
&& !limit_is_unreachable && rows_read <= offset + limit)
|
||||
{
|
||||
/// Return the whole chunk.
|
||||
|
||||
@ -237,7 +242,7 @@ LimitTransform::Status LimitTransform::preparePair(PortsData & data)
|
||||
|
||||
bool may_need_more_data_for_ties = previous_row_chunk || rows_read - rows <= offset + limit;
|
||||
/// No more data is needed.
|
||||
if (!always_read_till_end && (rows_read >= offset + limit) && !may_need_more_data_for_ties)
|
||||
if (!always_read_till_end && !limit_is_unreachable && rows_read >= offset + limit && !may_need_more_data_for_ties)
|
||||
input.close();
|
||||
|
||||
output.push(std::move(data.current_chunk));
|
||||
@ -249,13 +254,15 @@ LimitTransform::Status LimitTransform::preparePair(PortsData & data)
|
||||
void LimitTransform::splitChunk(PortsData & data)
|
||||
{
|
||||
auto current_chunk_sort_columns = extractSortColumns(data.current_chunk.getColumns());
|
||||
size_t num_rows = data.current_chunk.getNumRows();
|
||||
size_t num_columns = data.current_chunk.getNumColumns();
|
||||
UInt64 num_rows = data.current_chunk.getNumRows();
|
||||
UInt64 num_columns = data.current_chunk.getNumColumns();
|
||||
|
||||
if (previous_row_chunk && rows_read >= offset + limit)
|
||||
bool limit_is_unreachable = (limit > std::numeric_limits<UInt64>::max() - offset);
|
||||
|
||||
if (previous_row_chunk && !limit_is_unreachable && rows_read >= offset + limit)
|
||||
{
|
||||
/// Scan until the first row, which is not equal to previous_row_chunk (for WITH TIES)
|
||||
size_t current_row_num = 0;
|
||||
UInt64 current_row_num = 0;
|
||||
for (; current_row_num < num_rows; ++current_row_num)
|
||||
{
|
||||
if (!sortColumnsEqualAt(current_chunk_sort_columns, current_row_num))
|
||||
@ -267,7 +274,7 @@ void LimitTransform::splitChunk(PortsData & data)
|
||||
if (current_row_num < num_rows)
|
||||
{
|
||||
previous_row_chunk = {};
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
for (UInt64 i = 0; i < num_columns; ++i)
|
||||
columns[i] = columns[i]->cut(0, current_row_num);
|
||||
}
|
||||
|
||||
@ -276,19 +283,51 @@ void LimitTransform::splitChunk(PortsData & data)
|
||||
}
|
||||
|
||||
/// return a piece of the block
|
||||
size_t start = std::max(
|
||||
static_cast<Int64>(0),
|
||||
static_cast<Int64>(offset) - static_cast<Int64>(rows_read) + static_cast<Int64>(num_rows));
|
||||
UInt64 start = 0;
|
||||
|
||||
size_t length = std::min(
|
||||
static_cast<Int64>(limit), std::min(
|
||||
static_cast<Int64>(rows_read) - static_cast<Int64>(offset),
|
||||
static_cast<Int64>(limit) + static_cast<Int64>(offset) - static_cast<Int64>(rows_read) + static_cast<Int64>(num_rows)));
|
||||
/// ------------[....(...).]
|
||||
/// <----------------------> rows_read
|
||||
/// <----------> num_rows
|
||||
/// <---------------> offset
|
||||
/// <---> start
|
||||
|
||||
assert(offset < rows_read);
|
||||
|
||||
if (offset + num_rows > rows_read)
|
||||
start = offset + num_rows - rows_read;
|
||||
|
||||
/// ------------[....(...).]
|
||||
/// <----------------------> rows_read
|
||||
/// <----------> num_rows
|
||||
/// <---------------> offset
|
||||
/// <---> limit
|
||||
/// <---> length
|
||||
/// <---> start
|
||||
|
||||
/// Or:
|
||||
|
||||
/// -----------------(------[....)....]
|
||||
/// <---------------------------------> rows_read
|
||||
/// <---------> num_rows
|
||||
/// <---------------> offset
|
||||
/// <-----------> limit
|
||||
/// <----> length
|
||||
/// 0 = start
|
||||
|
||||
UInt64 length = num_rows - start;
|
||||
|
||||
if (!limit_is_unreachable && offset + limit < rows_read)
|
||||
{
|
||||
if (offset + limit < rows_read - num_rows)
|
||||
length = 0;
|
||||
else
|
||||
length = offset + limit - (rows_read - num_rows) - start;
|
||||
}
|
||||
|
||||
/// check if other rows in current block equals to last one in limit
|
||||
if (with_ties && length)
|
||||
{
|
||||
size_t current_row_num = start + length;
|
||||
UInt64 current_row_num = start + length;
|
||||
previous_row_chunk = makeChunkWithPreviousRow(data.current_chunk, current_row_num - 1);
|
||||
|
||||
for (; current_row_num < num_rows; ++current_row_num)
|
||||
@ -308,7 +347,7 @@ void LimitTransform::splitChunk(PortsData & data)
|
||||
|
||||
auto columns = data.current_chunk.detachColumns();
|
||||
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
for (UInt64 i = 0; i < num_columns; ++i)
|
||||
columns[i] = columns[i]->cut(start, length);
|
||||
|
||||
data.current_chunk.setColumns(std::move(columns), length);
|
||||
@ -324,7 +363,7 @@ ColumnRawPtrs LimitTransform::extractSortColumns(const Columns & columns) const
|
||||
return res;
|
||||
}
|
||||
|
||||
bool LimitTransform::sortColumnsEqualAt(const ColumnRawPtrs & current_chunk_sort_columns, size_t current_chunk_row_num) const
|
||||
bool LimitTransform::sortColumnsEqualAt(const ColumnRawPtrs & current_chunk_sort_columns, UInt64 current_chunk_row_num) const
|
||||
{
|
||||
assert(current_chunk_sort_columns.size() == previous_row_chunk.getNumColumns());
|
||||
size_t size = current_chunk_sort_columns.size();
|
||||
|
||||
@ -18,9 +18,9 @@ namespace DB
|
||||
class LimitTransform : public IProcessor
|
||||
{
|
||||
private:
|
||||
UInt64 limit;
|
||||
UInt64 offset;
|
||||
|
||||
size_t limit;
|
||||
size_t offset;
|
||||
bool always_read_till_end;
|
||||
|
||||
bool with_ties;
|
||||
@ -29,7 +29,7 @@ private:
|
||||
Chunk previous_row_chunk; /// for WITH TIES, contains only sort columns
|
||||
std::vector<size_t> sort_column_positions;
|
||||
|
||||
size_t rows_read = 0; /// including the last read block
|
||||
UInt64 rows_read = 0; /// including the last read block
|
||||
RowsBeforeLimitCounterPtr rows_before_limit_at_least;
|
||||
|
||||
/// State of port's pair.
|
||||
@ -46,13 +46,13 @@ private:
|
||||
std::vector<PortsData> ports_data;
|
||||
size_t num_finished_port_pairs = 0;
|
||||
|
||||
Chunk makeChunkWithPreviousRow(const Chunk & current_chunk, size_t row_num) const;
|
||||
Chunk makeChunkWithPreviousRow(const Chunk & current_chunk, UInt64 row_num) const;
|
||||
ColumnRawPtrs extractSortColumns(const Columns & columns) const;
|
||||
bool sortColumnsEqualAt(const ColumnRawPtrs & current_chunk_sort_columns, size_t current_chunk_row_num) const;
|
||||
bool sortColumnsEqualAt(const ColumnRawPtrs & current_chunk_sort_columns, UInt64 current_chunk_row_num) const;
|
||||
|
||||
public:
|
||||
LimitTransform(
|
||||
const Block & header_, size_t limit_, size_t offset_, size_t num_streams = 1,
|
||||
const Block & header_, UInt64 limit_, UInt64 offset_, size_t num_streams = 1,
|
||||
bool always_read_till_end_ = false, bool with_ties_ = false,
|
||||
SortDescription description_ = {});
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
OffsetTransform::OffsetTransform(
|
||||
const Block & header_, size_t offset_, size_t num_streams)
|
||||
const Block & header_, UInt64 offset_, size_t num_streams)
|
||||
: IProcessor(InputPorts(num_streams, header_), OutputPorts(num_streams, header_))
|
||||
, offset(offset_)
|
||||
{
|
||||
@ -135,7 +135,7 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data)
|
||||
|
||||
rows_read += rows;
|
||||
|
||||
if (rows_read < offset)
|
||||
if (rows_read <= offset)
|
||||
{
|
||||
data.current_chunk.clear();
|
||||
|
||||
@ -150,7 +150,7 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data)
|
||||
return Status::NeedData;
|
||||
}
|
||||
|
||||
if (!(rows_read >= offset + rows))
|
||||
if (!(rows <= std::numeric_limits<UInt64>::max() - offset && rows_read >= offset + rows))
|
||||
splitChunk(data);
|
||||
|
||||
output.push(std::move(data.current_chunk));
|
||||
@ -161,22 +161,30 @@ OffsetTransform::Status OffsetTransform::preparePair(PortsData & data)
|
||||
|
||||
void OffsetTransform::splitChunk(PortsData & data) const
|
||||
{
|
||||
size_t num_rows = data.current_chunk.getNumRows();
|
||||
size_t num_columns = data.current_chunk.getNumColumns();
|
||||
UInt64 num_rows = data.current_chunk.getNumRows();
|
||||
UInt64 num_columns = data.current_chunk.getNumColumns();
|
||||
|
||||
/// return a piece of the block
|
||||
size_t start = std::max(
|
||||
static_cast<Int64>(0),
|
||||
static_cast<Int64>(offset) - static_cast<Int64>(rows_read) + static_cast<Int64>(num_rows));
|
||||
UInt64 start = 0;
|
||||
|
||||
size_t length = static_cast<Int64>(rows_read) - static_cast<Int64>(offset);
|
||||
/// ------------[....(.....]
|
||||
/// <----------------------> rows_read
|
||||
/// <----------> num_rows
|
||||
/// <---------------> offset
|
||||
/// <---> start
|
||||
|
||||
if (length == num_rows)
|
||||
assert(offset < rows_read);
|
||||
|
||||
if (offset + num_rows > rows_read)
|
||||
start = offset + num_rows - rows_read;
|
||||
else
|
||||
return;
|
||||
|
||||
UInt64 length = num_rows - start;
|
||||
|
||||
auto columns = data.current_chunk.detachColumns();
|
||||
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
for (UInt64 i = 0; i < num_columns; ++i)
|
||||
columns[i] = columns[i]->cut(start, length);
|
||||
|
||||
data.current_chunk.setColumns(std::move(columns), length);
|
||||
|
||||
@ -13,10 +13,9 @@ namespace DB
|
||||
class OffsetTransform : public IProcessor
|
||||
{
|
||||
private:
|
||||
UInt64 offset;
|
||||
UInt64 rows_read = 0; /// including the last read block
|
||||
|
||||
size_t offset;
|
||||
|
||||
size_t rows_read = 0; /// including the last read block
|
||||
RowsBeforeLimitCounterPtr rows_before_limit_at_least;
|
||||
|
||||
/// State of port's pair.
|
||||
@ -34,7 +33,7 @@ private:
|
||||
size_t num_finished_port_pairs = 0;
|
||||
|
||||
public:
|
||||
OffsetTransform(const Block & header_, size_t offset_, size_t num_streams = 1);
|
||||
OffsetTransform(const Block & header_, UInt64 offset_, size_t num_streams = 1);
|
||||
|
||||
String getName() const override { return "Offset"; }
|
||||
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
LimitByTransform::LimitByTransform(const Block & header, size_t group_length_, size_t group_offset_, const Names & columns)
|
||||
LimitByTransform::LimitByTransform(const Block & header, UInt64 group_length_, UInt64 group_offset_, const Names & columns)
|
||||
: ISimpleTransform(header, header, true)
|
||||
, group_length(group_length_)
|
||||
, group_offset(group_offset_)
|
||||
@ -25,13 +25,13 @@ LimitByTransform::LimitByTransform(const Block & header, size_t group_length_, s
|
||||
|
||||
void LimitByTransform::transform(Chunk & chunk)
|
||||
{
|
||||
size_t num_rows = chunk.getNumRows();
|
||||
UInt64 num_rows = chunk.getNumRows();
|
||||
auto columns = chunk.detachColumns();
|
||||
|
||||
IColumn::Filter filter(num_rows);
|
||||
size_t inserted_count = 0;
|
||||
UInt64 inserted_count = 0;
|
||||
|
||||
for (size_t row = 0; row < num_rows; ++row)
|
||||
for (UInt64 row = 0; row < num_rows; ++row)
|
||||
{
|
||||
UInt128 key(0, 0);
|
||||
SipHash hash;
|
||||
@ -42,9 +42,10 @@ void LimitByTransform::transform(Chunk & chunk)
|
||||
hash.get128(key.low, key.high);
|
||||
|
||||
auto count = keys_counts[key]++;
|
||||
if (count >= group_offset && count < group_length + group_offset)
|
||||
if (count >= group_offset
|
||||
&& (group_length > std::numeric_limits<UInt64>::max() - group_offset || count < group_length + group_offset))
|
||||
{
|
||||
inserted_count++;
|
||||
++inserted_count;
|
||||
filter[row] = 1;
|
||||
}
|
||||
else
|
||||
|
||||
@ -10,7 +10,7 @@ namespace DB
|
||||
class LimitByTransform : public ISimpleTransform
|
||||
{
|
||||
public:
|
||||
LimitByTransform(const Block & header, size_t group_length_, size_t group_offset_, const Names & columns);
|
||||
LimitByTransform(const Block & header, UInt64 group_length_, UInt64 group_offset_, const Names & columns);
|
||||
|
||||
String getName() const override { return "LimitByTransform"; }
|
||||
|
||||
@ -22,8 +22,8 @@ private:
|
||||
|
||||
MapHashed keys_counts;
|
||||
std::vector<size_t> key_positions;
|
||||
const size_t group_length;
|
||||
const size_t group_offset;
|
||||
const UInt64 group_length;
|
||||
const UInt64 group_offset;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1003,6 +1003,9 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con
|
||||
}
|
||||
}
|
||||
|
||||
if (all_columns.empty())
|
||||
throw Exception{"Cannot DROP or CLEAR all columns", ErrorCodes::BAD_ARGUMENTS};
|
||||
|
||||
validateColumnsDefaultsAndGetSampleBlock(default_expr_list, all_columns.getAll(), context);
|
||||
}
|
||||
|
||||
|
||||
@ -121,6 +121,11 @@ public:
|
||||
return columns.size();
|
||||
}
|
||||
|
||||
bool empty() const
|
||||
{
|
||||
return columns.empty();
|
||||
}
|
||||
|
||||
/// Keep the sequence of columns and allow to lookup by name.
|
||||
using Container = boost::multi_index_container<
|
||||
ColumnDescription,
|
||||
|
||||
@ -102,6 +102,10 @@ void IStorage::checkAlterIsPossible(const AlterCommands & commands, const Settin
|
||||
}
|
||||
}
|
||||
|
||||
void IStorage::checkAlterPartitionIsPossible(const PartitionCommands & /*commands*/, const StorageMetadataPtr & /*metadata_snapshot*/, const Settings & /*settings*/) const
|
||||
{
|
||||
throw Exception("Table engine " + getName() + " doesn't support partitioning", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
StorageID IStorage::getStorageID() const
|
||||
{
|
||||
|
||||
@ -360,6 +360,9 @@ public:
|
||||
throw Exception("Partition operations are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/// Checks that partition commands can be applied to storage.
|
||||
virtual void checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & metadata_snapshot, const Settings & settings) const;
|
||||
|
||||
/** Perform any background work. For example, combining parts in a MergeTree type table.
|
||||
* Returns whether any work has been done.
|
||||
*/
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
|
||||
#include <cppkafka/cppkafka.h>
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <algorithm>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -388,24 +389,43 @@ bool ReadBufferFromKafkaConsumer::poll()
|
||||
}
|
||||
}
|
||||
|
||||
while (auto err = current->get_error())
|
||||
filterMessageErrors();
|
||||
if (current == messages.end())
|
||||
{
|
||||
++current;
|
||||
|
||||
// TODO: should throw exception instead
|
||||
LOG_ERROR(log, "Consumer error: {}", err);
|
||||
if (current == messages.end())
|
||||
{
|
||||
LOG_ERROR(log, "No actual messages polled, errors only.");
|
||||
stalled_status = ERRORS_RETURNED;
|
||||
return false;
|
||||
}
|
||||
LOG_ERROR(log, "Only errors left");
|
||||
stalled_status = ERRORS_RETURNED;
|
||||
return false;
|
||||
}
|
||||
|
||||
stalled_status = NOT_STALLED;
|
||||
allowed = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t ReadBufferFromKafkaConsumer::filterMessageErrors()
|
||||
{
|
||||
assert(current == messages.begin());
|
||||
|
||||
auto new_end = std::remove_if(messages.begin(), messages.end(), [this](auto & message)
|
||||
{
|
||||
if (auto error = message.get_error())
|
||||
{
|
||||
LOG_ERROR(log, "Consumer error: {}", error);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
size_t skipped = std::distance(messages.end(), new_end);
|
||||
if (skipped)
|
||||
{
|
||||
LOG_ERROR(log, "There were {} messages with an error", skipped);
|
||||
messages.erase(new_end, messages.end());
|
||||
}
|
||||
|
||||
return skipped;
|
||||
}
|
||||
|
||||
void ReadBufferFromKafkaConsumer::resetIfStopped()
|
||||
{
|
||||
// we can react on stop only during fetching data
|
||||
|
||||
@ -102,6 +102,8 @@ private:
|
||||
void drain();
|
||||
void cleanUnprocessed();
|
||||
void resetIfStopped();
|
||||
/// Return number of messages with an error.
|
||||
size_t filterMessageErrors();
|
||||
|
||||
bool nextImpl() override;
|
||||
};
|
||||
|
||||
@ -110,6 +110,7 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_DISK;
|
||||
extern const int NOT_ENOUGH_SPACE;
|
||||
extern const int ALTER_OF_COLUMN_IS_FORBIDDEN;
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
}
|
||||
|
||||
|
||||
@ -1421,12 +1422,20 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S
|
||||
columns_in_keys.insert(columns_alter_type_metadata_only.begin(), columns_alter_type_metadata_only.end());
|
||||
columns_in_keys.insert(columns_alter_type_check_safe_for_partition.begin(), columns_alter_type_check_safe_for_partition.end());
|
||||
|
||||
NameSet dropped_columns;
|
||||
|
||||
std::map<String, const IDataType *> old_types;
|
||||
for (const auto & column : old_metadata.getColumns().getAllPhysical())
|
||||
old_types.emplace(column.name, column.type.get());
|
||||
|
||||
for (const AlterCommand & command : commands)
|
||||
{
|
||||
/// Just validate partition expression
|
||||
if (command.partition)
|
||||
{
|
||||
getPartitionIDFromQuery(command.partition, global_context);
|
||||
}
|
||||
|
||||
if (command.type == AlterCommand::MODIFY_ORDER_BY && !is_custom_partitioned)
|
||||
{
|
||||
throw Exception(
|
||||
@ -1456,6 +1465,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S
|
||||
"Trying to ALTER DROP key " + backQuoteIfNeed(command.column_name) + " column which is a part of key expression",
|
||||
ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN);
|
||||
}
|
||||
dropped_columns.emplace(command.column_name);
|
||||
}
|
||||
else if (command.isModifyingData(getInMemoryMetadata()))
|
||||
{
|
||||
@ -1530,6 +1540,27 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S
|
||||
checkStoragePolicy(global_context.getStoragePolicy(changed_setting.value.safeGet<String>()));
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto & part : getDataPartsVector())
|
||||
{
|
||||
bool at_least_one_column_rest = false;
|
||||
for (const auto & column : part->getColumns())
|
||||
{
|
||||
if (!dropped_columns.count(column.name))
|
||||
{
|
||||
at_least_one_column_rest = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!at_least_one_column_rest)
|
||||
{
|
||||
std::string postfix;
|
||||
if (dropped_columns.size() > 1)
|
||||
postfix = "s";
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Cannot drop or clear column{} '{}', because all columns in part '{}' will be removed from disk. Empty parts are not allowed", postfix, boost::algorithm::join(dropped_columns, ", "), part->name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_uncompressed, size_t rows_count) const
|
||||
@ -2525,6 +2556,45 @@ void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const StorageM
|
||||
context);
|
||||
}
|
||||
|
||||
void MergeTreeData::checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & /*metadata_snapshot*/, const Settings & settings) const
|
||||
{
|
||||
for (const auto & command : commands)
|
||||
{
|
||||
if (command.type == PartitionCommand::DROP_DETACHED_PARTITION
|
||||
&& !settings.allow_drop_detached)
|
||||
throw DB::Exception("Cannot execute query: DROP DETACHED PART is disabled "
|
||||
"(see allow_drop_detached setting)", ErrorCodes::SUPPORT_IS_DISABLED);
|
||||
|
||||
if (command.partition && command.type != PartitionCommand::DROP_DETACHED_PARTITION)
|
||||
{
|
||||
if (command.part)
|
||||
{
|
||||
auto part_name = command.partition->as<ASTLiteral &>().value.safeGet<String>();
|
||||
/// We able to parse it
|
||||
MergeTreePartInfo::fromPartName(part_name, format_version);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// We able to parse it
|
||||
getPartitionIDFromQuery(command.partition, global_context);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void MergeTreeData::checkPartitionCanBeDropped(const ASTPtr & partition)
|
||||
{
|
||||
const String partition_id = getPartitionIDFromQuery(partition, global_context);
|
||||
auto parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id);
|
||||
|
||||
UInt64 partition_size = 0;
|
||||
|
||||
for (const auto & part : parts_to_remove)
|
||||
partition_size += part->getBytesOnDisk();
|
||||
|
||||
auto table_id = getStorageID();
|
||||
global_context.checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, partition_size);
|
||||
}
|
||||
|
||||
void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & name, bool moving_part, const Context & context)
|
||||
{
|
||||
@ -2626,7 +2696,7 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String
|
||||
}
|
||||
|
||||
|
||||
String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context & context)
|
||||
String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context & context) const
|
||||
{
|
||||
const auto & partition_ast = ast->as<ASTPartition &>();
|
||||
|
||||
@ -3045,7 +3115,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVector() const
|
||||
}
|
||||
|
||||
MergeTreeData::DataPartPtr MergeTreeData::getAnyPartInPartition(
|
||||
const String & partition_id, DataPartsLock & /*data_parts_lock*/)
|
||||
const String & partition_id, DataPartsLock & /*data_parts_lock*/) const
|
||||
{
|
||||
auto it = data_parts_by_state_and_info.lower_bound(DataPartStateAndPartitionID{DataPartState::Committed, partition_id});
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user