Merge branch 'master' into rocksdb_metacache

.github/workflows/nightly.yml

@@ -6,7 +6,7 @@ env:
 
 "on":
   schedule:
-    - cron: '0 0 * * *'
+    - cron: '13 3 * * *'
 
 jobs:
   DockerHubPushAarch64:

.potato.yml

@@ -1,27 +0,0 @@
-# This is the configuration file with settings for Potato.
-# Potato is an internal Yandex technology that allows us to sync internal [Yandex.Tracker](https://yandex.com/tracker/) and GitHub.
-
-# For all PRs where documentation is needed, just add a 'pr-feature' label and we will include it into documentation sprints.
-
-# The project name.
-name: clickhouse
-# Object handlers defines which handlers we use.
-handlers:
-  # The handler for creating an Yandex.Tracker issue.
-  - name: issue-create
-    params:
-      triggers:
-        # The trigger for creating the Yandex.Tracker issue. When the specified event occurs, it transfers PR data to Yandex.Tracker.
-        github:pullRequest:labeled:
-          data:
-          # The Yandex.Tracker queue to create the issue in. Each issue in Tracker belongs to one of the project queues.
-          queue: CLICKHOUSEDOCS
-          # The issue title.
-          summary: '[Potato] Pull Request #{{pullRequest.number}}'
-          # The issue description.
-          description: >
-                {{pullRequest.description}}
-
-                Ссылка на Pull Request: {{pullRequest.webUrl}}
-          # The condition for creating the Yandex.Tracker issue.
-          condition: eventPayload.labels.filter(label => ['pr-feature'].includes(label.name)).length

base/base/LineReader.cpp

@@ -109,10 +109,10 @@ void LineReader::Suggest::addWords(Words && new_words)
         std::lock_guard lock(mutex);
         addNewWords(words, new_words, std::less<std::string>{});
         addNewWords(words_no_case, new_words_no_case, NoCaseCompare{});
-    }
 
-    assert(std::is_sorted(words.begin(), words.end()));
-    assert(std::is_sorted(words_no_case.begin(), words_no_case.end(), NoCaseCompare{}));
+        assert(std::is_sorted(words.begin(), words.end()));
+        assert(std::is_sorted(words_no_case.begin(), words_no_case.end(), NoCaseCompare{}));
+    }
 }
 
 LineReader::LineReader(const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_)

base/base/sort.h

@@ -2,6 +2,63 @@
 
 #include <pdqsort.h>
 
+#ifndef NDEBUG
+
+#include <pcg_random.hpp>
+#include <base/getThreadId.h>
+
+/** Same as libcxx std::__debug_less. Just without dependency on private part of standard library.
+  * Check that Comparator induce strict weak ordering.
+  */
+template <typename Comparator>
+class DebugLessComparator
+{
+public:
+    constexpr DebugLessComparator(Comparator & cmp_)
+        : cmp(cmp_)
+    {}
+
+    template <typename LhsType, typename RhsType>
+    constexpr bool operator()(const LhsType & lhs, const RhsType & rhs)
+    {
+        bool lhs_less_than_rhs = cmp(lhs, rhs);
+        if (lhs_less_than_rhs)
+            assert(!cmp(rhs, lhs));
+
+        return lhs_less_than_rhs;
+    }
+
+    template <typename LhsType, typename RhsType>
+    constexpr bool operator()(LhsType & lhs, RhsType & rhs)
+    {
+        bool lhs_less_than_rhs = cmp(lhs, rhs);
+        if (lhs_less_than_rhs)
+            assert(!cmp(rhs, lhs));
+
+        return lhs_less_than_rhs;
+    }
+
+private:
+    Comparator & cmp;
+};
+
+template <typename Comparator>
+using ComparatorWrapper = DebugLessComparator<Comparator>;
+
+template <typename RandomIt>
+void shuffle(RandomIt first, RandomIt last)
+{
+    static thread_local pcg64 rng(getThreadId());
+    std::shuffle(first, last, rng);
+}
+
+#else
+
+template <typename Comparator>
+using ComparatorWrapper = Comparator;
+
+#endif
+
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wold-style-cast"
 
@@ -10,19 +67,48 @@
 template <typename RandomIt>
 void nth_element(RandomIt first, RandomIt nth, RandomIt last)
 {
-    ::miniselect::floyd_rivest_select(first, nth, last);
-}
+    using value_type = typename std::iterator_traits<RandomIt>::value_type;
+    using comparator = std::less<value_type>;
 
-template <typename RandomIt>
-void partial_sort(RandomIt first, RandomIt middle, RandomIt last)
-{
-    ::miniselect::floyd_rivest_partial_sort(first, middle, last);
+    comparator compare;
+    ComparatorWrapper<comparator> compare_wrapper = compare;
+
+#ifndef NDEBUG
+    ::shuffle(first, last);
+#endif
+
+    ::miniselect::floyd_rivest_select(first, nth, last, compare_wrapper);
+
+#ifndef NDEBUG
+    ::shuffle(first, nth);
+
+    if (nth != last)
+        ::shuffle(nth + 1, last);
+#endif
 }
 
 template <typename RandomIt, typename Compare>
 void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare)
 {
-    ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare);
+#ifndef NDEBUG
+    ::shuffle(first, last);
+#endif
+
+    ComparatorWrapper<Compare> compare_wrapper = compare;
+    ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare_wrapper);
+
+#ifndef NDEBUG
+    ::shuffle(middle, last);
+#endif
+}
+
+template <typename RandomIt>
+void partial_sort(RandomIt first, RandomIt middle, RandomIt last)
+{
+    using value_type = typename std::iterator_traits<RandomIt>::value_type;
+    using comparator = std::less<value_type>;
+
+    ::partial_sort(first, middle, last, comparator());
 }
 
 #pragma GCC diagnostic pop
@@ -30,7 +116,12 @@ void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compar
 template <typename RandomIt, typename Compare>
 void sort(RandomIt first, RandomIt last, Compare compare)
 {
-    ::pdqsort(first, last, compare);
+#ifndef NDEBUG
+    ::shuffle(first, last);
+#endif
+
+    ComparatorWrapper<Compare> compare_wrapper = compare;
+    ::pdqsort(first, last, compare_wrapper);
 }
 
 template <typename RandomIt>
@@ -38,5 +129,5 @@ void sort(RandomIt first, RandomIt last)
 {
     using value_type = typename std::iterator_traits<RandomIt>::value_type;
     using comparator = std::less<value_type>;
-    ::pdqsort(first, last, comparator());
+    ::sort(first, last, comparator());
 }

docker-compose.yml

@@ -1,15 +0,0 @@
-version: "2"
-
-services:
-  builder:
-    image: clickhouse/clickhouse-builder
-    build: docker/builder
-  client:
-    image: clickhouse/clickhouse-client
-    build: docker/client
-    command: ['--host', 'server']
-  server:
-    image: clickhouse/clickhouse-server
-    build: docker/server
-    ports:
-      - 8123:8123

docker/test/fuzzer/run-fuzzer.sh

@@ -185,15 +185,14 @@ handle SIGUSR2 nostop noprint pass
 handle SIG$RTMIN nostop noprint pass
 info signals
 continue
+gcore
 backtrace full
-info locals
+thread apply all backtrace full
 info registers
 disassemble /s
 up
-info locals
 disassemble /s
 up
-info locals
 disassemble /s
 p \"done\"
 detach
@@ -314,6 +313,11 @@ quit
             || echo "Fuzzer failed ($fuzzer_exit_code). See the logs." ; } \
             | tail -1 > description.txt
     fi
+
+    if test -f core.*; then
+        pigz core.*
+        mv core.*.gz core.gz
+    fi
 }
 
 case "$stage" in
@@ -345,6 +349,10 @@ case "$stage" in
     time fuzz
     ;&
 "report")
+CORE_LINK=''
+if [ -f core.gz ]; then
+    CORE_LINK='<a href="core.gz">core.gz</a>'
+fi
 cat > report.html <<EOF ||:
 <!DOCTYPE html>
 <html lang="en">
@@ -386,6 +394,7 @@ th { cursor: pointer; }
 <a href="fuzzer.log">fuzzer.log</a>
 <a href="server.log">server.log</a>
 <a href="main.log">main.log</a>
+${CORE_LINK}
 </p>
 <table>
 <tr><th>Test name</th><th>Test status</th><th>Description</th></tr>

docker/test/integration/kerberized_hadoop/Dockerfile

@@ -15,9 +15,10 @@ RUN curl -o krb5-libs-1.10.3-65.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault.
     rm -fr *.rpm
 
 RUN cd /tmp && \
-	curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \
-	tar xzf commons-daemon-1.0.15-src.tar.gz && \
-	cd commons-daemon-1.0.15-src/src/native/unix && \
-	./configure && \
-	make && \
-	cp ./jsvc /usr/local/hadoop-2.7.0/sbin
+    curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \
+    tar xzf commons-daemon-1.0.15-src.tar.gz && \
+    cd commons-daemon-1.0.15-src/src/native/unix && \
+    ./configure && \
+    make && \
+    cp ./jsvc /usr/local/hadoop-2.7.0/sbin && \
+    [ -e /usr/local/hadoop ] || ln -s ./hadoop-2.7.0 /usr/local/hadoop

docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml

@@ -4,7 +4,7 @@ services:
   kerberizedhdfs1:
     cap_add:
       - DAC_READ_SEARCH
-    image: clickhouse/kerberized-hadoop
+    image: clickhouse/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest}
     hostname: kerberizedhdfs1
     restart: always
     volumes:

docker/test/integration/runner/dockerd-entrypoint.sh

@@ -45,6 +45,7 @@ export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest}
 export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest}
 export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest}
 export DOCKER_KERBEROS_KDC_TAG=${DOCKER_KERBEROS_KDC_TAG:=latest}
+export DOCKER_KERBERIZED_HADOOP_TAG=${DOCKER_KERBERIZED_HADOOP_TAG:=latest}
 
 cd /ClickHouse/tests/integration
 exec "$@"

docker/test/stress/run.sh

@@ -148,14 +148,12 @@ info signals
 continue
 gcore
 backtrace full
-info locals
+thread apply all backtrace full
 info registers
 disassemble /s
 up
-info locals
 disassemble /s
 up
-info locals
 disassemble /s
 p \"done\"
 detach
@@ -269,5 +267,5 @@ clickhouse-local --structure "test String, res String" -q "SELECT 'failure', tes
 # Default filename is 'core.PROCESS_ID'
 for core in core.*; do
     pigz $core
-    mv $core.gz /output/
+    mv $core.gz /test_output/
 done

docs/en/engines/table-engines/special/distributed.md

@@ -209,6 +209,8 @@ When querying a `Distributed` table, `SELECT` queries are sent to all shards and
 
 When the `max_parallel_replicas` option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas).
 
+To learn more about how distibuted `in` and `global in` queries are processed, refer to [this](../../../sql-reference/operators/in.md#select-distributed-subqueries) documentation. 
+
 ## Virtual Columns {#virtual-columns}
 
 -   `_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md).

docs/en/engines/table-engines/special/url.md

@@ -7,18 +7,29 @@ toc_title: URL
 
 Queries data to/from a remote HTTP/HTTPS server. This engine is similar to the [File](../../../engines/table-engines/special/file.md) engine.
 
-Syntax: `URL(URL, Format)`
+Syntax: `URL(URL [,Format] [,CompressionMethod])`
+
+- The `URL` parameter must conform to the structure of a Uniform Resource Locator. The specified URL must point to a server that uses HTTP or HTTPS. This does not require any additional headers for getting a response from the server.
+
+- The `Format` must be one that ClickHouse can use in `SELECT` queries and, if necessary, in `INSERTs`. For the full list of supported formats, see [Formats](../../../interfaces/formats.md#formats).
+
+- `CompressionMethod` indicates that whether the HTTP body should be compressed. If the compression is enabled, the HTTP packets sent by the URL engine contain 'Content-Encoding' header to indicate which compression method is used. 
+
+To enable compression, please first make sure the remote HTTP endpoint indicated by the `URL` parameter supports corresponding compression algorithm.
+   
+The supported `CompressionMethod` should be one of following:
+- gzip or gz
+- deflate
+- brotli or br
+- lzma or xz
+- zstd or zst
+- lz4
+- bz2
+- snappy
+- none
 
 ## Usage {#using-the-engine-in-the-clickhouse-server}
 
-The `format` must be one that ClickHouse can use in
-`SELECT` queries and, if necessary, in `INSERTs`. For the full list of supported formats, see
-[Formats](../../../interfaces/formats.md#formats).
-
-The `URL` must conform to the structure of a Uniform Resource Locator. The specified URL must point to a server
-that uses HTTP or HTTPS. This does not require any
-additional headers for getting a response from the server.
-
 `INSERT` and `SELECT` queries are transformed to `POST` and `GET` requests,
 respectively. For processing `POST` requests, the remote server must support
 [Chunked transfer encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding).

docs/en/introduction/adopters.md

@@ -67,6 +67,7 @@ toc_title: Adopters
 | <a href="https://geniee.co.jp" class="favicon">Geniee</a> | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) |
 | <a href="https://www.genotek.ru/" class="favicon">Genotek</a> | Bioinformatics | Main product | — | — | [Video, August 2020](https://youtu.be/v3KyZbz9lEE) |
 | <a href="https://gigapipe.com/" class="favicon">Gigapipe</a> | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) |
+| <a href="https://gigasheet.co/" class="favicon">Gigasheet</a> | Analytics | Main product | — | — | Direct Reference, February 2022|
 | <a href="https://glaber.io/" class="favicon">Glaber</a> | Monitoring | Main product | — | — | [Website](https://glaber.io/) |
 | <a href="https://graphcdn.io/" class="favicon">GraphCDN</a> | CDN | Traffic Analytics | — | — | [Blog Post in English, August 2021](https://altinity.com/blog/delivering-insight-on-graphql-apis-with-clickhouse-at-graphcdn/) |
 | <a href="https://www.grouparoo.com" class="favicon">Grouparoo</a> | Data Warehouse Integrations | Main product | — | — | [Official Website, November 2021](https://www.grouparoo.com/integrations) |

docs/en/operations/settings/settings.md

@@ -1803,6 +1803,20 @@ If an INSERTed block is skipped due to deduplication in the source table, there
 At the same time, this behaviour “breaks” `INSERT` idempotency. If an `INSERT` into the main table was successful and `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` allows for changing this behaviour. On retry, a materialized view will receive the repeat insert and will perform a deduplication check by itself,
 ignoring check result for the source table, and will insert rows lost because of the first failure.
 
+## insert_deduplication_token {#insert_deduplication_token}
+
+The setting allows a user to provide own deduplication semantic in MergeTree/ReplicatedMergeTree  
+For example, by providing a unique value for the setting in each INSERT statement,
+user can avoid the same inserted data being deduplicated
+
+Possilbe values:
+
+-  Any string
+
+Default value: empty string (disabled)
+
+`insert_deduplication_token` is used for deduplication _only_ when not empty
+
 ## max_network_bytes {#settings-max-network-bytes}
 
 Limits the data volume (in bytes) that is received or transmitted over the network when executing a query. This setting applies to every individual query.

docs/en/sql-reference/statements/alter/column.md

@@ -197,12 +197,13 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
 
 ## MATERIALIZE COLUMN {#materialize-column}
 
-Materializes the column in the parts where the column is missing. This is useful in case of creating a new column with complicated `DEFAULT` or `MATERIALIZED` expression. Calculation of the column directly on `SELECT` query can cause bigger request execution time, so it is reasonable to use `MATERIALIZE COLUMN` for such columns. To perform same manipulation for existing column, use `FINAL` modifier.
+Materializes or updates a column with an expression for a default value (`DEFAULT` or `MATERIALIZED`).
+It is used if it is necessary to add or update a column with a complicated expression, because evaluating such an expression directly on `SELECT` executing turns out to be expensive. 
 
 Syntax:
 
 ```sql
-ALTER TABLE table MATERIALIZE COLUMN col [FINAL];
+ALTER TABLE table MATERIALIZE COLUMN col;
 ```
 
 **Example**
@@ -211,20 +212,34 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL];
 DROP TABLE IF EXISTS tmp;
 SET mutations_sync = 2;
 CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple();
-INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10;
+INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5;
 ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x);
 
 ALTER TABLE tmp MATERIALIZE COLUMN s;
 
+SELECT groupArray(x), groupArray(s) FROM (select x,s from tmp order by x);
+
+┌─groupArray(x)─┬─groupArray(s)─────────┐
+│ [0,1,2,3,4]   │ ['0','1','2','3','4'] │
+└───────────────┴───────────────────────┘
+
+ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(round(100/x));
+
+INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5,5;
+
 SELECT groupArray(x), groupArray(s) FROM tmp;
-```
 
-**Result:**
+┌─groupArray(x)─────────┬─groupArray(s)──────────────────────────────────┐
+│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','20','17','14','12','11'] │
+└───────────────────────┴────────────────────────────────────────────────┘
 
-```sql
-┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐
-│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │
-└───────────────────────┴───────────────────────────────────────────┘
+ALTER TABLE tmp MATERIALIZE COLUMN s;
+
+SELECT groupArray(x), groupArray(s) FROM tmp;
+
+┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────────────────┐
+│ [0,1,2,3,4,5,6,7,8,9] │ ['inf','100','50','33','25','20','17','14','12','11'] │
+└───────────────────────┴───────────────────────────────────────────────────────┘
 ```
 
 **See Also**

docs/ru/sql-reference/statements/alter/column.md

@@ -197,12 +197,13 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
 
 ## MATERIALIZE COLUMN {#materialize-column}
 
-Материализует столбец таблицы в кусках, в которых отсутствуют значения. Используется, если необходимо создать новый столбец со сложным материализованным выражением или выражением для заполнения по умолчанию (`DEFAULT`), потому как вычисление такого столбца прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным. Чтобы совершить ту же операцию для существующего столбца, используйте модификатор `FINAL`.
+Материализует или обновляет столбец таблицы с выражением для значения по умолчанию (`DEFAULT` или `MATERIALIZED`).
+Используется, если необходимо добавить или обновить столбец со сложным выражением, потому как вычисление такого выражения прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным.
 
 Синтаксис:
 
 ```sql
-ALTER TABLE table MATERIALIZE COLUMN col [FINAL];
+ALTER TABLE table MATERIALIZE COLUMN col;
 ```
 
 **Пример**
@@ -211,21 +212,39 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL];
 DROP TABLE IF EXISTS tmp;
 SET mutations_sync = 2;
 CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple();
-INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10;
+INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5;
 ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x);
 
 ALTER TABLE tmp MATERIALIZE COLUMN s;
 
+SELECT groupArray(x), groupArray(s) FROM (select x,s from tmp order by x);
+
+┌─groupArray(x)─┬─groupArray(s)─────────┐
+│ [0,1,2,3,4]   │ ['0','1','2','3','4'] │
+└───────────────┴───────────────────────┘
+
+ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(round(100/x));
+
+INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5,5;
+
 SELECT groupArray(x), groupArray(s) FROM tmp;
+
+┌─groupArray(x)─────────┬─groupArray(s)──────────────────────────────────┐
+│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','20','17','14','12','11'] │
+└───────────────────────┴────────────────────────────────────────────────┘
+
+ALTER TABLE tmp MATERIALIZE COLUMN s;
+
+SELECT groupArray(x), groupArray(s) FROM tmp;
+
+┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────────────────┐
+│ [0,1,2,3,4,5,6,7,8,9] │ ['inf','100','50','33','25','20','17','14','12','11'] │
+└───────────────────────┴───────────────────────────────────────────────────────┘
 ```
 
-**Результат:**
+**Смотрите также**
 
-```sql
-┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐
-│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │
-└───────────────────────┴───────────────────────────────────────────┘
-```
+- [MATERIALIZED](../../statements/create/table.md#materialized).
 
 ## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter}
 

docs/zh/engines/database-engines/index.md

@@ -14,7 +14,7 @@ toc_title: Introduction
 
 -   [MySQL](../../engines/database-engines/mysql.md)
 
--   [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md)
+-   [MaterializeMySQL](../../engines/database-engines/materialized-mysql.md)
 
 -   [Lazy](../../engines/database-engines/lazy.md)
 

docs/zh/engines/database-engines/materialized-mysql.md

@@ -1 +0,0 @@
-../../../en/engines/database-engines/materialized-mysql.md

docs/zh/engines/database-engines/materialized-mysql.md

@@ -0,0 +1,274 @@
+---
+toc_priority: 29
+toc_title: MaterializedMySQL
+---
+
+# [experimental] MaterializedMySQL {#materialized-mysql}
+
+!!! warning "警告"
+    这是一个实验性的特性，不应该在生产中使用.
+
+
+创建ClickHouse数据库，包含MySQL中所有的表，以及这些表中的所有数据。
+
+ClickHouse服务器作为MySQL副本工作。它读取binlog并执行DDL和DML查询。
+
+## 创建数据库 {#creating-a-database}
+
+``` sql
+CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
+ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
+[TABLE OVERRIDE table1 (...), TABLE OVERRIDE table2 (...)]
+```
+
+**引擎参数**
+
+-   `host:port` — MySQL 服务地址.
+-   `database` — MySQL 数据库名称.
+-   `user` — MySQL 用户名.
+-   `password` — MySQL 用户密码.
+
+**引擎配置**
+
+
+- `max_rows_in_buffer` — 允许在内存中缓存数据的最大行数(对于单个表和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值:`65 505`。
+- `max_bytes_in_buffer` - 允许在内存中缓存数据的最大字节数(对于单个表和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `1 048 576 `。
+- `max_rows_in_buffers` - 允许在内存中缓存数据的最大行数(用于数据库和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `65 505`。
+- `max_bytes_in_buffers` - 允许在内存中缓存数据的最大字节数(用于数据库和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `1 048 576`。
+- `max_flush_data_time ` - 允许数据在内存中缓存的最大毫秒数(对于数据库和无法查询的缓存数据)。当超过这个时间，数据将被物化。默认值: `1000`。
+- `max_wait_time_when_mysql_unavailable` - MySQL不可用时的重试间隔(毫秒)。负值禁用重试。默认值:`1000`。
+— `allows_query_when_mysql_lost `—允许在MySQL丢失时查询物化表。默认值:`0`(`false`)。
+
+```sql
+CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***')
+     SETTINGS
+        allows_query_when_mysql_lost=true,
+        max_wait_time_when_mysql_unavailable=10000;
+```
+
+**MySQL服务器端配置**
+
+为了`MaterializedMySQL`的正确工作，有一些必须设置的`MySQL`端配置设置:
+
+- `default_authentication_plugin = mysql_native_password `，因为 `MaterializedMySQL` 只能授权使用该方法。
+- `gtid_mode = on`，因为基于GTID的日志记录是提供正确的 `MaterializedMySQL`复制的强制要求。
+
+!!! attention "注意"
+    当打开`gtid_mode`时，您还应该指定`enforce_gtid_consistency = on`。
+
+## 虚拟列 {#virtual-columns}
+
+当使用`MaterializeMySQL`数据库引擎时，[ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md)表与虚拟的`_sign`和`_version`列一起使用。
+
+- `_version` — 事务版本. 类型 [UInt64](../../sql-reference/data-types/int-uint.md).
+- `_sign` — 删除标记. 类型 [Int8](../../sql-reference/data-types/int-uint.md). 可能的值:
+    - `1` — 行没有删除,
+    - `-1` — 行已被删除.
+
+## 支持的数据类型 {#data_types-support}
+
+| MySQL                   | ClickHouse                                                   |
+|-------------------------|--------------------------------------------------------------|
+| TINY                    | [Int8](../../sql-reference/data-types/int-uint.md)           |
+| SHORT                   | [Int16](../../sql-reference/data-types/int-uint.md)          |
+| INT24                   | [Int32](../../sql-reference/data-types/int-uint.md)          |
+| LONG                    | [UInt32](../../sql-reference/data-types/int-uint.md)         |
+| LONGLONG                | [UInt64](../../sql-reference/data-types/int-uint.md)         |
+| FLOAT                   | [Float32](../../sql-reference/data-types/float.md)           |
+| DOUBLE                  | [Float64](../../sql-reference/data-types/float.md)           |
+| DECIMAL, NEWDECIMAL     | [Decimal](../../sql-reference/data-types/decimal.md)         |
+| DATE, NEWDATE           | [Date](../../sql-reference/data-types/date.md)               |
+| DATETIME, TIMESTAMP     | [DateTime](../../sql-reference/data-types/datetime.md)       |
+| DATETIME2, TIMESTAMP2   | [DateTime64](../../sql-reference/data-types/datetime64.md)   |
+| YEAR                    | [UInt16](../../sql-reference/data-types/int-uint.md)         |
+| TIME                    | [Int64](../../sql-reference/data-types/int-uint.md)          |
+| ENUM                    | [Enum](../../sql-reference/data-types/enum.md)               |
+| STRING                  | [String](../../sql-reference/data-types/string.md)           |
+| VARCHAR, VAR_STRING     | [String](../../sql-reference/data-types/string.md)           |
+| BLOB                    | [String](../../sql-reference/data-types/string.md)           |
+| GEOMETRY                | [String](../../sql-reference/data-types/string.md)           |
+| BINARY                  | [FixedString](../../sql-reference/data-types/fixedstring.md) |
+| BIT                     | [UInt64](../../sql-reference/data-types/int-uint.md)         |
+| SET                     | [UInt64](../../sql-reference/data-types/int-uint.md)         |
+
+[Nullable](../../sql-reference/data-types/nullable.md) 已经被支持.
+
+MySQL中的Time 类型，会被ClickHouse转换成微秒来存储
+
+不支持其他类型。如果MySQL表包含此类类型的列，ClickHouse抛出异常"Unhandled data type"并停止复制。
+
+## 规范和推荐用法 {#specifics-and-recommendations}
+
+### 兼容性限制 {#compatibility-restrictions}
+
+除了数据类型的限制之外，还有一些限制与`MySQL`数据库相比有所不同，这应该在复制之前解决:
+
+- `MySQL` 中的每个表都应该包含 `PRIMARY KEY`。
+- 对于表的复制，那些包含 `ENUM` 字段值超出范围的行(在 `ENUM` 签名中指定)将不起作用。
+
+### DDL Queries {#ddl-queries}
+
+MySQL DDL 语句会被转换成对应的ClickHouse DDL 语句，比如： ([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create/index.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md)). 如果ClickHouse 无法解析某些语句DDL 操作，则会跳过。
+
+
+### 数据复制 {#data-replication}
+
+MaterializedMySQL不支持直接的 `INSERT`， `DELETE` 和 `UPDATE` 查询。然而，它们在数据复制方面得到了支持:
+
+- MySQL `INSERT`查询被转换为`_sign=1`的INSERT查询。
+- MySQL `DELETE`查询被转换为`INSERT`，并且`_sign=-1`。
+- 如果主键被修改了，MySQL的 `UPDATE` 查询将被转换为 `INSERT` 带 `_sign=1`  和INSERT 带有_sign=-1;如果主键没有被修改，则转换为`INSERT`和`_sign=1`。
+
+###  MaterializedMySQL 数据表查询 {#select}
+
+`SELECT` 查询从 `MaterializedMySQL`表有一些细节:
+
+ - 如果在SELECT查询中没有指定`_version`，则 [FINAL](../../sql-reference/statements/select/from.md#select-from- FINAL)修饰符被使用，所以只有带有 `MAX(_version)`的行会返回每个主键值。
+
+ - 如果在SELECT查询中没有指定 `_sign`，则默认使用 `WHERE _sign=1 `。所以被删除的行不是
+包含在结果集中。
+
+ - 结果包括列注释，以防MySQL数据库表中存在这些列注释。
+
+### 索引转换 {#index-conversion}
+
+在ClickHouse表中，MySQL的 `PRIMARY KEY` 和 `INDEX` 子句被转换为 `ORDER BY` 元组。
+
+ClickHouse只有一个物理排序，由 `order by` 条件决定。要创建一个新的物理排序，请使用[materialized views](../../sql-reference/statements/create/view.md#materialized)。
+
+**注意**
+
+- `_sign=-1` 的行不会被物理地从表中删除。
+- 级联 `UPDATE/DELETE` 查询不支持 `MaterializedMySQL` 引擎，因为他们在 MySQL binlog中不可见的
+— 复制很容易被破坏。
+— 禁止对数据库和表进行手工操作。
+- `MaterializedMySQL` 受[optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert)设置的影响。当MySQL服务器中的一个表发生变化时，数据会合并到 `MaterializedMySQL` 数据库中相应的表中。
+
+### 表重写 {#table-overrides}
+
+表覆盖可用于自定义ClickHouse DDL查询，从而允许您对应用程序进行模式优化。这对于控制分区特别有用，分区对MaterializedMySQL的整体性能非常重要。
+
+这些是你可以对MaterializedMySQL表重写的模式转换操作:
+
+ * 修改列类型。必须与原始类型兼容，否则复制将失败。例如，可以将`UInt32`列修改为`UInt64`，不能将 `String` 列修改为 `Array(String)`。
+ * 修改 [column TTL](../table-engines/mergetree-family/mergetree/#mergetree-column-ttl).
+ * 修改 [column compression codec](../../sql-reference/statements/create/table/#codecs).
+ * 增加 [ALIAS columns](../../sql-reference/statements/create/table/#alias).
+ * 增加 [skipping indexes](../table-engines/mergetree-family/mergetree/#table_engine-mergetree-data_skipping-indexes)
+ * 增加 [projections](../table-engines/mergetree-family/mergetree/#projections). 
+ 请注意，当使用 `SELECT ... FINAL ` (MaterializedMySQL默认是这样做的) 时，预测优化是被禁用的，所以这里是受限的， `INDEX ... TYPE hypothesis `[在v21.12的博客文章中描述]](https://clickhouse.com/blog/en/2021/clickhouse-v21.12-released/)可能在这种情况下更有用。
+ * 修改 [PARTITION BY](../table-engines/mergetree-family/custom-partitioning-key/)
+ * 修改 [ORDER BY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
+ * 修改 [PRIMARY KEY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
+ * 增加 [SAMPLE BY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
+ * 增加 [table TTL](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
+
+```sql
+CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
+[SETTINGS ...]
+[TABLE OVERRIDE table_name (
+    [COLUMNS (
+        [col_name [datatype] [ALIAS expr] [CODEC(...)] [TTL expr], ...]
+        [INDEX index_name expr TYPE indextype[(...)] GRANULARITY val, ...]
+        [PROJECTION projection_name (SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]), ...]
+    )]
+    [ORDER BY expr]
+    [PRIMARY KEY expr]
+    [PARTITION BY expr]
+    [SAMPLE BY expr]
+    [TTL expr]
+), ...]
+```
+
+示例:
+
+```sql
+CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
+TABLE OVERRIDE table1 (
+    COLUMNS (
+        userid UUID,
+        category LowCardinality(String),
+        timestamp DateTime CODEC(Delta, Default)
+    )
+    PARTITION BY toYear(timestamp)
+),
+TABLE OVERRIDE table2 (
+    COLUMNS (
+        client_ip String TTL created + INTERVAL 72 HOUR
+    )
+    SAMPLE BY ip_hash
+)
+```
+
+
+`COLUMNS`列表是稀疏的;根据指定修改现有列，添加额外的ALIAS列。不可能添加普通列或实体化列。具有不同类型的已修改列必须可从原始类型赋值。在执行`CREATE DATABASE` 查询时，目前还没有验证这个或类似的问题，因此需要格外小心。
+
+您可以为还不存在的表指定重写。
+
+!!! warning "警告"
+    如果使用时不小心，很容易用表重写中断复制。例如:
+
+    * 如果一个ALIAS列被添加了一个表覆盖，并且一个具有相同名称的列后来被添加到源MySQL表，在ClickHouse中转换后的ALTER table查询将失败并停止复制。
+    * 目前可以添加引用可空列的覆盖，而非空列是必需的，例如 `ORDER BY` 或 `PARTITION BY`。这将导致CREATE TABLE查询失败，也会导致复制停止。
+
+## 使用示例 {#examples-of-use}
+
+ MySQL 查询语句:
+
+``` sql
+mysql> CREATE DATABASE db;
+mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT);
+mysql> INSERT INTO db.test VALUES (1, 11), (2, 22);
+mysql> DELETE FROM db.test WHERE a=1;
+mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16);
+mysql> UPDATE db.test SET c='Wow!', b=222;
+mysql> SELECT * FROM test;
+```
+
+```text
+┌─a─┬───b─┬─c────┐
+│ 2 │ 222 │ Wow! │
+└───┴─────┴──────┘
+```
+
+ClickHouse中的数据库，与MySQL服务器交换数据:
+
+创建的数据库和表:
+
+``` sql
+CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***');
+SHOW TABLES FROM mysql;
+```
+
+``` text
+┌─name─┐
+│ test │
+└──────┘
+```
+
+数据插入之后：
+
+``` sql
+SELECT * FROM mysql.test;
+```
+
+``` text
+┌─a─┬──b─┐
+│ 1 │ 11 │
+│ 2 │ 22 │
+└───┴────┘
+```
+
+删除数据后，添加列并更新:
+
+``` sql
+SELECT * FROM mysql.test;
+```
+
+``` text
+┌─a─┬───b─┬─c────┐
+│ 2 │ 222 │ Wow! │
+└───┴─────┴──────┘
+```
+
+[来源文章](https://clickhouse.com/docs/en/engines/database-engines/materialized-mysql/) <!--hide-->

programs/local/LocalServer.cpp

@@ -313,9 +313,15 @@ void LocalServer::cleanup()
 }
 
 
+static bool checkIfStdinIsRegularFile()
+{
+    struct stat file_stat;
+    return fstat(STDIN_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode);
+}
+
 std::string LocalServer::getInitialCreateTableQuery()
 {
-    if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format"))
+    if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!checkIfStdinIsRegularFile() || !config().has("query")))
         return {};
 
     auto table_name = backQuoteIfNeed(config().getString("table-name", "table"));
@@ -337,8 +343,9 @@ std::string LocalServer::getInitialCreateTableQuery()
         format_from_file_name = FormatFactory::instance().getFormatFromFileName(file_name, false);
     }
 
-    auto data_format
-        = backQuoteIfNeed(config().getString("table-data-format", format_from_file_name.empty() ? "TSV" : format_from_file_name));
+    auto data_format = backQuoteIfNeed(
+        config().getString("table-data-format", config().getString("format", format_from_file_name.empty() ? "TSV" : format_from_file_name)));
+
 
     if (table_structure == "auto")
         table_structure = "";
@@ -518,22 +525,17 @@ void LocalServer::processConfig()
 
         if (config().has("multiquery"))
             is_multiquery = true;
-
-        load_suggestions = true;
     }
     else
     {
-        if (delayed_interactive)
-        {
-            load_suggestions = true;
-        }
-
         need_render_progress = config().getBool("progress", false);
         echo_queries = config().hasOption("echo") || config().hasOption("verbose");
         ignore_error = config().getBool("ignore-error", false);
         is_multiquery = true;
     }
+
     print_stack_trace = config().getBool("stacktrace", false);
+    load_suggestions = (is_interactive || delayed_interactive) && !config().getBool("disable_suggestion", false);
 
     auto logging = (config().has("logger.console")
                     || config().has("logger.level")

programs/server/Server.cpp

@@ -22,6 +22,7 @@
 #include <base/getMemoryAmount.h>
 #include <base/errnoToString.h>
 #include <base/coverage.h>
+#include <Common/MemoryTracker.h>
 #include <Common/ClickHouseRevision.h>
 #include <Common/DNSResolver.h>
 #include <Common/CurrentMetrics.h>
@@ -955,6 +956,14 @@ if (ThreadFuzzer::instance().isEffective())
             total_memory_tracker.setDescription("(total)");
             total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
 
+            auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker();
+            if (config->has("global_memory_usage_overcommit_max_wait_microseconds"))
+            {
+                UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 0);
+                global_overcommit_tracker->setMaxWaitTime(max_overcommit_wait_time);
+            }
+            total_memory_tracker.setOvercommitTracker(global_overcommit_tracker);
+
             // FIXME logging-related things need synchronization -- see the 'Logger * log' saved
             // in a lot of places. For now, disable updating log configuration without server restart.
             //setTextLog(global_context->getTextLog());

programs/static-files-disk-uploader/static-files-disk-uploader.cpp

@@ -31,7 +31,7 @@ namespace ErrorCodes
  * If test-mode option is added, files will be put by given url via PUT request.
  */
 
-void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, WriteBuffer & metadata_buf)
+void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, bool link, WriteBuffer & metadata_buf)
 {
     String remote_path;
     RE2::FullMatch(file_path.string(), EXTRACT_PATH_PATTERN, &remote_path);
@@ -52,22 +52,29 @@ void processFile(const fs::path & file_path, const fs::path & dst_path, bool tes
 
     auto dst_file_path = fs::path(dst_path) / remote_path;
 
-    auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
-    std::shared_ptr<WriteBuffer> dst_buf;
-
-    /// test mode for integration tests.
-    if (test_mode)
-        dst_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT);
+    if (link)
+    {
+        fs::create_symlink(file_path, dst_file_path);
+    }
     else
-        dst_buf = std::make_shared<WriteBufferFromFile>(dst_file_path);
+    {
+        auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
+        std::shared_ptr<WriteBuffer> dst_buf;
 
-    copyData(*src_buf, *dst_buf);
-    dst_buf->next();
-    dst_buf->finalize();
+        /// test mode for integration tests.
+        if (test_mode)
+            dst_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT);
+        else
+            dst_buf = std::make_shared<WriteBufferFromFile>(dst_file_path);
+
+        copyData(*src_buf, *dst_buf);
+        dst_buf->next();
+        dst_buf->finalize();
+    }
 };
 
 
-void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode)
+void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode, bool link)
 {
     std::cerr << "Data path: " << data_path << ", destination path: " << dst_path << std::endl;
 
@@ -94,7 +101,7 @@ void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_
     {
         if (dir_it->is_directory())
         {
-            processFile(dir_it->path(), dst_path, test_mode, *root_meta);
+            processFile(dir_it->path(), dst_path, test_mode, link, *root_meta);
 
             String directory_prefix;
             RE2::FullMatch(dir_it->path().string(), EXTRACT_PATH_PATTERN, &directory_prefix);
@@ -115,14 +122,14 @@ void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_
 
             fs::directory_iterator files_end;
             for (fs::directory_iterator file_it(dir_it->path()); file_it != files_end; ++file_it)
-                processFile(file_it->path(), dst_path, test_mode, *directory_meta);
+                processFile(file_it->path(), dst_path, test_mode, link, *directory_meta);
 
             directory_meta->next();
             directory_meta->finalize();
         }
         else
         {
-            processFile(dir_it->path(), dst_path, test_mode, *root_meta);
+            processFile(dir_it->path(), dst_path, test_mode, link, *root_meta);
         }
     }
     root_meta->next();
@@ -141,6 +148,7 @@ try
         ("help,h", "produce help message")
         ("metadata-path", po::value<std::string>(), "Metadata path (select data_paths from system.tables where name='table_name'")
         ("test-mode", "Use test mode, which will put data on given url via PUT")
+        ("link", "Create symlinks instead of copying")
         ("url", po::value<std::string>(), "Web server url for test mode")
         ("output-dir", po::value<std::string>(), "Directory to put files in non-test mode");
 
@@ -186,7 +194,7 @@ try
             root_path = fs::current_path();
     }
 
-    processTableFiles(fs_path, root_path, test_mode);
+    processTableFiles(fs_path, root_path, test_mode, options.count("link"));
 
     return 0;
 }

src/Access/Common/AccessType.h

@@ -113,9 +113,9 @@ enum class AccessType
     M(ALTER_ROLE, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(DROP_ROLE, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(ROLE_ADMIN, "", GLOBAL, ACCESS_MANAGEMENT) /* allows to grant and revoke the roles which are not granted to the current user with admin option */\
-    M(CREATE_ROW_POLICY, "CREATE POLICY", GLOBAL, ACCESS_MANAGEMENT) \
-    M(ALTER_ROW_POLICY, "ALTER POLICY", GLOBAL, ACCESS_MANAGEMENT) \
-    M(DROP_ROW_POLICY, "DROP POLICY", GLOBAL, ACCESS_MANAGEMENT) \
+    M(CREATE_ROW_POLICY, "CREATE POLICY", TABLE, ACCESS_MANAGEMENT) \
+    M(ALTER_ROW_POLICY, "ALTER POLICY", TABLE, ACCESS_MANAGEMENT) \
+    M(DROP_ROW_POLICY, "DROP POLICY", TABLE, ACCESS_MANAGEMENT) \
     M(CREATE_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(ALTER_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(DROP_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \
@@ -124,7 +124,7 @@ enum class AccessType
     M(DROP_SETTINGS_PROFILE, "DROP PROFILE", GLOBAL, ACCESS_MANAGEMENT) \
     M(SHOW_USERS, "SHOW CREATE USER", GLOBAL, SHOW_ACCESS) \
     M(SHOW_ROLES, "SHOW CREATE ROLE", GLOBAL, SHOW_ACCESS) \
-    M(SHOW_ROW_POLICIES, "SHOW POLICIES, SHOW CREATE ROW POLICY, SHOW CREATE POLICY", GLOBAL, SHOW_ACCESS) \
+    M(SHOW_ROW_POLICIES, "SHOW POLICIES, SHOW CREATE ROW POLICY, SHOW CREATE POLICY", TABLE, SHOW_ACCESS) \
     M(SHOW_QUOTAS, "SHOW CREATE QUOTA", GLOBAL, SHOW_ACCESS) \
     M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \
     M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \

src/Access/tests/gtest_access_rights_ops.cpp

@@ -45,7 +45,15 @@ TEST(AccessRights, Union)
     lhs.grant(AccessType::INSERT);
     rhs.grant(AccessType::ALL, "db1");
     lhs.makeUnion(rhs);
-    ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
+    ASSERT_EQ(lhs.toString(),
+              "GRANT INSERT ON *.*, "
+              "GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, "
+              "CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, "
+              "TRUNCATE, OPTIMIZE, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, "
+              "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
+              "SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
+              "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
+              "SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
 }
 
 

src/Common/ColumnsHashing.h

@@ -387,47 +387,52 @@ struct HashMethodSingleLowCardinalityColumn : public SingleColumnMethod
     }
 
     template <typename Data>
-    ALWAYS_INLINE FindResult findFromRow(Data & data, size_t row_, Arena & pool)
+    ALWAYS_INLINE FindResult findKey(Data & data, size_t row_, Arena & pool)
     {
         size_t row = getIndexAt(row_);
 
         if (is_nullable && row == 0)
         {
             if constexpr (has_mapped)
-                return FindResult(data.hasNullKeyData() ? &data.getNullKeyData() : nullptr, data.hasNullKeyData());
+                return FindResult(data.hasNullKeyData() ? &data.getNullKeyData() : nullptr, data.hasNullKeyData(), 0);
             else
-                return FindResult(data.hasNullKeyData());
+                return FindResult(data.hasNullKeyData(), 0);
         }
 
         if (visit_cache[row] != VisitValue::Empty)
         {
             if constexpr (has_mapped)
-                return FindResult(&mapped_cache[row], visit_cache[row] == VisitValue::Found);
+                return FindResult(&mapped_cache[row], visit_cache[row] == VisitValue::Found, 0);
             else
-                return FindResult(visit_cache[row] == VisitValue::Found);
+                return FindResult(visit_cache[row] == VisitValue::Found, 0);
         }
 
         auto key_holder = getKeyHolder(row_, pool);
 
-        typename Data::iterator it;
+        typename Data::LookupResult it;
         if (saved_hash)
-            it = data.find(*key_holder, saved_hash[row]);
+            it = data.find(keyHolderGetKey(key_holder), saved_hash[row]);
         else
-            it = data.find(*key_holder);
+            it = data.find(keyHolderGetKey(key_holder));
 
-        bool found = it != data.end();
+        bool found = it;
         visit_cache[row] = found ? VisitValue::Found : VisitValue::NotFound;
 
         if constexpr (has_mapped)
         {
             if (found)
-                mapped_cache[row] = it->second;
+                mapped_cache[row] = it->getMapped();
         }
 
+        size_t offset = 0;
+
+        if constexpr (FindResult::has_offset)
+            offset = found ? data.offsetInternal(it) : 0;
+
         if constexpr (has_mapped)
-            return FindResult(&mapped_cache[row], found);
+            return FindResult(&mapped_cache[row], found, offset);
         else
-            return FindResult(found);
+            return FindResult(found, offset);
     }
 
     template <typename Data>

src/Common/MemoryTracker.cpp

@@ -1,6 +1,7 @@
 #include "MemoryTracker.h"
 
 #include <IO/WriteHelpers.h>
+#include <Common/VariableContext.h>
 #include <Interpreters/TraceCollector.h>
 #include <Common/Exception.h>
 #include <Common/LockMemoryExceptionInThread.h>
@@ -8,6 +9,7 @@
 #include <Common/formatReadable.h>
 #include <Common/ProfileEvents.h>
 #include <Common/thread_local_rng.h>
+#include <Common/OvercommitTracker.h>
 #include <base/logger_useful.h>
 
 #include <atomic>
@@ -95,7 +97,7 @@ void MemoryTracker::logMemoryUsage(Int64 current) const
 }
 
 
-void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
+void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryTracker * query_tracker)
 {
     if (size < 0)
         throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Negative size ({}) is passed to MemoryTracker. It is a bug.", size);
@@ -104,7 +106,8 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
     {
         /// Since the MemoryTrackerBlockerInThread should respect the level, we should go to the next parent.
         if (auto * loaded_next = parent.load(std::memory_order_relaxed))
-            loaded_next->allocImpl(size, throw_if_memory_exceeded);
+            loaded_next->allocImpl(size, throw_if_memory_exceeded,
+                level == VariableContext::Process ? this : query_tracker);
         return;
     }
 
@@ -186,18 +189,30 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
 
     if (unlikely(current_hard_limit && will_be > current_hard_limit) && memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded)
     {
-        /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
-        MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
-        ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
-        const auto * description = description_ptr.load(std::memory_order_relaxed);
-        throw DB::Exception(
-            DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,
-            "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}",
-            description ? " " : "",
-            description ? description : "",
-            formatReadableSizeWithBinarySuffix(will_be),
-            size,
-            formatReadableSizeWithBinarySuffix(current_hard_limit));
+        bool need_to_throw = true;
+        bool try_to_free_memory = overcommit_tracker != nullptr && query_tracker != nullptr;
+        if (try_to_free_memory)
+            need_to_throw = overcommit_tracker->needToStopQuery(query_tracker);
+
+        if (need_to_throw)
+        {
+            /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
+            MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
+            ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
+            const auto * description = description_ptr.load(std::memory_order_relaxed);
+            throw DB::Exception(
+                DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,
+                "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}",
+                description ? " " : "",
+                description ? description : "",
+                formatReadableSizeWithBinarySuffix(will_be),
+                size,
+                formatReadableSizeWithBinarySuffix(current_hard_limit));
+        }
+        else
+        {
+            will_be = amount.load(std::memory_order_relaxed);
+        }
     }
 
     bool peak_updated;
@@ -221,7 +236,8 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
     }
 
     if (auto * loaded_next = parent.load(std::memory_order_relaxed))
-        loaded_next->allocImpl(size, throw_if_memory_exceeded);
+        loaded_next->allocImpl(size, throw_if_memory_exceeded,
+            level == VariableContext::Process ? this : query_tracker);
 }
 
 void MemoryTracker::alloc(Int64 size)
@@ -302,10 +318,23 @@ void MemoryTracker::free(Int64 size)
 }
 
 
+OvercommitRatio MemoryTracker::getOvercommitRatio()
+{
+    return { amount.load(std::memory_order_relaxed), soft_limit.load(std::memory_order_relaxed) };
+}
+
+
+OvercommitRatio MemoryTracker::getOvercommitRatio(Int64 limit)
+{
+    return { amount.load(std::memory_order_relaxed), limit };
+}
+
+
 void MemoryTracker::resetCounters()
 {
     amount.store(0, std::memory_order_relaxed);
     peak.store(0, std::memory_order_relaxed);
+    soft_limit.store(0, std::memory_order_relaxed);
     hard_limit.store(0, std::memory_order_relaxed);
     profiler_limit.store(0, std::memory_order_relaxed);
 }
@@ -330,6 +359,12 @@ void MemoryTracker::set(Int64 to)
 }
 
 
+void MemoryTracker::setSoftLimit(Int64 value)
+{
+    soft_limit.store(value, std::memory_order_relaxed);
+}
+
+
 void MemoryTracker::setHardLimit(Int64 value)
 {
     hard_limit.store(value, std::memory_order_relaxed);

src/Common/MemoryTracker.h

@@ -28,6 +28,9 @@ extern thread_local bool memory_tracker_always_throw_logical_error_on_allocation
 #define ALLOW_ALLOCATIONS_IN_SCOPE static_assert(true)
 #endif
 
+struct OvercommitRatio;
+struct OvercommitTracker;
+
 /** Tracks memory consumption.
   * It throws an exception if amount of consumed memory become greater than certain limit.
   * The same memory tracker could be simultaneously used in different threads.
@@ -40,6 +43,7 @@ class MemoryTracker
 private:
     std::atomic<Int64> amount {0};
     std::atomic<Int64> peak {0};
+    std::atomic<Int64> soft_limit {0};
     std::atomic<Int64> hard_limit {0};
     std::atomic<Int64> profiler_limit {0};
 
@@ -61,6 +65,8 @@ private:
     /// This description will be used as prefix into log messages (if isn't nullptr)
     std::atomic<const char *> description_ptr = nullptr;
 
+    OvercommitTracker * overcommit_tracker = nullptr;
+
     bool updatePeak(Int64 will_be, bool log_memory_usage);
     void logMemoryUsage(Int64 current) const;
 
@@ -83,7 +89,7 @@ public:
 
     void allocNoThrow(Int64 size);
 
-    void allocImpl(Int64 size, bool throw_if_memory_exceeded);
+    void allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryTracker * query_tracker = nullptr);
 
     void realloc(Int64 old_size, Int64 new_size)
     {
@@ -108,8 +114,14 @@ public:
         return peak.load(std::memory_order_relaxed);
     }
 
+    void setSoftLimit(Int64 value);
     void setHardLimit(Int64 value);
 
+    Int64 getSoftLimit() const
+    {
+        return soft_limit.load(std::memory_order_relaxed);
+    }
+
     /** Set limit if it was not set.
       * Otherwise, set limit to new value, if new value is greater than previous limit.
       */
@@ -159,6 +171,14 @@ public:
         description_ptr.store(description, std::memory_order_relaxed);
     }
 
+    OvercommitRatio getOvercommitRatio();
+    OvercommitRatio getOvercommitRatio(Int64 limit);
+
+    void setOvercommitTracker(OvercommitTracker * tracker) noexcept
+    {
+        overcommit_tracker = tracker;
+    }
+
     /// Reset the accumulated data
     void resetCounters();
 

src/Common/OvercommitTracker.cpp

@@ -0,0 +1,119 @@
+#include "OvercommitTracker.h"
+
+#include <chrono>
+#include <mutex>
+#include <Interpreters/ProcessList.h>
+
+using namespace std::chrono_literals;
+
+OvercommitTracker::OvercommitTracker()
+    : max_wait_time(0us)
+    , picked_tracker(nullptr)
+    , cancelation_state(QueryCancelationState::NONE)
+{}
+
+void OvercommitTracker::setMaxWaitTime(UInt64 wait_time)
+{
+    std::lock_guard guard(overcommit_m);
+    max_wait_time = wait_time * 1us;
+}
+
+bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker)
+{
+    std::unique_lock<std::mutex> lk(overcommit_m);
+
+    pickQueryToExclude();
+    assert(cancelation_state == QueryCancelationState::RUNNING);
+
+    // If no query was chosen we need to stop current query.
+    // This may happen if no soft limit is set.
+    if (picked_tracker == nullptr)
+    {
+        cancelation_state = QueryCancelationState::NONE;
+        return true;
+    }
+    if (picked_tracker == tracker)
+        return true;
+    return !cv.wait_for(lk, max_wait_time, [this]()
+    {
+        return cancelation_state == QueryCancelationState::NONE;
+    });
+}
+
+void OvercommitTracker::unsubscribe(MemoryTracker * tracker)
+{
+    std::unique_lock<std::mutex> lk(overcommit_m);
+    if (picked_tracker == tracker)
+    {
+        LOG_DEBUG(getLogger(), "Picked query stopped");
+
+        picked_tracker = nullptr;
+        cancelation_state = QueryCancelationState::NONE;
+        cv.notify_all();
+    }
+}
+
+UserOvercommitTracker::UserOvercommitTracker(DB::ProcessListForUser * user_process_list_)
+    : user_process_list(user_process_list_)
+{}
+
+void UserOvercommitTracker::pickQueryToExcludeImpl()
+{
+    MemoryTracker * query_tracker = nullptr;
+    OvercommitRatio current_ratio{0, 0};
+    // At this moment query list must be read only.
+    // BlockQueryIfMemoryLimit is used in ProcessList to guarantee this.
+    auto & queries = user_process_list->queries;
+    LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", queries.size());
+    for (auto const & query : queries)
+    {
+        if (query.second->isKilled())
+            continue;
+
+        auto * memory_tracker = query.second->getMemoryTracker();
+        if (!memory_tracker)
+            continue;
+
+        auto ratio = memory_tracker->getOvercommitRatio();
+        LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit);
+        if (ratio.soft_limit != 0 && current_ratio < ratio)
+        {
+            query_tracker = memory_tracker;
+            current_ratio   = ratio;
+        }
+    }
+    LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}",
+        current_ratio.committed, current_ratio.soft_limit);
+    picked_tracker = query_tracker;
+}
+
+void GlobalOvercommitTracker::pickQueryToExcludeImpl()
+{
+    MemoryTracker * query_tracker = nullptr;
+    OvercommitRatio current_ratio{0, 0};
+    process_list->processEachQueryStatus([&](DB::QueryStatus const & query)
+    {
+        if (query.isKilled())
+            return;
+
+        Int64 user_soft_limit = 0;
+        if (auto const * user_process_list = query.getUserProcessList())
+            user_soft_limit = user_process_list->user_memory_tracker.getSoftLimit();
+        if (user_soft_limit == 0)
+            return;
+
+        auto * memory_tracker = query.getMemoryTracker();
+        if (!memory_tracker)
+            return;
+        auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit);
+        LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit);
+        if (current_ratio < ratio)
+        {
+            query_tracker = memory_tracker;
+            current_ratio   = ratio;
+        }
+    });
+    LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}",
+        current_ratio.committed, current_ratio.soft_limit);
+    picked_tracker = query_tracker;
+}

src/Common/OvercommitTracker.h

@@ -0,0 +1,155 @@
+#pragma once
+
+#include <base/logger_useful.h>
+#include <base/types.h>
+#include <boost/core/noncopyable.hpp>
+#include <Poco/Logger.h>
+#include <cassert>
+#include <chrono>
+#include <condition_variable>
+#include <mutex>
+#include <unordered_map>
+
+// This struct is used for the comparison of query memory usage.
+struct OvercommitRatio
+{
+    OvercommitRatio(Int64 committed_, Int64 soft_limit_)
+        : committed(committed_)
+        , soft_limit(soft_limit_)
+    {}
+
+    friend bool operator<(OvercommitRatio const & lhs, OvercommitRatio const & rhs) noexcept
+    {
+        // (a / b < c / d) <=> (a * d < c * b)
+        return (lhs.committed * rhs.soft_limit) < (rhs.committed * lhs.soft_limit)
+            || (lhs.soft_limit == 0 && rhs.soft_limit > 0)
+            || (lhs.committed == 0 && rhs.committed == 0 && lhs.soft_limit > rhs.soft_limit);
+    }
+
+    // actual query memory usage
+    Int64 committed;
+    // guaranteed amount of memory query can use
+    Int64 soft_limit;
+};
+
+class MemoryTracker;
+
+// Usually it's hard to set some reasonable hard memory limit
+// (especially, the default value). This class introduces new
+// mechanisim for the limiting of memory usage.
+// Soft limit represents guaranteed amount of memory query/user
+// may use. It's allowed to exceed this limit. But if hard limit
+// is reached, query with the biggest overcommit ratio
+// is killed to free memory.
+struct OvercommitTracker : boost::noncopyable
+{
+    OvercommitTracker();
+
+    void setMaxWaitTime(UInt64 wait_time);
+
+    bool needToStopQuery(MemoryTracker * tracker);
+
+    void unsubscribe(MemoryTracker * tracker);
+
+    virtual ~OvercommitTracker() = default;
+
+protected:
+    virtual void pickQueryToExcludeImpl() = 0;
+
+    mutable std::mutex overcommit_m;
+    mutable std::condition_variable cv;
+
+    std::chrono::microseconds max_wait_time;
+
+    enum class QueryCancelationState
+    {
+        NONE,
+        RUNNING,
+    };
+
+    // Specifies memory tracker of the chosen to stop query.
+    // If soft limit is not set, all the queries which reach hard limit must stop.
+    // This case is represented as picked tracker pointer is set to nullptr and
+    // overcommit tracker is in RUNNING state.
+    MemoryTracker * picked_tracker;
+    QueryCancelationState cancelation_state;
+
+    virtual Poco::Logger * getLogger() = 0;
+
+private:
+
+    void pickQueryToExclude()
+    {
+        if (cancelation_state != QueryCancelationState::RUNNING)
+        {
+            pickQueryToExcludeImpl();
+            cancelation_state = QueryCancelationState::RUNNING;
+        }
+    }
+
+    friend struct BlockQueryIfMemoryLimit;
+};
+
+namespace DB
+{
+    class ProcessList;
+    struct ProcessListForUser;
+}
+
+struct UserOvercommitTracker : OvercommitTracker
+{
+    explicit UserOvercommitTracker(DB::ProcessListForUser * user_process_list_);
+
+    ~UserOvercommitTracker() override = default;
+
+protected:
+    void pickQueryToExcludeImpl() override final;
+
+    Poco::Logger * getLogger() override final { return logger; }
+private:
+    DB::ProcessListForUser * user_process_list;
+    Poco::Logger * logger = &Poco::Logger::get("UserOvercommitTracker");
+};
+
+struct GlobalOvercommitTracker : OvercommitTracker
+{
+    explicit GlobalOvercommitTracker(DB::ProcessList * process_list_)
+        : process_list(process_list_)
+    {}
+
+    ~GlobalOvercommitTracker() override = default;
+
+protected:
+    void pickQueryToExcludeImpl() override final;
+
+    Poco::Logger * getLogger() override final { return logger; }
+private:
+    DB::ProcessList * process_list;
+    Poco::Logger * logger = &Poco::Logger::get("GlobalOvercommitTracker");
+};
+
+// UserOvercommitTracker requires to check the whole list of user's queries
+// to pick one to stop. BlockQueryIfMemoryLimit struct allows to wait until
+// query selection is finished. It's used in ProcessList to make user query
+// list immutable when UserOvercommitTracker reads it.
+struct BlockQueryIfMemoryLimit
+{
+    BlockQueryIfMemoryLimit(OvercommitTracker const & overcommit_tracker)
+        : mutex(overcommit_tracker.overcommit_m)
+        , lk(mutex)
+    {
+        if (overcommit_tracker.cancelation_state == OvercommitTracker::QueryCancelationState::RUNNING)
+        {
+            overcommit_tracker.cv.wait_for(lk, overcommit_tracker.max_wait_time, [&overcommit_tracker]()
+            {
+                return overcommit_tracker.cancelation_state == OvercommitTracker::QueryCancelationState::NONE;
+            });
+        }
+    }
+
+    ~BlockQueryIfMemoryLimit() = default;
+
+private:
+    std::mutex & mutex;
+    std::unique_lock<std::mutex> lk;
+};

src/Coordination/KeeperServer.cpp

@@ -286,10 +286,7 @@ RaftAppendResult KeeperServer::putRequestBatch(const KeeperStorage::RequestsForS
     for (const auto & [session_id, request] : requests_for_sessions)
         entries.push_back(getZooKeeperLogEntry(session_id, request));
 
-    {
-        std::lock_guard lock(append_entries_mutex);
-        return raft_instance->append_entries(entries);
-    }
+    return raft_instance->append_entries(entries);
 }
 
 bool KeeperServer::isLeader() const

src/Coordination/KeeperServer.h

@@ -28,8 +28,6 @@ private:
     nuraft::ptr<nuraft::asio_service> asio_service;
     nuraft::ptr<nuraft::rpc_listener> asio_listener;
 
-    std::mutex append_entries_mutex;
-
     std::mutex initialized_mutex;
     std::atomic<bool> initialized_flag = false;
     std::condition_variable initialized_cv;

src/Core/Settings.h

@@ -358,11 +358,15 @@ class IColumn;
     M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
     \
     M(UInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \
+    M(UInt64, max_guaranteed_memory_usage, 0, "Maximum guaranteed memory usage for processing of single query. It represents soft limit. Zero means unlimited.", 0) \
     M(UInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \
+    M(UInt64, max_guaranteed_memory_usage_for_user, 0, "Maximum guaranteed memory usage for processing all concurrently running queries for the user. It represents soft limit. Zero means unlimited.", 0) \
     M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \
     M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \
     M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
     \
+    M(UInt64, memory_usage_overcommit_max_wait_microseconds, 0, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown", 0) \
+    \
     M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \
     M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \
     M(UInt64, max_network_bandwidth_for_user, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running user queries. Zero means unlimited.", 0)\
@@ -547,7 +551,7 @@ class IColumn;
     M(Int64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \
     M(Int64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \
     \
-    M(UInt64, http_max_tries, 1, "Max attempts to read via http.", 0) \
+    M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \
     M(UInt64, http_retry_initial_backoff_ms, 100, "Min milliseconds for backoff, when retrying read via http", 0) \
     M(UInt64, http_retry_max_backoff_ms, 10000, "Max milliseconds for backoff, when retrying read via http", 0) \
     \

src/Disks/IStoragePolicy.cpp

@@ -0,0 +1,34 @@
+#include <Disks/IStoragePolicy.h>
+#include <Common/quoteString.h>
+#include <Common/Exception.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int UNKNOWN_VOLUME;
+    extern const int UNKNOWN_DISK;
+}
+
+DiskPtr IStoragePolicy::getDiskByName(const String & disk_name) const
+{
+    auto disk = tryGetDiskByName(disk_name);
+    if (!disk)
+        throw Exception(ErrorCodes::UNKNOWN_DISK,
+            "No such disk {} in storage policy {}", backQuote(disk_name), backQuote(getName()));
+
+    return disk;
+}
+
+VolumePtr IStoragePolicy::getVolumeByName(const String & volume_name) const
+{
+    auto volume = tryGetVolumeByName(volume_name);
+    if (!volume)
+        throw Exception(ErrorCodes::UNKNOWN_VOLUME,
+            "No such volume {} in storage policy {}", backQuote(volume_name), backQuote(getName()));
+
+    return volume;
+}
+
+}

src/Disks/IStoragePolicy.h

@@ -39,7 +39,8 @@ public:
     /// Used when it's not important, for example for
     /// mutations files
     virtual DiskPtr getAnyDisk() const = 0;
-    virtual DiskPtr getDiskByName(const String & disk_name) const = 0;
+    virtual DiskPtr tryGetDiskByName(const String & disk_name) const = 0;
+    DiskPtr getDiskByName(const String & disk_name) const;
     /// Get free space from most free disk
     virtual UInt64 getMaxUnreservedFreeSpace() const = 0;
     /// Reserves space on any volume with index > min_volume_index or returns nullptr
@@ -53,7 +54,8 @@ public:
     virtual ReservationPtr makeEmptyReservationOnLargestDisk() const = 0;
     /// Get volume by index.
     virtual VolumePtr getVolume(size_t index) const = 0;
-    virtual VolumePtr getVolumeByName(const String & volume_name) const = 0;
+    virtual VolumePtr tryGetVolumeByName(const String & volume_name) const = 0;
+    VolumePtr getVolumeByName(const String & volume_name) const;
     /// Checks if storage policy can be replaced by another one.
     virtual void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const = 0;
     /// Find volume index, which contains disk

src/Disks/StoragePolicy.cpp

@@ -179,7 +179,7 @@ DiskPtr StoragePolicy::getAnyDisk() const
 }
 
 
-DiskPtr StoragePolicy::getDiskByName(const String & disk_name) const
+DiskPtr StoragePolicy::tryGetDiskByName(const String & disk_name) const
 {
     for (auto && volume : volumes)
         for (auto && disk : volume->getDisks())
@@ -265,11 +265,11 @@ VolumePtr StoragePolicy::getVolume(size_t index) const
 }
 
 
-VolumePtr StoragePolicy::getVolumeByName(const String & volume_name) const
+VolumePtr StoragePolicy::tryGetVolumeByName(const String & volume_name) const
 {
     auto it = volume_index_by_volume_name.find(volume_name);
     if (it == volume_index_by_volume_name.end())
-        throw Exception("No such volume " + backQuote(volume_name) + " in storage policy " + backQuote(name), ErrorCodes::UNKNOWN_VOLUME);
+        return nullptr;
     return getVolume(it->second);
 }
 

src/Disks/StoragePolicy.h

@@ -52,7 +52,7 @@ public:
     /// mutations files
     DiskPtr getAnyDisk() const override;
 
-    DiskPtr getDiskByName(const String & disk_name) const override;
+    DiskPtr tryGetDiskByName(const String & disk_name) const override;
 
     /// Get free space from most free disk
     UInt64 getMaxUnreservedFreeSpace() const override;
@@ -84,7 +84,7 @@ public:
     /// Get volume by index.
     VolumePtr getVolume(size_t index) const override;
 
-    VolumePtr getVolumeByName(const String & volume_name) const override;
+    VolumePtr tryGetVolumeByName(const String & volume_name) const override;
 
     /// Checks if storage policy can be replaced by another one.
     void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const override;

src/Functions/geoToH3.cpp

@@ -11,6 +11,7 @@
 #include <Common/typeid_cast.h>
 #include <base/range.h>
 
+#include <constants.h>
 #include <h3api.h>
 
 
@@ -20,6 +21,8 @@ namespace ErrorCodes
 {
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int INCORRECT_DATA;
+    extern const int ILLEGAL_COLUMN;
+    extern const int ARGUMENT_OUT_OF_BOUND;
 }
 
 namespace
@@ -68,9 +71,35 @@ public:
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
-        const auto * col_lon = arguments[0].column.get();
-        const auto * col_lat = arguments[1].column.get();
-        const auto * col_res = arguments[2].column.get();
+        const auto * col_lon = checkAndGetColumn<ColumnFloat64>(arguments[0].column.get());
+        if (!col_lon)
+            throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Illegal type {} of argument {} of function {}. Must be Float64.",
+                    arguments[0].type->getName(),
+                    1,
+                    getName());
+        const auto & data_lon = col_lon->getData();
+
+        const auto * col_lat = checkAndGetColumn<ColumnFloat64>(arguments[1].column.get());
+        if (!col_lat)
+            throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Illegal type {} of argument {} of function {}. Must be Float64.",
+                    arguments[1].type->getName(),
+                    2,
+                    getName());
+        const auto & data_lat = col_lat->getData();
+
+        const auto * col_res = checkAndGetColumn<ColumnUInt8>(arguments[2].column.get());
+        if (!col_res)
+            throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Illegal type {} of argument {} of function {}. Must be UInt8.",
+                    arguments[2].type->getName(),
+                    3,
+                    getName());
+        const auto & data_res = col_res->getData();
 
         auto dst = ColumnVector<UInt64>::create();
         auto & dst_data = dst->getData();
@@ -78,9 +107,17 @@ public:
 
         for (size_t row = 0; row < input_rows_count; ++row)
         {
-            const double lon = col_lon->getFloat64(row);
-            const double lat = col_lat->getFloat64(row);
-            const UInt8 res = col_res->getUInt(row);
+            const double lon = data_lon[row];
+            const double lat = data_lat[row];
+            const UInt8 res = data_res[row];
+
+            if (res > MAX_H3_RES)
+                throw Exception(
+                        ErrorCodes::ARGUMENT_OUT_OF_BOUND,
+                        "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ",
+                        toString(res),
+                        getName(),
+                        MAX_H3_RES);
 
             LatLng coord;
             coord.lng = degsToRads(lon);

src/IO/ReadWriteBufferFromHTTP.h

@@ -19,6 +19,7 @@
 #include <Common/DNSResolver.h>
 #include <Common/RemoteHostFilter.h>
 #include <Common/config.h>
+#include <Common/config_version.h>
 #include <base/logger_useful.h>
 #include <Poco/URIStreamFactory.h>
 
@@ -291,6 +292,18 @@ namespace detail
                                 "0 < http_retry_initial_backoff_ms < settings.http_retry_max_backoff_ms (now 0 < {} < {})",
                                 settings.http_max_tries, settings.http_retry_initial_backoff_ms, settings.http_retry_max_backoff_ms);
 
+            // Configure User-Agent if it not already set.
+            const std::string user_agent = "User-Agent";
+            auto iter = std::find_if(http_header_entries.begin(), http_header_entries.end(), [&user_agent](const HTTPHeaderEntry & entry)
+            {
+                return std::get<0>(entry) == user_agent;
+            });
+
+            if (iter == http_header_entries.end())
+            {
+                http_header_entries.emplace_back(std::make_pair("User-Agent", fmt::format("ClickHouse/{}", VERSION_STRING)));
+            }
+
             if (!delay_initialization)
             {
                 initialize();

src/IO/WriteBufferFromHTTP.cpp

@@ -10,6 +10,7 @@ WriteBufferFromHTTP::WriteBufferFromHTTP(
     const Poco::URI & uri,
     const std::string & method,
     const std::string & content_type,
+    const std::string & content_encoding,
     const ConnectionTimeouts & timeouts,
     size_t buffer_size_)
     : WriteBufferFromOStream(buffer_size_)
@@ -24,6 +25,9 @@ WriteBufferFromHTTP::WriteBufferFromHTTP(
         request.set("Content-Type", content_type);
     }
 
+    if (!content_encoding.empty())
+        request.set("Content-Encoding", content_encoding);
+
     LOG_TRACE((&Poco::Logger::get("WriteBufferToHTTP")), "Sending request to {}", uri.toString());
 
     ostr = &session->sendRequest(request);
@@ -31,6 +35,10 @@ WriteBufferFromHTTP::WriteBufferFromHTTP(
 
 void WriteBufferFromHTTP::finalizeImpl()
 {
+    // for compressed body, the data is stored in buffered first
+    // here, make sure the content in the buffer has been flushed
+    this->nextImpl();
+
     receiveResponse(*session, request, response, false);
     /// TODO: Response body is ignored.
 }

src/IO/WriteBufferFromHTTP.h

@@ -21,6 +21,7 @@ public:
     explicit WriteBufferFromHTTP(const Poco::URI & uri,
                                  const std::string & method = Poco::Net::HTTPRequest::HTTP_POST, // POST or PUT only
                                  const std::string & content_type = "",
+                                 const std::string & content_encoding = "",
                                  const ConnectionTimeouts & timeouts = {},
                                  size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
 

src/IO/tests/gtest_archive_reader_and_writer.cpp

@@ -328,14 +328,16 @@ TEST_P(ArchiveReaderAndWriterTest, ArchiveNotExist)
 }
 
 
+#if USE_MINIZIP
+
 namespace
 {
     const char * supported_archive_file_exts[] =
     {
-#if USE_MINIZIP
         ".zip",
-#endif
     };
 }
 
 INSTANTIATE_TEST_SUITE_P(All, ArchiveReaderAndWriterTest, ::testing::ValuesIn(supported_archive_file_exts));
+
+#endif

src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp

@@ -5,6 +5,7 @@
 #include <Parsers/formatAST.h>
 #include <Access/AccessControl.h>
 #include <Access/Common/AccessFlags.h>
+#include <Access/Common/AccessRightsElement.h>
 #include <Access/RowPolicy.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeDDLQueryOnCluster.h>
@@ -45,22 +46,24 @@ namespace
 BlockIO InterpreterCreateRowPolicyQuery::execute()
 {
     auto & query = query_ptr->as<ASTCreateRowPolicyQuery &>();
-    auto & access_control = getContext()->getAccessControl();
-    getContext()->checkAccess(query.alter ? AccessType::ALTER_ROW_POLICY : AccessType::CREATE_ROW_POLICY);
+    auto required_access = getRequiredAccess();
 
     if (!query.cluster.empty())
     {
         query.replaceCurrentUserTag(getContext()->getUserName());
-        return executeDDLQueryOnCluster(query_ptr, getContext());
+        return executeDDLQueryOnCluster(query_ptr, getContext(), required_access);
     }
 
     assert(query.names->cluster.empty());
+    auto & access_control = getContext()->getAccessControl();
+    getContext()->checkAccess(required_access);
+
+    query.replaceEmptyDatabase(getContext()->getCurrentDatabase());
+
     std::optional<RolesOrUsersSet> roles_from_query;
     if (query.roles)
         roles_from_query = RolesOrUsersSet{*query.roles, access_control, getContext()->getUserID()};
 
-    query.replaceEmptyDatabase(getContext()->getCurrentDatabase());
-
     if (query.alter)
     {
         auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
@@ -105,4 +108,15 @@ void InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(RowPolicy & polic
     updateRowPolicyFromQueryImpl(policy, query, {}, {});
 }
 
+
+AccessRightsElements InterpreterCreateRowPolicyQuery::getRequiredAccess() const
+{
+    const auto & query = query_ptr->as<const ASTCreateRowPolicyQuery &>();
+    AccessRightsElements res;
+    auto access_type = (query.alter ? AccessType::ALTER_ROW_POLICY : AccessType::CREATE_ROW_POLICY);
+    for (const auto & row_policy_name : query.names->full_names)
+        res.emplace_back(access_type, row_policy_name.database, row_policy_name.table_name);
+    return res;
+}
+
 }

src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h

@@ -6,8 +6,8 @@
 
 namespace DB
 {
-
 class ASTCreateRowPolicyQuery;
+class AccessRightsElements;
 struct RowPolicy;
 
 class InterpreterCreateRowPolicyQuery : public IInterpreter, WithMutableContext
@@ -20,6 +20,8 @@ public:
     static void updateRowPolicyFromQuery(RowPolicy & policy, const ASTCreateRowPolicyQuery & query);
 
 private:
+    AccessRightsElements getRequiredAccess() const;
+
     ASTPtr query_ptr;
 };
 

src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp

@@ -49,12 +49,37 @@ AccessRightsElements InterpreterDropAccessEntityQuery::getRequiredAccess() const
     AccessRightsElements res;
     switch (query.type)
     {
-        case AccessEntityType::USER: res.emplace_back(AccessType::DROP_USER); return res;
-        case AccessEntityType::ROLE: res.emplace_back(AccessType::DROP_ROLE); return res;
-        case AccessEntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::DROP_SETTINGS_PROFILE); return res;
-        case AccessEntityType::ROW_POLICY: res.emplace_back(AccessType::DROP_ROW_POLICY); return res;
-        case AccessEntityType::QUOTA: res.emplace_back(AccessType::DROP_QUOTA); return res;
-        case AccessEntityType::MAX: break;
+        case AccessEntityType::USER:
+        {
+            res.emplace_back(AccessType::DROP_USER);
+            return res;
+        }
+        case AccessEntityType::ROLE:
+        {
+            res.emplace_back(AccessType::DROP_ROLE);
+            return res;
+        }
+        case AccessEntityType::SETTINGS_PROFILE:
+        {
+            res.emplace_back(AccessType::DROP_SETTINGS_PROFILE);
+            return res;
+        }
+        case AccessEntityType::ROW_POLICY:
+        {
+            if (query.row_policy_names)
+            {
+                for (const auto & row_policy_name : query.row_policy_names->full_names)
+                    res.emplace_back(AccessType::DROP_ROW_POLICY, row_policy_name.database, row_policy_name.table_name);
+            }
+            return res;
+        }
+        case AccessEntityType::QUOTA:
+        {
+            res.emplace_back(AccessType::DROP_QUOTA);
+            return res;
+        }
+        case AccessEntityType::MAX:
+            break;
     }
     throw Exception(
         toString(query.type) + ": type is not supported by DROP query", ErrorCodes::NOT_IMPLEMENTED);

src/Interpreters/Access/InterpreterDropAccessEntityQuery.h

@@ -6,7 +6,6 @@
 
 namespace DB
 {
-
 class AccessRightsElements;
 
 class InterpreterDropAccessEntityQuery : public IInterpreter, WithMutableContext

src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp

@@ -377,12 +377,48 @@ AccessRightsElements InterpreterShowCreateAccessEntityQuery::getRequiredAccess()
     AccessRightsElements res;
     switch (show_query.type)
     {
-        case AccessEntityType::USER: res.emplace_back(AccessType::SHOW_USERS); return res;
-        case AccessEntityType::ROLE: res.emplace_back(AccessType::SHOW_ROLES); return res;
-        case AccessEntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); return res;
-        case AccessEntityType::ROW_POLICY: res.emplace_back(AccessType::SHOW_ROW_POLICIES); return res;
-        case AccessEntityType::QUOTA: res.emplace_back(AccessType::SHOW_QUOTAS); return res;
-        case AccessEntityType::MAX: break;
+        case AccessEntityType::USER:
+        {
+            res.emplace_back(AccessType::SHOW_USERS);
+            return res;
+        }
+        case AccessEntityType::ROLE:
+        {
+            res.emplace_back(AccessType::SHOW_ROLES);
+            return res;
+        }
+        case AccessEntityType::SETTINGS_PROFILE:
+        {
+            res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES);
+            return res;
+        }
+        case AccessEntityType::ROW_POLICY:
+        {
+            if (show_query.row_policy_names)
+            {
+                for (const auto & row_policy_name : show_query.row_policy_names->full_names)
+                    res.emplace_back(AccessType::SHOW_ROW_POLICIES, row_policy_name.database, row_policy_name.table_name);
+            }
+            else if (show_query.database_and_table_name)
+            {
+                if (show_query.database_and_table_name->second.empty())
+                    res.emplace_back(AccessType::SHOW_ROW_POLICIES, show_query.database_and_table_name->first);
+                else
+                    res.emplace_back(AccessType::SHOW_ROW_POLICIES, show_query.database_and_table_name->first, show_query.database_and_table_name->second);
+            }
+            else
+            {
+                res.emplace_back(AccessType::SHOW_ROW_POLICIES);
+            }
+            return res;
+        }
+        case AccessEntityType::QUOTA:
+        {
+            res.emplace_back(AccessType::SHOW_QUOTAS);
+            return res;
+        }
+        case AccessEntityType::MAX:
+            break;
     }
     throw Exception(toString(show_query.type) + ": type is not supported by SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED);
 }

src/Interpreters/Context.cpp

@@ -212,6 +212,7 @@ struct ContextSharedPart
     mutable MarkCachePtr index_mark_cache;                  /// Cache of marks in compressed files of MergeTree indices.
     mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads.
     ProcessList process_list;                               /// Executing queries at the moment.
+    GlobalOvercommitTracker global_overcommit_tracker;
     MergeList merge_list;                                   /// The list of executable merge (for (Replicated)?MergeTree)
     ReplicatedFetchList replicated_fetch_list;
     ConfigurationPtr users_config;                          /// Config with the users, profiles and quotas sections.
@@ -284,7 +285,9 @@ struct ContextSharedPart
 #endif
 
     ContextSharedPart()
-        : access_control(std::make_unique<AccessControl>()), macros(std::make_unique<Macros>())
+        : access_control(std::make_unique<AccessControl>())
+        , global_overcommit_tracker(&process_list)
+        , macros(std::make_unique<Macros>())
     {
         /// TODO: make it singleton (?)
         static std::atomic<size_t> num_calls{0};
@@ -492,6 +495,7 @@ std::unique_lock<std::recursive_mutex> Context::getLock() const
 
 ProcessList & Context::getProcessList() { return shared->process_list; }
 const ProcessList & Context::getProcessList() const { return shared->process_list; }
+OvercommitTracker * Context::getGlobalOvercommitTracker() const { return &shared->global_overcommit_tracker; }
 MergeList & Context::getMergeList() { return shared->merge_list; }
 const MergeList & Context::getMergeList() const { return shared->merge_list; }
 ReplicatedFetchList & Context::getReplicatedFetchList() { return shared->replicated_fetch_list; }

src/Interpreters/Context.h

@@ -30,6 +30,8 @@
 namespace Poco::Net { class IPAddress; }
 namespace zkutil { class ZooKeeper; }
 
+struct OvercommitTracker;
+
 namespace DB
 {
 
@@ -664,6 +666,8 @@ public:
     ProcessList & getProcessList();
     const ProcessList & getProcessList() const;
 
+    OvercommitTracker * getGlobalOvercommitTracker() const;
+
     MergeList & getMergeList();
     const MergeList & getMergeList() const;
 

src/Interpreters/ProcessList.cpp

@@ -195,33 +195,21 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
                     ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING);
         }
 
-        auto process_it = processes.emplace(processes.end(),
-            query_context, query_, client_info, priorities.insert(settings.priority), query_kind);
-
-        increaseQueryKindAmount(query_kind);
-
-        res = std::make_shared<Entry>(*this, process_it);
-
         ProcessListForUser & user_process_list = user_to_queries[client_info.current_user];
-        user_process_list.queries.emplace(client_info.current_query_id, &res->get());
-
-        process_it->setUserProcessList(&user_process_list);
-
-        /// Track memory usage for all simultaneously running queries from single user.
-        user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user);
-        user_process_list.user_memory_tracker.setDescription("(for user)");
 
         /// Actualize thread group info
-        if (auto thread_group = CurrentThread::getGroup())
+        auto thread_group = CurrentThread::getGroup();
+        if (thread_group)
         {
             std::lock_guard lock_thread_group(thread_group->mutex);
             thread_group->performance_counters.setParent(&user_process_list.user_performance_counters);
             thread_group->memory_tracker.setParent(&user_process_list.user_memory_tracker);
-            thread_group->query = process_it->query;
-            thread_group->normalized_query_hash = normalizedQueryHash<false>(process_it->query);
+            thread_group->query = query_;
+            thread_group->normalized_query_hash = normalizedQueryHash<false>(query_);
 
             /// Set query-level memory trackers
             thread_group->memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage);
+            thread_group->memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage);
 
             if (query_context->hasTraceCollector())
             {
@@ -236,10 +224,28 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
 
             /// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values
             ///  since allocation and deallocation could happen in different threads
-
-            process_it->thread_group = std::move(thread_group);
         }
 
+        auto process_it = processes.emplace(processes.end(),
+            query_context, query_, client_info, priorities.insert(settings.priority), std::move(thread_group), query_kind);
+
+        increaseQueryKindAmount(query_kind);
+
+        res = std::make_shared<Entry>(*this, process_it);
+
+        process_it->setUserProcessList(&user_process_list);
+
+        {
+            BlockQueryIfMemoryLimit block_query{user_process_list.user_overcommit_tracker};
+            user_process_list.queries.emplace(client_info.current_query_id, &res->get());
+        }
+
+        /// Track memory usage for all simultaneously running queries from single user.
+        user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user);
+        user_process_list.user_memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage_for_user);
+        user_process_list.user_memory_tracker.setDescription("(for user)");
+        user_process_list.user_overcommit_tracker.setMaxWaitTime(settings.memory_usage_overcommit_max_wait_microseconds);
+
         if (!user_process_list.user_throttler)
         {
             if (settings.max_network_bandwidth_for_user)
@@ -268,9 +274,6 @@ ProcessListEntry::~ProcessListEntry()
 
     const QueryStatus * process_list_element_ptr = &*it;
 
-    /// This removes the memory_tracker of one request.
-    parent.processes.erase(it);
-
     auto user_process_list_it = parent.user_to_queries.find(user);
     if (user_process_list_it == parent.user_to_queries.end())
     {
@@ -286,11 +289,15 @@ ProcessListEntry::~ProcessListEntry()
     {
         if (running_query->second == process_list_element_ptr)
         {
+            BlockQueryIfMemoryLimit block_query{user_process_list.user_overcommit_tracker};
             user_process_list.queries.erase(running_query->first);
             found = true;
         }
     }
 
+    /// This removes the memory_tracker of one request.
+    parent.processes.erase(it);
+
     if (!found)
     {
         LOG_ERROR(&Poco::Logger::get("ProcessList"), "Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser");
@@ -312,10 +319,16 @@ ProcessListEntry::~ProcessListEntry()
 
 
 QueryStatus::QueryStatus(
-    ContextPtr context_, const String & query_, const ClientInfo & client_info_, QueryPriorities::Handle && priority_handle_, IAST::QueryKind query_kind_)
+    ContextPtr context_,
+    const String & query_,
+    const ClientInfo & client_info_,
+    QueryPriorities::Handle && priority_handle_,
+    ThreadGroupStatusPtr && thread_group_,
+    IAST::QueryKind query_kind_)
     : WithContext(context_)
     , query(query_)
     , client_info(client_info_)
+    , thread_group(std::move(thread_group_))
     , priority_handle(std::move(priority_handle_))
     , query_kind(query_kind_)
     , num_queries_increment(CurrentMetrics::Query)
@@ -328,6 +341,14 @@ QueryStatus::QueryStatus(
 QueryStatus::~QueryStatus()
 {
     assert(executors.empty());
+
+    if (auto * memory_tracker = getMemoryTracker())
+    {
+        if (user_process_list)
+            user_process_list->user_overcommit_tracker.unsubscribe(memory_tracker);
+        if (auto shared_context = getContext())
+            shared_context->getGlobalOvercommitTracker()->unsubscribe(memory_tracker);
+    }
 }
 
 CancellationCode QueryStatus::cancelQuery(bool)
@@ -481,7 +502,11 @@ ProcessList::Info ProcessList::getInfo(bool get_thread_list, bool get_profile_ev
 }
 
 
-ProcessListForUser::ProcessListForUser() = default;
+ProcessListForUser::ProcessListForUser()
+    : user_overcommit_tracker(this)
+{
+    user_memory_tracker.setOvercommitTracker(&user_overcommit_tracker);
+}
 
 
 ProcessListForUserInfo ProcessListForUser::getInfo(bool get_profile_events) const

src/Interpreters/ProcessList.h

@@ -16,6 +16,7 @@
 #include <Common/ProfileEvents.h>
 #include <Common/Stopwatch.h>
 #include <Common/Throttler.h>
+#include <Common/OvercommitTracker.h>
 
 #include <condition_variable>
 #include <list>
@@ -76,6 +77,7 @@ protected:
     friend class ThreadStatus;
     friend class CurrentThread;
     friend class ProcessListEntry;
+    friend struct ::GlobalOvercommitTracker;
 
     String query;
     ClientInfo client_info;
@@ -132,6 +134,7 @@ public:
         const String & query_,
         const ClientInfo & client_info_,
         QueryPriorities::Handle && priority_handle_,
+        ThreadGroupStatusPtr && thread_group_,
         IAST::QueryKind query_kind_
         );
 
@@ -154,6 +157,13 @@ public:
 
     ThrottlerPtr getUserNetworkThrottler();
 
+    MemoryTracker * getMemoryTracker() const
+    {
+        if (!thread_group)
+            return nullptr;
+        return &thread_group->memory_tracker;
+    }
+
     bool updateProgressIn(const Progress & value)
     {
         CurrentThread::updateProgressIn(value);
@@ -216,6 +226,8 @@ struct ProcessListForUser
     /// Limit and counter for memory of all simultaneously running queries of single user.
     MemoryTracker user_memory_tracker{VariableContext::User};
 
+    UserOvercommitTracker user_overcommit_tracker;
+
     /// Count network usage for all simultaneously running queries of single user.
     ThrottlerPtr user_throttler;
 
@@ -337,6 +349,14 @@ public:
         max_size = max_size_;
     }
 
+    template <typename F>
+    void processEachQueryStatus(F && func) const
+    {
+        std::lock_guard lk(mutex);
+        for (auto && query : processes)
+            func(query);
+    }
+
     void setMaxInsertQueriesAmount(size_t max_insert_queries_amount_)
     {
         std::lock_guard lock(mutex);

src/Parsers/ASTTTLElement.cpp

@@ -1,13 +1,17 @@
-
 #include <Columns/Collator.h>
 #include <Common/quoteString.h>
 #include <Parsers/ASTTTLElement.h>
 #include <IO/Operators.h>
-
+#include <base/EnumReflection.h>
 
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
 ASTPtr ASTTTLElement::clone() const
 {
     auto clone = std::make_shared<ASTTTLElement>(*this);
@@ -29,13 +33,21 @@ ASTPtr ASTTTLElement::clone() const
 void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     ttl()->formatImpl(settings, state, frame);
-    if (mode == TTLMode::MOVE && destination_type == DataDestinationType::DISK)
+    if (mode == TTLMode::MOVE)
     {
-        settings.ostr << " TO DISK " << quoteString(destination_name);
-    }
-    else if (mode == TTLMode::MOVE && destination_type == DataDestinationType::VOLUME)
-    {
-        settings.ostr << " TO VOLUME " << quoteString(destination_name);
+        if (destination_type == DataDestinationType::DISK)
+            settings.ostr << " TO DISK ";
+        else if (destination_type == DataDestinationType::VOLUME)
+            settings.ostr << " TO VOLUME ";
+        else
+            throw Exception(ErrorCodes::LOGICAL_ERROR,
+                "Unsupported destination type {} for TTL MOVE",
+                    magic_enum::enum_name(destination_type));
+
+        if (if_exists)
+            settings.ostr << "IF EXISTS ";
+
+        settings.ostr << quoteString(destination_name);
     }
     else if (mode == TTLMode::GROUP_BY)
     {

src/Parsers/ASTTTLElement.h

@@ -16,16 +16,18 @@ public:
     TTLMode mode;
     DataDestinationType destination_type;
     String destination_name;
+    bool if_exists = false;
 
     ASTs group_by_key;
     ASTs group_by_assignments;
 
     ASTPtr recompression_codec;
 
-    ASTTTLElement(TTLMode mode_, DataDestinationType destination_type_, const String & destination_name_)
+    ASTTTLElement(TTLMode mode_, DataDestinationType destination_type_, const String & destination_name_, bool if_exists_)
         : mode(mode_)
         , destination_type(destination_type_)
         , destination_name(destination_name_)
+        , if_exists(if_exists_)
         , ttl_expr_pos(-1)
         , where_expr_pos(-1)
     {

src/Parsers/ASTTablesInSelectQuery.cpp

@@ -247,10 +247,12 @@ void ASTTableJoin::formatImpl(const FormatSettings & settings, FormatState & sta
 
 void ASTArrayJoin::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
+    std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' ');
     frame.expression_list_prepend_whitespace = true;
 
     settings.ostr << (settings.hilite ? hilite_keyword : "")
         << settings.nl_or_ws
+        << indent_str
         << (kind == Kind::Left ? "LEFT " : "") << "ARRAY JOIN" << (settings.hilite ? hilite_none : "");
 
     settings.one_line

src/Parsers/ExpressionElementParsers.cpp

@@ -2360,6 +2360,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
     ParserKeyword s_to_disk("TO DISK");
     ParserKeyword s_to_volume("TO VOLUME");
+    ParserKeyword s_if_exists("IF EXISTS");
     ParserKeyword s_delete("DELETE");
     ParserKeyword s_where("WHERE");
     ParserKeyword s_group_by("GROUP BY");
@@ -2414,9 +2415,13 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ASTPtr group_by_key;
     ASTPtr recompression_codec;
     ASTPtr group_by_assignments;
+    bool if_exists = false;
 
     if (mode == TTLMode::MOVE)
     {
+        if (s_if_exists.ignore(pos))
+            if_exists = true;
+
         ASTPtr ast_space_name;
         if (!parser_string_literal.parse(pos, ast_space_name, expected))
             return false;
@@ -2448,7 +2453,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             return false;
     }
 
-    auto ttl_element = std::make_shared<ASTTTLElement>(mode, destination_type, destination_name);
+    auto ttl_element = std::make_shared<ASTTTLElement>(mode, destination_type, destination_name, if_exists);
     ttl_element->setTTL(std::move(ttl_expr));
     if (where_expr)
         ttl_element->setWhere(std::move(where_expr));

src/Processors/Transforms/buildPushingToViewsChain.cpp

@@ -331,7 +331,7 @@ Chain buildPushingToViewsChain(
         {
             auto executing_inner_query = std::make_shared<ExecutingInnerQueryFromViewTransform>(
                 storage_header, views_data->views.back(), views_data);
-            executing_inner_query->setRuntimeData(view_thread_status, elapsed_counter_ms);
+            executing_inner_query->setRuntimeData(view_thread_status, view_counter_ms);
 
             out.addSource(std::move(executing_inner_query));
         }
@@ -381,7 +381,7 @@ Chain buildPushingToViewsChain(
         processors.emplace_front(std::move(copying_data));
         processors.emplace_back(std::move(finalizing_views));
         result_chain = Chain(std::move(processors));
-        result_chain.setNumThreads(max_parallel_streams);
+        result_chain.setNumThreads(std::min(views_data->max_threads, max_parallel_streams));
     }
 
     if (auto * live_view = dynamic_cast<StorageLiveView *>(storage.get()))

src/Server/GRPCServer.cpp

@@ -642,6 +642,9 @@ namespace
         void throwIfFailedToReadQueryInfo();
         bool isQueryCancelled();
 
+        void addQueryDetailsToResult();
+        void addOutputFormatToResult();
+        void addOutputColumnsNamesAndTypesToResult(const Block & headers);
         void addProgressToResult();
         void addTotalsToResult(const Block & totals);
         void addExtremesToResult(const Block & extremes);
@@ -667,6 +670,7 @@ namespace
         CompressionMethod input_compression_method = CompressionMethod::None;
         PODArray<char> output;
         String output_format;
+        bool send_output_columns_names_and_types = false;
         CompressionMethod output_compression_method = CompressionMethod::None;
         int output_compression_level = 0;
 
@@ -888,6 +892,8 @@ namespace
         if (output_format.empty())
             output_format = query_context->getDefaultFormat();
 
+        send_output_columns_names_and_types = query_info.send_output_columns();
+
         /// Choose compression.
         String input_compression_method_str = query_info.input_compression_type();
         if (input_compression_method_str.empty())
@@ -1150,6 +1156,9 @@ namespace
 
     void Call::generateOutput()
     {
+        /// We add query_id and time_zone to the first result anyway.
+        addQueryDetailsToResult();
+
         if (!io.pipeline.initialized() || io.pipeline.pushing())
             return;
 
@@ -1189,6 +1198,9 @@ namespace
                 return true;
             };
 
+            addOutputFormatToResult();
+            addOutputColumnsNamesAndTypesToResult(header);
+
             Block block;
             while (check_for_cancel())
             {
@@ -1439,6 +1451,29 @@ namespace
         return false;
     }
 
+    void Call::addQueryDetailsToResult()
+    {
+        *result.mutable_query_id() = query_context->getClientInfo().current_query_id;
+        *result.mutable_time_zone() = DateLUT::instance().getTimeZone();
+    }
+
+    void Call::addOutputFormatToResult()
+    {
+        *result.mutable_output_format() = output_format;
+    }
+
+    void Call::addOutputColumnsNamesAndTypesToResult(const Block & header)
+    {
+        if (!send_output_columns_names_and_types)
+            return;
+        for (const auto & column : header)
+        {
+            auto & name_and_type = *result.add_output_columns();
+            *name_and_type.mutable_name() = column.name;
+            *name_and_type.mutable_type() = column.type->getName();
+        }
+    }
+
     void Call::addProgressToResult()
     {
         auto values = progress.fetchAndResetPiecewiseAtomically();

src/Server/grpc_protos/clickhouse_grpc.proto

@@ -82,6 +82,9 @@ message QueryInfo {
    // Default output format. If not specified, 'TabSeparated' is used.
    string output_format = 7;
 
+   // Set it if you want the names and the types of output columns to be sent to the client.
+   bool send_output_columns = 24;
+
    repeated ExternalTable external_tables = 8;
 
    string user_name = 9;
@@ -187,7 +190,17 @@ message Exception {
 
 // Result of execution of a query which is sent back by the ClickHouse server to the client.
 message Result {
-   // Output of the query, represented in the `output_format` or in a format specified in `query`.
+   string query_id = 9;
+   string time_zone = 10;
+   
+   // The format in which `output`, `totals` and `extremes` are written.
+   // It's either the same as `output_format` specified in `QueryInfo` or the format specified in the query itself.
+   string output_format = 11;
+
+   // The names and types of columns of the result written in `output`.
+   repeated NameAndType output_columns = 12;
+
+   // Output of the query, represented in the `output_format`.
    bytes output = 1;
    bytes totals = 2;
    bytes extremes = 3;

src/Storages/HDFS/StorageHDFSCluster.cpp

@@ -2,17 +2,9 @@
 
 #if USE_HDFS
 
-#include <Common/Exception.h>
-#include <Common/Throttler.h>
 #include <Client/Connection.h>
 #include <Core/QueryProcessingStage.h>
-#include <Core/UUID.h>
-#include <Columns/ColumnsNumber.h>
-#include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeString.h>
-#include <IO/ReadHelpers.h>
-#include <IO/WriteBufferFromS3.h>
-#include <IO/WriteHelpers.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/getHeaderForProcessingStage.h>
 #include <Interpreters/SelectQueryOptions.h>
@@ -21,7 +13,6 @@
 #include <Processors/Transforms/AddingDefaultsTransform.h>
 #include <QueryPipeline/narrowBlockInputStreams.h>
 #include <QueryPipeline/Pipe.h>
-#include <Processors/Sources/SourceWithProgress.h>
 #include <Processors/Sources/RemoteSource.h>
 #include <QueryPipeline/RemoteQueryExecutor.h>
 #include <Parsers/queryToString.h>
@@ -29,16 +20,13 @@
 #include <Storages/IStorage.h>
 #include <Storages/SelectQueryInfo.h>
 #include <Storages/HDFS/StorageHDFSCluster.h>
-#include <base/logger_useful.h>
 
-#include <ios>
 #include <memory>
-#include <string>
-#include <thread>
-#include <cassert>
+
 
 namespace DB
 {
+
 StorageHDFSCluster::StorageHDFSCluster(
     String cluster_name_,
     const String & uri_,

src/Storages/MergeTree/MergeTreeData.cpp

@@ -656,13 +656,14 @@ void MergeTreeData::checkTTLExpressions(const StorageInMemoryMetadata & new_meta
     {
         for (const auto & move_ttl : new_table_ttl.move_ttl)
         {
-            if (!getDestinationForMoveTTL(move_ttl))
+            if (!move_ttl.if_exists && !getDestinationForMoveTTL(move_ttl))
             {
                 String message;
                 if (move_ttl.destination_type == DataDestinationType::DISK)
-                    message = "No such disk " + backQuote(move_ttl.destination_name) + " for given storage policy.";
+                    message = "No such disk " + backQuote(move_ttl.destination_name) + " for given storage policy";
                 else
-                    message = "No such volume " + backQuote(move_ttl.destination_name) + " for given storage policy.";
+                    message = "No such volume " + backQuote(move_ttl.destination_name) + " for given storage policy";
+
                 throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION);
             }
         }
@@ -3368,9 +3369,6 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String &
         parts = getDataPartsVectorInPartition(MergeTreeDataPartState::Active, partition_id);
 
     auto disk = getStoragePolicy()->getDiskByName(name);
-    if (!disk)
-        throw Exception("Disk " + name + " does not exists on policy " + getStoragePolicy()->getName(), ErrorCodes::UNKNOWN_DISK);
-
     parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr)
         {
             return part_ptr->volume->getDisk()->getName() == disk->getName();
@@ -4117,10 +4115,10 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(
         SpacePtr destination_ptr = getDestinationForMoveTTL(*move_ttl_entry, is_insert);
         if (!destination_ptr)
         {
-            if (move_ttl_entry->destination_type == DataDestinationType::VOLUME)
+            if (move_ttl_entry->destination_type == DataDestinationType::VOLUME && !move_ttl_entry->if_exists)
                 LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found or rule is not applicable at the moment",
                     move_ttl_entry->destination_name, log_name);
-            else if (move_ttl_entry->destination_type == DataDestinationType::DISK)
+            else if (move_ttl_entry->destination_type == DataDestinationType::DISK && !move_ttl_entry->if_exists)
                 LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found or rule is not applicable at the moment",
                     move_ttl_entry->destination_name, log_name);
         }
@@ -4154,7 +4152,7 @@ SpacePtr MergeTreeData::getDestinationForMoveTTL(const TTLDescription & move_ttl
     auto policy = getStoragePolicy();
     if (move_ttl.destination_type == DataDestinationType::VOLUME)
     {
-        auto volume = policy->getVolumeByName(move_ttl.destination_name);
+        auto volume = policy->tryGetVolumeByName(move_ttl.destination_name);
 
         if (!volume)
             return {};
@@ -4166,7 +4164,8 @@ SpacePtr MergeTreeData::getDestinationForMoveTTL(const TTLDescription & move_ttl
     }
     else if (move_ttl.destination_type == DataDestinationType::DISK)
     {
-        auto disk = policy->getDiskByName(move_ttl.destination_name);
+        auto disk = policy->tryGetDiskByName(move_ttl.destination_name);
+
         if (!disk)
             return {};
 

src/Storages/StorageFile.cpp

@@ -199,18 +199,27 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user
         fs_table_path = user_files_absolute_path / fs_table_path;
 
     Strings paths;
+
     /// Do not use fs::canonical or fs::weakly_canonical.
     /// Otherwise it will not allow to work with symlinks in `user_files_path` directory.
     String path = fs::absolute(fs_table_path).lexically_normal(); /// Normalize path.
-    if (path.find_first_of("*?{") == std::string::npos)
+
+    if (path.find(PartitionedSink::PARTITION_ID_WILDCARD) != std::string::npos)
+    {
+        paths.push_back(path);
+    }
+    else if (path.find_first_of("*?{") == std::string::npos)
     {
         std::error_code error;
         if (fs::exists(path))
             total_bytes_to_read += fs::file_size(path, error);
+
         paths.push_back(path);
     }
     else
+    {
         paths = listFilesWithRegexpMatching("/", path, total_bytes_to_read);
+    }
 
     for (const auto & cur_path : paths)
         checkCreationIsAllowed(context, user_files_absolute_path, cur_path);
@@ -313,7 +322,11 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us
     is_db_table = false;
     paths = getPathsList(table_path_, user_files_path, args.getContext(), total_bytes_to_read);
     is_path_with_globs = paths.size() > 1;
-    path_for_partitioned_write = table_path_;
+    if (!paths.empty())
+        path_for_partitioned_write = paths.front();
+    else
+        path_for_partitioned_write = table_path_;
+
     setStorageMetadata(args);
 }
 
@@ -853,6 +866,7 @@ SinkToStoragePtr StorageFile::write(
     {
         if (path_for_partitioned_write.empty())
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty path for partitioned write");
+
         fs::create_directories(fs::path(path_for_partitioned_write).parent_path());
 
         return std::make_shared<PartitionedStorageFileSink>(

src/Storages/StorageLog.cpp

@@ -732,8 +732,21 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID &
     renameInMemory(new_table_id);
 }
 
-void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &)
+static std::chrono::seconds getLockTimeout(ContextPtr context)
 {
+    const Settings & settings = context->getSettingsRef();
+    Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds();
+    if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout)
+        lock_timeout = settings.max_execution_time.totalSeconds();
+    return std::chrono::seconds{lock_timeout};
+}
+
+void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr context, TableExclusiveLockHolder &)
+{
+    WriteLock lock{rwlock, getLockTimeout(context)};
+    if (!lock)
+        throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED);
+
     disk->clearDirectory(table_path);
 
     for (auto & data_file : data_files)
@@ -750,16 +763,6 @@ void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr
 }
 
 
-static std::chrono::seconds getLockTimeout(ContextPtr context)
-{
-    const Settings & settings = context->getSettingsRef();
-    Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds();
-    if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout)
-        lock_timeout = settings.max_execution_time.totalSeconds();
-    return std::chrono::seconds{lock_timeout};
-}
-
-
 Pipe StorageLog::read(
     const Names & column_names,
     const StorageMetadataPtr & metadata_snapshot,

src/Storages/StorageURL.cpp

@@ -346,9 +346,10 @@ StorageURLSink::StorageURLSink(
     : SinkToStorage(sample_block)
 {
     std::string content_type = FormatFactory::instance().getContentType(format, context, format_settings);
+    std::string content_encoding = toContentEncodingName(compression_method);
 
     write_buf = wrapWriteBufferWithCompressionMethod(
-            std::make_unique<WriteBufferFromHTTP>(Poco::URI(uri), http_method, content_type, timeouts),
+            std::make_unique<WriteBufferFromHTTP>(Poco::URI(uri), http_method, content_type, content_encoding, timeouts),
             compression_method, 3);
     writer = FormatFactory::instance().getOutputFormat(format, *write_buf, sample_block,
         context, {} /* write callback */, format_settings);

src/Storages/TTLDescription.cpp

@@ -112,6 +112,7 @@ TTLDescription::TTLDescription(const TTLDescription & other)
     , aggregate_descriptions(other.aggregate_descriptions)
     , destination_type(other.destination_type)
     , destination_name(other.destination_name)
+    , if_exists(other.if_exists)
     , recompression_codec(other.recompression_codec)
 {
     if (other.expression)
@@ -149,6 +150,7 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other)
     aggregate_descriptions = other.aggregate_descriptions;
     destination_type = other.destination_type;
     destination_name = other.destination_name;
+    if_exists = other.if_exists;
 
     if (other.recompression_codec)
         recompression_codec = other.recompression_codec->clone();
@@ -185,9 +187,10 @@ TTLDescription TTLDescription::getTTLFromAST(
     }
     else /// rows TTL
     {
+        result.mode = ttl_element->mode;
         result.destination_type = ttl_element->destination_type;
         result.destination_name = ttl_element->destination_name;
-        result.mode = ttl_element->mode;
+        result.if_exists = ttl_element->if_exists;
 
         if (ttl_element->mode == TTLMode::DELETE)
         {

src/Storages/TTLDescription.h

@@ -75,6 +75,10 @@ struct TTLDescription
     /// Name of destination disk or volume
     String destination_name;
 
+    /// If true, do nothing if DISK or VOLUME doesn't exist .
+    /// Only valid for table MOVE TTLs.
+    bool if_exists = false;
+
     /// Codec name which will be used to recompress data
     ASTPtr recompression_codec;
 

tests/ci/ast_fuzzer_check.py

@@ -98,6 +98,7 @@ if __name__ == "__main__":
         'server.log': os.path.join(workspace_path, 'server.log'),
         'fuzzer.log': os.path.join(workspace_path, 'fuzzer.log'),
         'report.html': os.path.join(workspace_path, 'report.html'),
+        'core.gz': os.path.join(workspace_path, 'core.gz'),
     }
 
     s3_helper = S3Helper('https://s3.amazonaws.com')

tests/ci/integration_test_check.py

@@ -23,6 +23,8 @@ from rerun_helper import RerunHelper
 from tee_popen import TeePopen
 
 
+# When update, update
+# integration/ci-runner.py:ClickhouseIntegrationTestsRunner.get_images_names too
 IMAGES = [
     "clickhouse/integration-tests-runner",
     "clickhouse/mysql-golang-client",
@@ -32,6 +34,7 @@ IMAGES = [
     "clickhouse/postgresql-java-client",
     "clickhouse/integration-test",
     "clickhouse/kerberos-kdc",
+    "clickhouse/kerberized-hadoop",
     "clickhouse/integration-helper",
     "clickhouse/dotnet-client",
 ]

tests/integration/ci-runner.py

@@ -1,17 +1,16 @@
 #!/usr/bin/env python3
 
-import logging
-import subprocess
-import os
-import glob
-import time
-import shutil
 from collections import defaultdict
-import random
-import json
 import csv
-# for crc32
-import zlib
+import glob
+import json
+import logging
+import os
+import random
+import shutil
+import subprocess
+import time
+import zlib  # for crc32
 
 
 MAX_RETRY = 3
@@ -25,54 +24,62 @@ CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-library-bridge"
 TRIES_COUNT = 10
 MAX_TIME_SECONDS = 3600
 
-MAX_TIME_IN_SANDBOX = 20 * 60   # 20 minutes
-TASK_TIMEOUT = 8 * 60 * 60      # 8 hours
+MAX_TIME_IN_SANDBOX = 20 * 60  # 20 minutes
+TASK_TIMEOUT = 8 * 60 * 60  # 8 hours
+
 
 def stringhash(s):
-    return zlib.crc32(s.encode('utf-8'))
+    return zlib.crc32(s.encode("utf-8"))
+
 
 def get_tests_to_run(pr_info):
     result = set([])
-    changed_files = pr_info['changed_files']
+    changed_files = pr_info["changed_files"]
 
     if changed_files is None:
         return []
 
     for fpath in changed_files:
-        if 'tests/integration/test_' in fpath:
-            logging.info('File %s changed and seems like integration test', fpath)
-            result.add(fpath.split('/')[2])
+        if "tests/integration/test_" in fpath:
+            logging.info("File %s changed and seems like integration test", fpath)
+            result.add(fpath.split("/")[2])
     return list(result)
 
 
 def filter_existing_tests(tests_to_run, repo_path):
     result = []
     for relative_test_path in tests_to_run:
-        if os.path.exists(os.path.join(repo_path, 'tests/integration', relative_test_path)):
+        if os.path.exists(
+            os.path.join(repo_path, "tests/integration", relative_test_path)
+        ):
             result.append(relative_test_path)
         else:
-            logging.info("Skipping test %s, seems like it was removed", relative_test_path)
+            logging.info(
+                "Skipping test %s, seems like it was removed", relative_test_path
+            )
     return result
 
 
 def _get_deselect_option(tests):
-    return ' '.join(['--deselect {}'.format(t) for t in tests])
+    return " ".join([f"--deselect {t}" for t in tests])
+
 
 # https://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks
 def chunks(lst, n):
     """Yield successive n-sized chunks from lst."""
     for i in range(0, len(lst), n):
-        yield lst[i:i + n]
+        yield lst[i : i + n]
+
 
 def get_counters(fname):
     counters = {
-        "ERROR":   set([]),
-        "PASSED":  set([]),
-        "FAILED":  set([]),
+        "ERROR": set([]),
+        "PASSED": set([]),
+        "FAILED": set([]),
         "SKIPPED": set([]),
     }
 
-    with open(fname, 'r') as out:
+    with open(fname, "r") as out:
         for line in out:
             line = line.strip()
             # Example of log:
@@ -81,10 +88,10 @@ def get_counters(fname):
             #     [gw0] [  7%] ERROR test_mysql_protocol/test.py::test_golang_client
             #
             # And only the line with test status should be matched
-            if not('.py::' in line and ' ' in line):
+            if not (".py::" in line and " " in line):
                 continue
 
-            line_arr = line.strip().split(' ')
+            line_arr = line.strip().split(" ")
             if len(line_arr) < 2:
                 logging.debug("Strange line %s", line)
                 continue
@@ -97,9 +104,9 @@ def get_counters(fname):
             if state in counters:
                 counters[state].add(test_name)
             else:
-                # will skip lines line:
-                #     30.76s call     test_host_ip_change/test.py::test_ip_change_drop_dns_cache
-                #     5.71s teardown  test_host_ip_change/test.py::test_user_access_ip_change[node1]
+                # will skip lines like:
+                #     30.76s call     test_host_ip_change/test.py::test_ip_drop_cache
+                #     5.71s teardown  test_host_ip_change/test.py::test_ip_change[node1]
                 # and similar
                 logging.debug("Strange state in line %s", line)
 
@@ -109,13 +116,13 @@ def get_counters(fname):
 def parse_test_times(fname):
     read = False
     description_output = []
-    with open(fname, 'r') as out:
+    with open(fname, "r") as out:
         for line in out:
-            if read and '==' in line:
+            if read and "==" in line:
                 break
             if read and line.strip():
                 description_output.append(line.strip())
-            if 'slowest durations' in line:
+            if "slowest durations" in line:
                 read = True
     return description_output
 
@@ -123,10 +130,10 @@ def parse_test_times(fname):
 def get_test_times(output):
     result = defaultdict(float)
     for line in output:
-        if '.py' in line:
-            line_arr = line.strip().split(' ')
+        if ".py" in line:
+            line_arr = line.strip().split(" ")
             test_time = line_arr[0]
-            test_name = ' '.join([elem for elem in line_arr[2:] if elem])
+            test_name = " ".join([elem for elem in line_arr[2:] if elem])
             if test_name not in result:
                 result[test_name] = 0.0
             result[test_name] += float(test_time[:-1])
@@ -134,21 +141,28 @@ def get_test_times(output):
 
 
 def clear_ip_tables_and_restart_daemons():
-    logging.info("Dump iptables after run %s", subprocess.check_output("sudo iptables -L", shell=True))
+    logging.info(
+        "Dump iptables after run %s",
+        subprocess.check_output("sudo iptables -L", shell=True),
+    )
     try:
         logging.info("Killing all alive docker containers")
-        subprocess.check_output("timeout -s 9 10m docker kill $(docker ps -q)", shell=True)
+        subprocess.check_output(
+            "timeout -s 9 10m docker kill $(docker ps -q)", shell=True
+        )
     except subprocess.CalledProcessError as err:
         logging.info("docker kill excepted: " + str(err))
 
     try:
         logging.info("Removing all docker containers")
-        subprocess.check_output("timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True)
+        subprocess.check_output(
+            "timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True
+        )
     except subprocess.CalledProcessError as err:
         logging.info("docker rm excepted: " + str(err))
 
     # don't restart docker if it's disabled
-    if os.environ.get("CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER", '1') == '1':
+    if os.environ.get("CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER", "1") == "1":
         try:
             logging.info("Stopping docker daemon")
             subprocess.check_output("service docker stop", shell=True)
@@ -177,27 +191,33 @@ def clear_ip_tables_and_restart_daemons():
             # when rules will be empty, it will raise exception
             subprocess.check_output("sudo iptables -D DOCKER-USER 1", shell=True)
     except subprocess.CalledProcessError as err:
-        logging.info("All iptables rules cleared, " + str(iptables_iter) + "iterations, last error: " + str(err))
+        logging.info(
+            "All iptables rules cleared, "
+            + str(iptables_iter)
+            + "iterations, last error: "
+            + str(err)
+        )
 
 
 class ClickhouseIntegrationTestsRunner:
-
     def __init__(self, result_path, params):
         self.result_path = result_path
         self.params = params
 
-        self.image_versions = self.params['docker_images_with_versions']
-        self.shuffle_groups = self.params['shuffle_test_groups']
-        self.flaky_check = 'flaky check' in self.params['context_name']
+        self.image_versions = self.params["docker_images_with_versions"]
+        self.shuffle_groups = self.params["shuffle_test_groups"]
+        self.flaky_check = "flaky check" in self.params["context_name"]
         # if use_tmpfs is not set we assume it to be true, otherwise check
-        self.use_tmpfs = 'use_tmpfs' not in self.params or self.params['use_tmpfs']
-        self.disable_net_host = 'disable_net_host' in self.params and self.params['disable_net_host']
+        self.use_tmpfs = "use_tmpfs" not in self.params or self.params["use_tmpfs"]
+        self.disable_net_host = (
+            "disable_net_host" in self.params and self.params["disable_net_host"]
+        )
         self.start_time = time.time()
         self.soft_deadline_time = self.start_time + (TASK_TIMEOUT - MAX_TIME_IN_SANDBOX)
 
-        if 'run_by_hash_total' in self.params:
-            self.run_by_hash_total = self.params['run_by_hash_total']
-            self.run_by_hash_num = self.params['run_by_hash_num']
+        if "run_by_hash_total" in self.params:
+            self.run_by_hash_total = self.params["run_by_hash_total"]
+            self.run_by_hash_num = self.params["run_by_hash_num"]
         else:
             self.run_by_hash_total = 0
             self.run_by_hash_num = 0
@@ -206,7 +226,7 @@ class ClickhouseIntegrationTestsRunner:
         return self.result_path
 
     def base_path(self):
-        return os.path.join(str(self.result_path), '../')
+        return os.path.join(str(self.result_path), "../")
 
     def should_skip_tests(self):
         return []
@@ -214,8 +234,10 @@ class ClickhouseIntegrationTestsRunner:
     def get_image_with_version(self, name):
         if name in self.image_versions:
             return name + ":" + self.image_versions[name]
-        logging.warn("Cannot find image %s in params list %s", name, self.image_versions)
-        if ':' not in name:
+        logging.warn(
+            "Cannot find image %s in params list %s", name, self.image_versions
+        )
+        if ":" not in name:
             return name + ":latest"
         return name
 
@@ -223,31 +245,44 @@ class ClickhouseIntegrationTestsRunner:
         name = self.get_images_names()[0]
         if name in self.image_versions:
             return self.image_versions[name]
-        logging.warn("Cannot find image %s in params list %s", name, self.image_versions)
-        return 'latest'
+        logging.warn(
+            "Cannot find image %s in params list %s", name, self.image_versions
+        )
+        return "latest"
 
     def shuffle_test_groups(self):
         return self.shuffle_groups != 0
 
     @staticmethod
     def get_images_names():
-        return ["clickhouse/integration-tests-runner", "clickhouse/mysql-golang-client",
-                "clickhouse/mysql-java-client", "clickhouse/mysql-js-client",
-                "clickhouse/mysql-php-client", "clickhouse/postgresql-java-client",
-                "clickhouse/integration-test", "clickhouse/kerberos-kdc",
-                "clickhouse/dotnet-client",
-                "clickhouse/integration-helper", ]
-
+        return [
+            "clickhouse/dotnet-client",
+            "clickhouse/integration-helper",
+            "clickhouse/integration-test",
+            "clickhouse/integration-tests-runner",
+            "clickhouse/kerberized-hadoop",
+            "clickhouse/kerberos-kdc",
+            "clickhouse/mysql-golang-client",
+            "clickhouse/mysql-java-client",
+            "clickhouse/mysql-js-client",
+            "clickhouse/mysql-php-client",
+            "clickhouse/postgresql-java-client",
+        ]
 
     def _can_run_with(self, path, opt):
-        with open(path, 'r') as script:
+        with open(path, "r") as script:
             for line in script:
                 if opt in line:
                     return True
         return False
 
     def _install_clickhouse(self, debs_path):
-        for package in ('clickhouse-common-static_', 'clickhouse-server_', 'clickhouse-client', 'clickhouse-common-static-dbg_'):  # order matters
+        for package in (
+            "clickhouse-common-static_",
+            "clickhouse-server_",
+            "clickhouse-client",
+            "clickhouse-common-static-dbg_",
+        ):  # order matters
             logging.info("Installing package %s", package)
             for f in os.listdir(debs_path):
                 if package in f:
@@ -255,10 +290,12 @@ class ClickhouseIntegrationTestsRunner:
                     logging.info("Package found in %s", full_path)
                     log_name = "install_" + f + ".log"
                     log_path = os.path.join(str(self.path()), log_name)
-                    with open(log_path, 'w') as log:
+                    with open(log_path, "w") as log:
                         cmd = "dpkg -x {} .".format(full_path)
                         logging.info("Executing installation cmd %s", cmd)
-                        retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait()
+                        retcode = subprocess.Popen(
+                            cmd, shell=True, stderr=log, stdout=log
+                        ).wait()
                         if retcode == 0:
                             logging.info("Installation of %s successfull", full_path)
                         else:
@@ -267,18 +304,35 @@ class ClickhouseIntegrationTestsRunner:
             else:
                 raise Exception("Package with {} not found".format(package))
         logging.info("Unstripping binary")
-        # logging.info("Unstring %s", subprocess.check_output("eu-unstrip /usr/bin/clickhouse {}".format(CLICKHOUSE_BINARY_PATH), shell=True))
+        # logging.info(
+        #     "Unstring %s",
+        #     subprocess.check_output(
+        #         "eu-unstrip /usr/bin/clickhouse {}".format(CLICKHOUSE_BINARY_PATH),
+        #         shell=True,
+        #     ),
+        # )
 
         logging.info("All packages installed")
         os.chmod(CLICKHOUSE_BINARY_PATH, 0o777)
         os.chmod(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, 0o777)
         os.chmod(CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, 0o777)
-        shutil.copy(CLICKHOUSE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH"))
-        shutil.copy(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"))
-        shutil.copy(CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH"))
+        shutil.copy(
+            CLICKHOUSE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH")
+        )
+        shutil.copy(
+            CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH,
+            os.getenv("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"),
+        )
+        shutil.copy(
+            CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH,
+            os.getenv("CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH"),
+        )
 
     def _compress_logs(self, dir, relpaths, result_path):
-        subprocess.check_call("tar czf {} -C {} {}".format(result_path, dir, ' '.join(relpaths)), shell=True)  # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL
+        subprocess.check_call(  # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL
+            "tar czf {} -C {} {}".format(result_path, dir, " ".join(relpaths)),
+            shell=True,
+        )
 
     def _get_runner_opts(self):
         result = []
@@ -292,22 +346,40 @@ class ClickhouseIntegrationTestsRunner:
         image_cmd = self._get_runner_image_cmd(repo_path)
         out_file = "all_tests.txt"
         out_file_full = "all_tests_full.txt"
-        cmd = "cd {repo_path}/tests/integration && " \
-            "timeout -s 9 1h ./runner {runner_opts} {image_cmd} ' --setup-plan' " \
-            "| tee {out_file_full} | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' | sed 's/ *$//g' " \
+        cmd = (
+            "cd {repo_path}/tests/integration && "
+            "timeout -s 9 1h ./runner {runner_opts} {image_cmd} ' --setup-plan' "
+            "| tee {out_file_full} | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' | sed 's/ *$//g' "
             "| grep -v 'SKIPPED' | sort -u  > {out_file}".format(
-                repo_path=repo_path, runner_opts=self._get_runner_opts(), image_cmd=image_cmd, out_file=out_file, out_file_full=out_file_full)
+                repo_path=repo_path,
+                runner_opts=self._get_runner_opts(),
+                image_cmd=image_cmd,
+                out_file=out_file,
+                out_file_full=out_file_full,
+            )
+        )
 
         logging.info("Getting all tests with cmd '%s'", cmd)
-        subprocess.check_call(cmd, shell=True)  # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL
+        subprocess.check_call(  # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL
+            cmd, shell=True
+        )
 
-        all_tests_file_path = "{repo_path}/tests/integration/{out_file}".format(repo_path=repo_path, out_file=out_file)
-        if not os.path.isfile(all_tests_file_path) or os.path.getsize(all_tests_file_path) == 0:
-            all_tests_full_file_path = "{repo_path}/tests/integration/{out_file}".format(repo_path=repo_path, out_file=out_file_full)
+        all_tests_file_path = "{repo_path}/tests/integration/{out_file}".format(
+            repo_path=repo_path, out_file=out_file
+        )
+        if (
+            not os.path.isfile(all_tests_file_path)
+            or os.path.getsize(all_tests_file_path) == 0
+        ):
+            all_tests_full_file_path = (
+                "{repo_path}/tests/integration/{out_file}".format(
+                    repo_path=repo_path, out_file=out_file_full
+                )
+            )
             if os.path.isfile(all_tests_full_file_path):
                 # log runner output
                 logging.info("runner output:")
-                with open(all_tests_full_file_path, 'r') as all_tests_full_file:
+                with open(all_tests_full_file_path, "r") as all_tests_full_file:
                     for line in all_tests_full_file:
                         line = line.rstrip()
                         if line:
@@ -315,7 +387,11 @@ class ClickhouseIntegrationTestsRunner:
             else:
                 logging.info("runner output '%s' is empty", all_tests_full_file_path)
 
-            raise Exception("There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(all_tests_file_path))
+            raise Exception(
+                "There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(
+                    all_tests_file_path
+                )
+            )
 
         all_tests = []
         with open(all_tests_file_path, "r") as all_tests_file:
@@ -324,9 +400,18 @@ class ClickhouseIntegrationTestsRunner:
         return list(sorted(all_tests))
 
     def _get_parallel_tests_skip_list(self, repo_path):
-        skip_list_file_path = "{}/tests/integration/parallel_skip.json".format(repo_path)
-        if not os.path.isfile(skip_list_file_path) or os.path.getsize(skip_list_file_path) == 0:
-            raise Exception("There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(skip_list_file_path))
+        skip_list_file_path = "{}/tests/integration/parallel_skip.json".format(
+            repo_path
+        )
+        if (
+            not os.path.isfile(skip_list_file_path)
+            or os.path.getsize(skip_list_file_path) == 0
+        ):
+            raise Exception(
+                "There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(
+                    skip_list_file_path
+                )
+            )
 
         skip_list_tests = []
         with open(skip_list_file_path, "r") as skip_list_file:
@@ -336,7 +421,7 @@ class ClickhouseIntegrationTestsRunner:
     def group_test_by_file(self, tests):
         result = {}
         for test in tests:
-            test_file = test.split('::')[0]
+            test_file = test.split("::")[0]
             if test_file not in result:
                 result[test_file] = []
             result[test_file].append(test)
@@ -344,7 +429,10 @@ class ClickhouseIntegrationTestsRunner:
 
     def _update_counters(self, main_counters, current_counters):
         for test in current_counters["PASSED"]:
-            if test not in main_counters["PASSED"] and test not in main_counters["FLAKY"]:
+            if (
+                test not in main_counters["PASSED"]
+                and test not in main_counters["FLAKY"]
+            ):
                 is_flaky = False
                 if test in main_counters["FAILED"]:
                     main_counters["FAILED"].remove(test)
@@ -369,45 +457,63 @@ class ClickhouseIntegrationTestsRunner:
                     main_counters[state].append(test)
 
     def _get_runner_image_cmd(self, repo_path):
-        image_cmd = ''
-        if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-image-version'):
+        image_cmd = ""
+        if self._can_run_with(
+            os.path.join(repo_path, "tests/integration", "runner"),
+            "--docker-image-version",
+        ):
             for img in self.get_images_names():
                 if img == "clickhouse/integration-tests-runner":
                     runner_version = self.get_single_image_version()
-                    logging.info("Can run with custom docker image version %s", runner_version)
-                    image_cmd += ' --docker-image-version={} '.format(runner_version)
+                    logging.info(
+                        "Can run with custom docker image version %s", runner_version
+                    )
+                    image_cmd += " --docker-image-version={} ".format(runner_version)
                 else:
-                    if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-compose-images-tags'):
-                        image_cmd += '--docker-compose-images-tags={} '.format(self.get_image_with_version(img))
+                    if self._can_run_with(
+                        os.path.join(repo_path, "tests/integration", "runner"),
+                        "--docker-compose-images-tags",
+                    ):
+                        image_cmd += "--docker-compose-images-tags={} ".format(
+                            self.get_image_with_version(img)
+                        )
         else:
-            image_cmd = ''
+            image_cmd = ""
             logging.info("Cannot run with custom docker image version :(")
         return image_cmd
 
     def _find_test_data_dirs(self, repo_path, test_names):
         relpaths = {}
         for test_name in test_names:
-            if '/' in test_name:
-                test_dir = test_name[:test_name.find('/')]
+            if "/" in test_name:
+                test_dir = test_name[: test_name.find("/")]
             else:
                 test_dir = test_name
             if os.path.isdir(os.path.join(repo_path, "tests/integration", test_dir)):
-                for name in os.listdir(os.path.join(repo_path, "tests/integration", test_dir)):
+                for name in os.listdir(
+                    os.path.join(repo_path, "tests/integration", test_dir)
+                ):
                     relpath = os.path.join(os.path.join(test_dir, name))
-                    mtime = os.path.getmtime(os.path.join(repo_path, "tests/integration", relpath))
+                    mtime = os.path.getmtime(
+                        os.path.join(repo_path, "tests/integration", relpath)
+                    )
                     relpaths[relpath] = mtime
         return relpaths
 
     def _get_test_data_dirs_difference(self, new_snapshot, old_snapshot):
         res = set()
         for path in new_snapshot:
-            if (not path in old_snapshot) or (old_snapshot[path] != new_snapshot[path]):
+            if (path not in old_snapshot) or (old_snapshot[path] != new_snapshot[path]):
                 res.add(path)
         return res
 
-    def try_run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers):
+    def try_run_test_group(
+        self, repo_path, test_group, tests_in_group, num_tries, num_workers
+    ):
         try:
-            return self.run_test_group(repo_path, test_group, tests_in_group, num_tries, num_workers)
+            return self.run_test_group(
+                repo_path, test_group, tests_in_group, num_tries, num_workers
+            )
         except Exception as e:
             logging.info("Failed to run {}:\n{}".format(str(test_group), str(e)))
             counters = {
@@ -423,7 +529,9 @@ class ClickhouseIntegrationTestsRunner:
                 tests_times[test] = 0
             return counters, tests_times, []
 
-    def run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers):
+    def run_test_group(
+        self, repo_path, test_group, tests_in_group, num_tries, num_workers
+    ):
         counters = {
             "ERROR": [],
             "PASSED": [],
@@ -441,7 +549,7 @@ class ClickhouseIntegrationTestsRunner:
             return counters, tests_times, []
 
         image_cmd = self._get_runner_image_cmd(repo_path)
-        test_group_str = test_group.replace('/', '_').replace('.', '_')
+        test_group_str = test_group.replace("/", "_").replace(".", "_")
 
         log_paths = []
         test_data_dirs = {}
@@ -453,8 +561,8 @@ class ClickhouseIntegrationTestsRunner:
             test_names = set([])
             for test_name in tests_in_group:
                 if test_name not in counters["PASSED"]:
-                    if '[' in test_name:
-                        test_names.add(test_name[:test_name.find('[')])
+                    if "[" in test_name:
+                        test_names.add(test_name[: test_name.find("[")])
                     else:
                         test_names.add(test_name)
 
@@ -464,47 +572,83 @@ class ClickhouseIntegrationTestsRunner:
             info_basename = test_group_str + "_" + str(i) + ".nfo"
             info_path = os.path.join(repo_path, "tests/integration", info_basename)
 
-            test_cmd = ' '.join([test for test in sorted(test_names)])
-            parallel_cmd = " --parallel {} ".format(num_workers) if num_workers > 0 else ""
+            test_cmd = " ".join([test for test in sorted(test_names)])
+            parallel_cmd = (
+                " --parallel {} ".format(num_workers) if num_workers > 0 else ""
+            )
             # -r -- show extra test summary:
             # -f -- (f)ailed
             # -E -- (E)rror
             # -p -- (p)assed
             # -s -- (s)kipped
             cmd = "cd {}/tests/integration && timeout -s 9 1h ./runner {} {} -t {} {} '-rfEps --run-id={} --color=no --durations=0 {}' | tee {}".format(
-                repo_path, self._get_runner_opts(), image_cmd, test_cmd, parallel_cmd, i, _get_deselect_option(self.should_skip_tests()), info_path)
+                repo_path,
+                self._get_runner_opts(),
+                image_cmd,
+                test_cmd,
+                parallel_cmd,
+                i,
+                _get_deselect_option(self.should_skip_tests()),
+                info_path,
+            )
 
             log_basename = test_group_str + "_" + str(i) + ".log"
             log_path = os.path.join(repo_path, "tests/integration", log_basename)
-            with open(log_path, 'w') as log:
+            with open(log_path, "w") as log:
                 logging.info("Executing cmd: %s", cmd)
-                retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait()
+                retcode = subprocess.Popen(
+                    cmd, shell=True, stderr=log, stdout=log
+                ).wait()
                 if retcode == 0:
                     logging.info("Run %s group successfully", test_group)
                 else:
                     logging.info("Some tests failed")
 
             extra_logs_names = [log_basename]
-            log_result_path = os.path.join(str(self.path()), 'integration_run_' + log_basename)
+            log_result_path = os.path.join(
+                str(self.path()), "integration_run_" + log_basename
+            )
             shutil.copy(log_path, log_result_path)
             log_paths.append(log_result_path)
 
-            for pytest_log_path in glob.glob(os.path.join(repo_path, "tests/integration/pytest*.log")):
-                new_name = test_group_str + "_" + str(i) + "_" + os.path.basename(pytest_log_path)
-                os.rename(pytest_log_path, os.path.join(repo_path, "tests/integration", new_name))
+            for pytest_log_path in glob.glob(
+                os.path.join(repo_path, "tests/integration/pytest*.log")
+            ):
+                new_name = (
+                    test_group_str
+                    + "_"
+                    + str(i)
+                    + "_"
+                    + os.path.basename(pytest_log_path)
+                )
+                os.rename(
+                    pytest_log_path,
+                    os.path.join(repo_path, "tests/integration", new_name),
+                )
                 extra_logs_names.append(new_name)
 
             dockerd_log_path = os.path.join(repo_path, "tests/integration/dockerd.log")
             if os.path.exists(dockerd_log_path):
-                new_name = test_group_str + "_" + str(i) + "_" + os.path.basename(dockerd_log_path)
-                os.rename(dockerd_log_path, os.path.join(repo_path, "tests/integration", new_name))
+                new_name = (
+                    test_group_str
+                    + "_"
+                    + str(i)
+                    + "_"
+                    + os.path.basename(dockerd_log_path)
+                )
+                os.rename(
+                    dockerd_log_path,
+                    os.path.join(repo_path, "tests/integration", new_name),
+                )
                 extra_logs_names.append(new_name)
 
             if os.path.exists(info_path):
                 extra_logs_names.append(info_basename)
                 new_counters = get_counters(info_path)
                 for state, tests in new_counters.items():
-                    logging.info("Tests with %s state (%s): %s", state, len(tests), tests)
+                    logging.info(
+                        "Tests with %s state (%s): %s", state, len(tests), tests
+                    )
                 times_lines = parse_test_times(info_path)
                 new_tests_times = get_test_times(times_lines)
                 self._update_counters(counters, new_counters)
@@ -512,19 +656,35 @@ class ClickhouseIntegrationTestsRunner:
                     tests_times[test_name] = test_time
 
             test_data_dirs_new = self._find_test_data_dirs(repo_path, test_names)
-            test_data_dirs_diff = self._get_test_data_dirs_difference(test_data_dirs_new, test_data_dirs)
+            test_data_dirs_diff = self._get_test_data_dirs_difference(
+                test_data_dirs_new, test_data_dirs
+            )
             test_data_dirs = test_data_dirs_new
 
             if extra_logs_names or test_data_dirs_diff:
-                extras_result_path = os.path.join(str(self.path()), "integration_run_" + test_group_str + "_" + str(i) + ".tar.gz")
-                self._compress_logs(os.path.join(repo_path, "tests/integration"), extra_logs_names + list(test_data_dirs_diff), extras_result_path)
+                extras_result_path = os.path.join(
+                    str(self.path()),
+                    "integration_run_" + test_group_str + "_" + str(i) + ".tar.gz",
+                )
+                self._compress_logs(
+                    os.path.join(repo_path, "tests/integration"),
+                    extra_logs_names + list(test_data_dirs_diff),
+                    extras_result_path,
+                )
                 log_paths.append(extras_result_path)
 
             if len(counters["PASSED"]) + len(counters["FLAKY"]) == len(tests_in_group):
                 logging.info("All tests from group %s passed", test_group)
                 break
-            if len(counters["PASSED"]) + len(counters["FLAKY"]) >= 0 and len(counters["FAILED"]) == 0 and len(counters["ERROR"]) == 0:
-                logging.info("Seems like all tests passed but some of them are skipped or deselected. Ignoring them and finishing group.")
+            if (
+                len(counters["PASSED"]) + len(counters["FLAKY"]) >= 0
+                and len(counters["FAILED"]) == 0
+                and len(counters["ERROR"]) == 0
+            ):
+                logging.info(
+                    "Seems like all tests passed but some of them are skipped or "
+                    "deselected. Ignoring them and finishing group."
+                )
                 break
         else:
             # Mark all non tried tests as errors, with '::' in name
@@ -532,26 +692,28 @@ class ClickhouseIntegrationTestsRunner:
             # we run whole test dirs like "test_odbc_interaction" and don't
             # want to mark them as error so we filter by '::'.
             for test in tests_in_group:
-                if (test not in counters["PASSED"] and
-                    test not in counters["ERROR"] and
-                    test not in counters["SKIPPED"] and
-                    test not in counters["FAILED"] and
-                    '::' in test):
+                if (
+                    test not in counters["PASSED"]
+                    and test not in counters["ERROR"]
+                    and test not in counters["SKIPPED"]
+                    and test not in counters["FAILED"]
+                    and "::" in test
+                ):
                     counters["ERROR"].append(test)
 
         return counters, tests_times, log_paths
 
     def run_flaky_check(self, repo_path, build_path):
-        pr_info = self.params['pr_info']
+        pr_info = self.params["pr_info"]
 
         # pytest swears, if we require to run some tests which was renamed or deleted
         tests_to_run = filter_existing_tests(get_tests_to_run(pr_info), repo_path)
         if not tests_to_run:
             logging.info("No tests to run found")
-            return 'success', 'Nothing to run', [('Nothing to run', 'OK')], ''
+            return "success", "Nothing to run", [("Nothing to run", "OK")], ""
 
         self._install_clickhouse(build_path)
-        logging.info("Found '%s' tests to run", ' '.join(tests_to_run))
+        logging.info("Found '%s' tests to run", " ".join(tests_to_run))
         result_state = "success"
         description_prefix = "No flaky tests: "
         start = time.time()
@@ -561,17 +723,20 @@ class ClickhouseIntegrationTestsRunner:
         for i in range(TRIES_COUNT):
             final_retry += 1
             logging.info("Running tests for the %s time", i)
-            counters, tests_times, log_paths = self.try_run_test_group(repo_path, "flaky", tests_to_run, 1, 1)
+            counters, tests_times, log_paths = self.try_run_test_group(
+                repo_path, "flaky", tests_to_run, 1, 1
+            )
             logs += log_paths
             if counters["FAILED"]:
-                logging.info("Found failed tests: %s", ' '.join(counters["FAILED"]))
+                logging.info("Found failed tests: %s", " ".join(counters["FAILED"]))
                 description_prefix = "Flaky tests found: "
                 result_state = "failure"
                 break
             if counters["ERROR"]:
                 description_prefix = "Flaky tests found: "
-                logging.info("Found error tests: %s", ' '.join(counters["ERROR"]))
-                # NOTE "error" result state will restart the whole test task, so we use "failure" here
+                logging.info("Found error tests: %s", " ".join(counters["ERROR"]))
+                # NOTE "error" result state will restart the whole test task,
+                # so we use "failure" here
                 result_state = "failure"
                 break
             assert len(counters["FLAKY"]) == 0
@@ -591,8 +756,20 @@ class ClickhouseIntegrationTestsRunner:
                 text_state = "FAIL"
             else:
                 text_state = state
-            test_result += [(c + ' (✕' + str(final_retry) + ')', text_state, "{:.2f}".format(tests_times[c])) for c in counters[state]]
-        status_text = description_prefix + ', '.join([str(n).lower().replace('failed', 'fail') + ': ' + str(len(c)) for n, c in counters.items()])
+            test_result += [
+                (
+                    c + " (✕" + str(final_retry) + ")",
+                    text_state,
+                    "{:.2f}".format(tests_times[c]),
+                )
+                for c in counters[state]
+            ]
+        status_text = description_prefix + ", ".join(
+            [
+                str(n).lower().replace("failed", "fail") + ": " + str(len(c))
+                for n, c in counters.items()
+            ]
+        )
 
         return result_state, status_text, test_result, logs
 
@@ -601,7 +778,10 @@ class ClickhouseIntegrationTestsRunner:
             return self.run_flaky_check(repo_path, build_path)
 
         self._install_clickhouse(build_path)
-        logging.info("Dump iptables before run %s", subprocess.check_output("sudo iptables -L", shell=True))
+        logging.info(
+            "Dump iptables before run %s",
+            subprocess.check_output("sudo iptables -L", shell=True),
+        )
         all_tests = self._get_all_tests(repo_path)
 
         if self.run_by_hash_total != 0:
@@ -613,18 +793,36 @@ class ClickhouseIntegrationTestsRunner:
             all_tests = all_filtered_by_hash_tests
 
         parallel_skip_tests = self._get_parallel_tests_skip_list(repo_path)
-        logging.info("Found %s tests first 3 %s", len(all_tests), ' '.join(all_tests[:3]))
-        filtered_sequential_tests = list(filter(lambda test: test in all_tests, parallel_skip_tests))
-        filtered_parallel_tests = list(filter(lambda test: test not in parallel_skip_tests, all_tests))
-        not_found_tests = list(filter(lambda test: test not in all_tests, parallel_skip_tests))
-        logging.info("Found %s tests first 3 %s, parallel %s, other %s", len(all_tests), ' '.join(all_tests[:3]), len(filtered_parallel_tests), len(filtered_sequential_tests))
-        logging.info("Not found %s tests first 3 %s", len(not_found_tests), ' '.join(not_found_tests[:3]))
+        logging.info(
+            "Found %s tests first 3 %s", len(all_tests), " ".join(all_tests[:3])
+        )
+        filtered_sequential_tests = list(
+            filter(lambda test: test in all_tests, parallel_skip_tests)
+        )
+        filtered_parallel_tests = list(
+            filter(lambda test: test not in parallel_skip_tests, all_tests)
+        )
+        not_found_tests = list(
+            filter(lambda test: test not in all_tests, parallel_skip_tests)
+        )
+        logging.info(
+            "Found %s tests first 3 %s, parallel %s, other %s",
+            len(all_tests),
+            " ".join(all_tests[:3]),
+            len(filtered_parallel_tests),
+            len(filtered_sequential_tests),
+        )
+        logging.info(
+            "Not found %s tests first 3 %s",
+            len(not_found_tests),
+            " ".join(not_found_tests[:3]),
+        )
 
         grouped_tests = self.group_test_by_file(filtered_sequential_tests)
         i = 0
         for par_group in chunks(filtered_parallel_tests, PARALLEL_GROUP_SIZE):
-            grouped_tests["parallel{}".format(i)] = par_group
-            i+=1
+            grouped_tests[f"parallel{i}"] = par_group
+            i += 1
         logging.info("Found %s tests groups", len(grouped_tests))
 
         counters = {
@@ -646,12 +844,18 @@ class ClickhouseIntegrationTestsRunner:
 
         for group, tests in items_to_run:
             logging.info("Running test group %s containing %s tests", group, len(tests))
-            group_counters, group_test_times, log_paths = self.try_run_test_group(repo_path, group, tests, MAX_RETRY, NUM_WORKERS)
+            group_counters, group_test_times, log_paths = self.try_run_test_group(
+                repo_path, group, tests, MAX_RETRY, NUM_WORKERS
+            )
             total_tests = 0
             for counter, value in group_counters.items():
-                logging.info("Tests from group %s stats, %s count %s", group, counter, len(value))
+                logging.info(
+                    "Tests from group %s stats, %s count %s", group, counter, len(value)
+                )
                 counters[counter] += value
-                logging.info("Totally have %s with status %s", len(counters[counter]), counter)
+                logging.info(
+                    "Totally have %s with status %s", len(counters[counter]), counter
+                )
                 total_tests += len(counters[counter])
             logging.info("Totally finished tests %s/%s", total_tests, len(all_tests))
 
@@ -664,7 +868,9 @@ class ClickhouseIntegrationTestsRunner:
                 break
 
         if counters["FAILED"] or counters["ERROR"]:
-            logging.info("Overall status failure, because we have tests in FAILED or ERROR state")
+            logging.info(
+                "Overall status failure, because we have tests in FAILED or ERROR state"
+            )
             result_state = "failure"
         else:
             logging.info("Overall success!")
@@ -678,42 +884,49 @@ class ClickhouseIntegrationTestsRunner:
                 text_state = "FAIL"
             else:
                 text_state = state
-            test_result += [(c, text_state, "{:.2f}".format(tests_times[c]), tests_log_paths[c]) for c in counters[state]]
+            test_result += [
+                (c, text_state, "{:.2f}".format(tests_times[c]), tests_log_paths[c])
+                for c in counters[state]
+            ]
 
-        failed_sum = len(counters['FAILED']) + len(counters['ERROR'])
-        status_text = "fail: {}, passed: {}, flaky: {}".format(failed_sum, len(counters['PASSED']), len(counters['FLAKY']))
+        failed_sum = len(counters["FAILED"]) + len(counters["ERROR"])
+        status_text = "fail: {}, passed: {}, flaky: {}".format(
+            failed_sum, len(counters["PASSED"]), len(counters["FLAKY"])
+        )
 
         if self.soft_deadline_time < time.time():
             status_text = "Timeout, " + status_text
             result_state = "failure"
 
-        counters['FLAKY'] = []
+        counters["FLAKY"] = []
         if not counters or sum(len(counter) for counter in counters.values()) == 0:
             status_text = "No tests found for some reason! It's a bug"
             result_state = "failure"
 
-        if '(memory)' in self.params['context_name']:
+        if "(memory)" in self.params["context_name"]:
             result_state = "success"
 
         return result_state, status_text, test_result, []
 
+
 def write_results(results_file, status_file, results, status):
-    with open(results_file, 'w') as f:
-        out = csv.writer(f, delimiter='\t')
+    with open(results_file, "w") as f:
+        out = csv.writer(f, delimiter="\t")
         out.writerows(results)
-    with open(status_file, 'w') as f:
-        out = csv.writer(f, delimiter='\t')
+    with open(status_file, "w") as f:
+        out = csv.writer(f, delimiter="\t")
         out.writerow(status)
 
+
 if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
 
     repo_path = os.environ.get("CLICKHOUSE_TESTS_REPO_PATH")
     build_path = os.environ.get("CLICKHOUSE_TESTS_BUILD_PATH")
     result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH")
     params_path = os.environ.get("CLICKHOUSE_TESTS_JSON_PARAMS_PATH")
 
-    params = json.loads(open(params_path, 'r').read())
+    params = json.loads(open(params_path, "r").read())
     runner = ClickhouseIntegrationTestsRunner(result_path, params)
 
     logging.info("Running tests")

tests/integration/runner

@@ -238,6 +238,8 @@ if __name__ == "__main__":
                 env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag)
             elif image == "clickhouse/integration-test":
                 env_tags += "-e {}={} ".format("DOCKER_BASE_TAG", tag)
+            elif image == "clickhouse/kerberized-hadoop":
+                env_tags += "-e {}={} ".format("DOCKER_KERBERIZED_HADOOP_TAG", tag)
             elif image == "clickhouse/kerberos-kdc":
                 env_tags += "-e {}={} ".format("DOCKER_KERBEROS_KDC_TAG", tag)
             else:

tests/integration/test_grant_and_revoke/test.py

@@ -150,8 +150,11 @@ def test_grant_all_on_table():
     instance.query("CREATE USER A, B")
     instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION")
     instance.query("GRANT ALL ON test.table TO B", user='A')
-    assert instance.query(
-        "SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
+    assert instance.query("SHOW GRANTS FOR B") ==\
+        "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, "\
+        "DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, "\
+        "SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, "\
+        "SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
     instance.query("REVOKE ALL ON test.table FROM B", user='A')
     assert instance.query("SHOW GRANTS FOR B") == ""
 

tests/integration/test_grpc_protocol/test.py

@@ -2,6 +2,8 @@ import os
 import pytest
 import sys
 import time
+import pytz
+import uuid
 import grpc
 from helpers.cluster import ClickHouseCluster, run_and_check
 from threading import Thread
@@ -43,8 +45,8 @@ def create_channel():
         main_channel = channel
     return channel
 
-def query_common(query_text, settings={}, input_data=[], input_data_delimiter='', output_format='TabSeparated', external_tables=[],
-                 user_name='', password='', query_id='123', session_id='', stream_output=False, channel=None):
+def query_common(query_text, settings={}, input_data=[], input_data_delimiter='', output_format='TabSeparated', send_output_columns=False,
+                 external_tables=[], user_name='', password='', query_id='123', session_id='', stream_output=False, channel=None):
     if type(input_data) is not list:
         input_data = [input_data]
     if type(input_data_delimiter) is str:
@@ -58,7 +60,8 @@ def query_common(query_text, settings={}, input_data=[], input_data_delimiter=''
             input_data_part = input_data_part.encode(DEFAULT_ENCODING)
         return clickhouse_grpc_pb2.QueryInfo(query=query_text, settings=settings, input_data=input_data_part,
                                              input_data_delimiter=input_data_delimiter, output_format=output_format,
-                                             external_tables=external_tables, user_name=user_name, password=password, query_id=query_id,
+                                             send_output_columns=send_output_columns, external_tables=external_tables,
+                                             user_name=user_name, password=password, query_id=query_id,
                                              session_id=session_id, next_query_info=bool(input_data))
     def send_query_info():
         yield query_info()
@@ -204,6 +207,28 @@ def test_totals_and_extremes():
     assert query("SELECT x, y FROM t") == "1\t2\n2\t4\n3\t2\n3\t3\n3\t4\n"
     assert query_and_get_extremes("SELECT x, y FROM t", settings={"extremes": "1"}) == "1\t2\n3\t4\n"
 
+def test_get_query_details():
+    result = list(query_no_errors("CREATE TABLE t (a UInt8) ENGINE = Memory", query_id = '123'))[0]
+    assert result.query_id == '123'
+    pytz.timezone(result.time_zone)
+    assert result.output_format == ''
+    assert len(result.output_columns) == 0
+    assert result.output == b''
+    #
+    result = list(query_no_errors("SELECT 'a', 1", query_id = '', output_format = 'TabSeparated'))[0]
+    uuid.UUID(result.query_id)
+    pytz.timezone(result.time_zone)
+    assert result.output_format == 'TabSeparated'
+    assert len(result.output_columns) == 0
+    assert result.output == b'a\t1\n'
+    #
+    result = list(query_no_errors("SELECT 'a' AS x, 1 FORMAT JSONEachRow", query_id = '', send_output_columns=True))[0]
+    uuid.UUID(result.query_id)
+    pytz.timezone(result.time_zone)
+    assert result.output_format == 'JSONEachRow'
+    assert ([(col.name, col.type) for col in result.output_columns]) == [('x', 'String'), ('1', 'UInt8')]
+    assert result.output == b'{"x":"a","1":1}\n'
+
 def test_errors_handling():
     e = query_and_get_error("")
     #print(e)
@@ -225,6 +250,9 @@ def test_logs():
 
 def test_progress():
     results = query_no_errors("SELECT number, sleep(0.31) FROM numbers(8) SETTINGS max_block_size=2, interactive_delay=100000", stream_output=True)
+    for result in results:
+        result.time_zone = ''
+        result.query_id = ''
     #print(results)
     assert str(results) ==\
 """[progress {
@@ -232,6 +260,7 @@ def test_progress():
   read_bytes: 16
   total_rows_to_read: 8
 }
+output_format: "TabSeparated"
 , output: "0\\t0\\n1\\t0\\n"
 , progress {
   read_rows: 2

tests/integration/test_row_policy/test.py

@@ -389,6 +389,52 @@ def test_dcl_management():
     assert node.query("SHOW POLICIES") == ""
 
 
+def test_grant_create_row_policy():
+    copy_policy_xml('no_filters.xml')
+    assert node.query("SHOW POLICIES") == ""
+    node.query("CREATE USER X")
+
+    expected_error = "necessary to have grant CREATE ROW POLICY ON mydb.filtered_table1"
+    assert expected_error in node.query_and_get_error("CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user='X')
+    node.query("GRANT CREATE POLICY ON mydb.filtered_table1 TO X")
+    node.query("CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user='X')
+    expected_error = "necessary to have grant CREATE ROW POLICY ON mydb.filtered_table2"
+    assert expected_error in node.query_and_get_error("CREATE POLICY pA ON mydb.filtered_table2 FOR SELECT USING a<b", user='X')
+
+    expected_error = "necessary to have grant ALTER ROW POLICY ON mydb.filtered_table1"
+    assert expected_error in node.query_and_get_error("ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user='X')
+    node.query("GRANT ALTER POLICY ON mydb.filtered_table1 TO X")
+    node.query("ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user='X')
+    expected_error = "necessary to have grant ALTER ROW POLICY ON mydb.filtered_table2"
+    assert expected_error in node.query_and_get_error("ALTER POLICY pA ON mydb.filtered_table2 FOR SELECT USING a==b", user='X')
+
+    expected_error = "necessary to have grant DROP ROW POLICY ON mydb.filtered_table1"
+    assert expected_error in node.query_and_get_error("DROP POLICY pA ON mydb.filtered_table1", user='X')
+    node.query("GRANT DROP POLICY ON mydb.filtered_table1 TO X")
+    node.query("DROP POLICY pA ON mydb.filtered_table1", user='X')
+    expected_error = "necessary to have grant DROP ROW POLICY ON mydb.filtered_table2"
+    assert expected_error in node.query_and_get_error("DROP POLICY pA ON mydb.filtered_table2", user='X')
+
+    node.query("REVOKE ALL ON *.* FROM X")
+
+    expected_error = "necessary to have grant CREATE ROW POLICY ON mydb.filtered_table1"
+    assert expected_error in node.query_and_get_error("CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user='X')
+    node.query("GRANT CREATE POLICY ON *.* TO X")
+    node.query("CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user='X')
+    
+    expected_error = "necessary to have grant ALTER ROW POLICY ON mydb.filtered_table1"
+    assert expected_error in node.query_and_get_error("ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user='X')
+    node.query("GRANT ALTER POLICY ON *.* TO X")
+    node.query("ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user='X')
+
+    expected_error = "necessary to have grant DROP ROW POLICY ON mydb.filtered_table1"
+    assert expected_error in node.query_and_get_error("DROP POLICY pA ON mydb.filtered_table1", user='X')
+    node.query("GRANT DROP POLICY ON *.* TO X")
+    node.query("DROP POLICY pA ON mydb.filtered_table1", user='X')
+
+    node.query("DROP USER X")
+
+
 def test_users_xml_is_readonly():
     assert re.search("storage is readonly", node.query_and_get_error("DROP POLICY default ON mydb.filtered_table1"))
 

tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh

@@ -90,6 +90,7 @@ create_admin_user() {
 }
 
 create_keytabs() {
+  rm /tmp/keytab/*.keytab
 
 
   # kadmin.local -q "addprinc -randkey hdfs/kerberizedhdfs1.${DOMAIN_REALM}@${REALM}"

tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh

@@ -90,6 +90,7 @@ create_admin_user() {
 }
 
 create_keytabs() {
+  rm /tmp/keytab/*.keytab
 
   kadmin.local -q "addprinc -randkey zookeeper/kafka_kerberized_zookeeper@${REALM}"
   kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kafka_kerberized_zookeeper.keytab zookeeper/kafka_kerberized_zookeeper@${REALM}"

tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml

@@ -76,6 +76,14 @@
             </volumes>
         </jbod1_with_jbod2>
 
+        <only_jbod_1>
+            <volumes>
+                <main>
+                    <disk>jbod1</disk>
+                </main>
+            </volumes>
+        </only_jbod_1>
+
         <only_jbod2>
             <volumes>
                 <main>

tests/integration/test_ttl_move/test.py

@@ -1174,3 +1174,57 @@ def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine):
             node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name))
         except:
             pass
+
+
+@pytest.mark.parametrize("name,dest_type", [
+    pytest.param("replicated_mt_move_if_exists", "DISK", id="replicated_disk"),
+    pytest.param("replicated_mt_move_if_exists", "VOLUME", id="replicated_volume"),
+])
+def test_ttl_move_if_exists(started_cluster, name, dest_type):
+    name = unique_table_name(name)
+
+    try:
+        query_template = """
+            CREATE TABLE {name} (
+                s1 String,
+                d1 DateTime
+            ) ENGINE = ReplicatedMergeTree('/clickhouse/replicated_mt_move_if_exists', '{node_name}')
+            ORDER BY tuple()
+            TTL d1 TO {dest_type} {if_exists} 'external'
+            SETTINGS storage_policy='{policy}'
+        """
+
+        with pytest.raises(QueryRuntimeException):
+            node1.query(query_template.format( \
+                name=name, node_name=node1.name, dest_type=dest_type, \
+                if_exists='', policy='only_jbod_1'))
+
+        for (node, policy) in zip([node1, node2], ['only_jbod_1', 'small_jbod_with_external']):
+            node.query(query_template.format( \
+                name=name, node_name=node.name, dest_type=dest_type, \
+                if_exists='IF EXISTS', policy=policy))
+
+        data = []  # 10MB in total
+        for i in range(10):
+            data.append(("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time.time() - 1)))
+
+        node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data])))
+        node2.query("SYSTEM SYNC REPLICA {}".format(name))
+
+        time.sleep(5)
+
+        used_disks1 = get_used_disks_for_table(node1, name)
+        assert set(used_disks1) == {"jbod1"}
+
+        used_disks2 = get_used_disks_for_table(node2, name)
+        assert set(used_disks2) == {"external"}
+
+        assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10"
+        assert node2.query("SELECT count() FROM {name}".format(name=name)).strip() == "10"
+
+    finally:
+        try:
+            node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name))
+            node2.query("DROP TABLE IF EXISTS {} NO DELAY".format(name))
+        except:
+            pass

tests/performance/file_table_function.xml

@@ -0,0 +1,54 @@
+<test>
+
+    <substitutions>
+        <substitution>
+            <name>format</name>
+            <values>
+                <value>TabSeparated</value>
+                <value>TabSeparatedWithNames</value>
+                <value>TabSeparatedWithNamesAndTypes</value>
+                <value>CSV</value>
+                <value>CSVWithNames</value>
+                <value>Values</value>
+                <value>JSONEachRow</value>
+                <value>JSONCompactEachRow</value>
+                <value>JSONCompactEachRowWithNamesAndTypes</value>
+                <value>TSKV</value>
+                <value>RowBinary</value>
+                <value>Native</value>
+                <value>MsgPack</value>
+            </values>
+        </substitution>
+        <substitution>
+            <name>partitions_count</name>
+            <values>
+                <value>5</value>
+                <value>50</value>
+                <value>500</value>
+            </values>
+        </substitution>
+    </substitutions>
+
+    <query>
+        INSERT INTO FUNCTION file('test_file', '{format}', 'key UInt64, value UInt64')
+        SELECT number, number FROM numbers(1000000)
+    </query>
+
+    <query>
+        INSERT INTO FUNCTION file('test_file', '{format}', 'key UInt64, value1 UInt64, value2 UInt64, value3 UInt64, value4 UInt64, value5 UInt64')
+        SELECT number, number, number, number, number, number FROM numbers(1000000)
+    </query>
+
+    <query>
+        INSERT INTO FUNCTION file('test_file_{{_partition_id}}', '{format}', 'partition_id UInt64, value UInt64')
+        PARTITION BY partition_id
+        SELECT (number % {partitions_count}) as partition_id, number FROM numbers(1000000)
+    </query>
+
+    <query>
+        INSERT INTO FUNCTION file('test_file_{{_partition_id}}', '{format}', 'partition_id UInt64, value1 UInt64, value2 UInt64, value3 UInt64, value4 UInt64, value5 UInt64')
+        PARTITION BY partition_id
+        SELECT (number % {partitions_count}) as partition_id, number, number, number, number, number FROM numbers(1000000)
+    </query>
+
+</test>

tests/queries/0_stateless/00702_join_on_dups.sql

@@ -33,7 +33,7 @@ select s.*, j.* from (select * from X) as s right join (select * from Y) as j on
 select 'full';
 select X.*, Y.* from X full join Y on X.id = Y.id order by X.id, X.x_a, X.x_b, Y.id, Y.y_a, Y.y_b;
 select 'full subs';
-select s.*, j.* from (select * from X) as s full join (select * from Y) as j on s.id = j.id order by s.id, s.x_a;
+select s.*, j.* from (select * from X) as s full join (select * from Y) as j on s.id = j.id order by s.id, s.x_a, s.x_b, j.id, j.y_a, j.y_b;
 --select 'full expr';
 --select X.*, Y.* from X full join Y on (X.id + 1) = (Y.id + 1) order by id;
 
@@ -48,14 +48,14 @@ select 'self inner nullable vs not nullable 2';
 select Y.*, s.* from Y inner join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by Y.id, Y.y_a, Y.y_b, s.id, s.y_a, s.y_b;
 
 select 'self left';
-select X.*, s.* from X left join (select * from X) as s on X.id = s.id order by X.id, X.x_a, s.x_a;
+select X.*, s.* from X left join (select * from X) as s on X.id = s.id order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b;
 select 'self left nullable';
-select X.*, s.* from X left join (select * from X) as s on X.x_b = s.x_b order by X.id, X.x_a;
+select X.*, s.* from X left join (select * from X) as s on X.x_b = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b;
 select 'self left nullable vs not nullable';
-select X.*, s.* from X left join (select * from X) as s on X.id = s.x_b order by X.id, X.x_a;
+select X.*, s.* from X left join (select * from X) as s on X.id = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b;
 -- TODO: s.y_b == '' instead of NULL
 select 'self left nullable vs not nullable 2';
-select Y.*, s.* from Y left join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by Y.id, Y.y_a;
+select Y.*, s.* from Y left join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by Y.id, Y.y_a, Y.y_b, s.id, s.y_a, s.y_b;
 
 select 'self right';
 select X.*, s.* from X right join (select * from X) as s on X.id = s.id order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b;

tests/queries/0_stateless/00702_join_with_using_dups.sql

@@ -23,7 +23,7 @@ select 'right subs';
 select s.*, j.* from (select * from X) as s right join (select * from Y) as j using id order by s.id, j.id, s.x_name, j.y_name;
 
 select 'full';
-select X.*, Y.* from X full join Y using id order by X.id, Y.id;
+select X.*, Y.* from X full join Y using id order by X.id, Y.id, X.x_name, Y.y_name;
 select 'full subs';
 select s.*, j.* from (select * from X) as s full join (select * from Y) as j using id order by s.id, j.id, s.x_name, j.y_name;
 

tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh

@@ -30,7 +30,6 @@ $CLICKHOUSE_CLIENT $settings -q "$touching_many_parts_query" &> /dev/null
 
 $CLICKHOUSE_CLIENT $settings -q "SYSTEM FLUSH LOGS"
 
-
-$CLICKHOUSE_CLIENT $settings -q "SELECT ProfileEvents['FileOpen'] FROM system.query_log WHERE query='$touching_many_parts_query' and current_database = currentDatabase()  ORDER BY event_time DESC LIMIT 1;"
+$CLICKHOUSE_CLIENT $settings -q "SELECT ProfileEvents['FileOpen'] as opened_files FROM system.query_log WHERE query='$touching_many_parts_query' and current_database = currentDatabase() ORDER BY event_time DESC, opened_files DESC LIMIT 1;"
 
 $CLICKHOUSE_CLIENT $settings -q "DROP TABLE IF EXISTS merge_tree_table;"

tests/queries/0_stateless/00926_geo_to_h3.reference

@@ -4,12 +4,12 @@
 644325528491955313
 644325528627451570
 644325529094369568
-644325528491955313
+639821928864584823
 644325528491955313
 644325528491955313
 644325528627451570
 644325529094369568
-55.720762	37.598135	644325528491955313
+55.720762	37.598135	639821928864584823
 55.720762	37.598135	644325528491955313
 55.72076201	37.598135	644325528491955313
 55.763241	37.660183	644325528627451570

tests/queries/0_stateless/00926_geo_to_h3.sql

@@ -11,9 +11,10 @@ INSERT INTO table1 VALUES(55.72076201, 37.59813500, 15);
 INSERT INTO table1 VALUES(55.72076200, 37.59813500, 14);
 
 select geoToH3(37.63098076, 55.77922738, 15);
+select geoToH3(37.63098076, 55.77922738, 24); -- { serverError 69 }
 select geoToH3(lon, lat, resolution) from table1 order by lat, lon, resolution;
-select geoToH3(lon, lat, 15) AS k from table1 order by lat, lon, k;
-select lat, lon, geoToH3(lon, lat, 15) AS k from table1 order by lat, lon, k;
+select geoToH3(lon, lat, resolution) AS k from table1 order by lat, lon, k;
+select lat, lon, geoToH3(lon, lat, resolution) AS k from table1 order by lat, lon, k;
 select geoToH3(lon, lat, resolution) AS k, count(*) from table1 group by k order by k;
 
 DROP TABLE table1

tests/queries/0_stateless/01030_limit_by_with_ties_error.sh

@@ -37,5 +37,5 @@ $CLICKHOUSE_CLIENT --query="""
 				('upyachka', 'a'), ('test', 'b'), ('foo', 'c'),
 				('bar', 'd'))
 		ORDER BY Payload LIMIT 1 BY Phrase
-	) ORDER BY Payload, Payload
+	) ORDER BY Payload, Phrase
 	"""

tests/queries/0_stateless/01031_semi_anti_join.sql

@@ -10,16 +10,16 @@ INSERT INTO t2 (x, s) VALUES (2, 'b1'), (2, 'b2'), (4, 'b3'), (4, 'b4'), (4, 'b5
 SET join_use_nulls = 0;
 
 SELECT 'semi left';
-SELECT t1.*, t2.* FROM t1 SEMI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x;
+SELECT t1.*, t2.* FROM t1 SEMI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s;
 
 SELECT 'semi right';
-SELECT t1.*, t2.* FROM t1 SEMI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x;
+SELECT t1.*, t2.* FROM t1 SEMI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s;
 
 SELECT 'anti left';
-SELECT t1.*, t2.* FROM t1 ANTI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x;
+SELECT t1.*, t2.* FROM t1 ANTI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s;
 
 SELECT 'anti right';
-SELECT t1.*, t2.* FROM t1 ANTI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x;
+SELECT t1.*, t2.* FROM t1 ANTI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s;
 
 DROP TABLE t1;
 DROP TABLE t2;

tests/queries/0_stateless/01086_odbc_roundtrip.sh

@@ -17,7 +17,7 @@ ${CLICKHOUSE_CLIENT} --query "select count() > 1 as ok from (select * from odbc(
 ${CLICKHOUSE_CLIENT} --query "CREATE TABLE t (x UInt8, y Float32, z String) ENGINE = Memory"
 ${CLICKHOUSE_CLIENT} --query "INSERT INTO t VALUES (1,0.1,'a я'),(2,0.2,'b ą'),(3,0.3,'c d')"
 
-${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (ANSI)}','$CLICKHOUSE_DATABASE','t') ORDER BY x"
-${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (Unicode)}','$CLICKHOUSE_DATABASE','t') ORDER BY x"
+${CLICKHOUSE_CLIENT} --query "SELECT x, y, z FROM odbc('DSN={ClickHouse DSN (ANSI)}','$CLICKHOUSE_DATABASE','t') ORDER BY x"
+${CLICKHOUSE_CLIENT} --query "SELECT x, y, z FROM odbc('DSN={ClickHouse DSN (Unicode)}','$CLICKHOUSE_DATABASE','t') ORDER BY x"
 
 ${CLICKHOUSE_CLIENT} --query "DROP TABLE t"

tests/queries/0_stateless/01271_show_privileges.reference

@@ -68,9 +68,9 @@ CREATE ROLE	[]	GLOBAL	ACCESS MANAGEMENT
 ALTER ROLE	[]	GLOBAL	ACCESS MANAGEMENT
 DROP ROLE	[]	GLOBAL	ACCESS MANAGEMENT
 ROLE ADMIN	[]	GLOBAL	ACCESS MANAGEMENT
-CREATE ROW POLICY	['CREATE POLICY']	GLOBAL	ACCESS MANAGEMENT
-ALTER ROW POLICY	['ALTER POLICY']	GLOBAL	ACCESS MANAGEMENT
-DROP ROW POLICY	['DROP POLICY']	GLOBAL	ACCESS MANAGEMENT
+CREATE ROW POLICY	['CREATE POLICY']	TABLE	ACCESS MANAGEMENT
+ALTER ROW POLICY	['ALTER POLICY']	TABLE	ACCESS MANAGEMENT
+DROP ROW POLICY	['DROP POLICY']	TABLE	ACCESS MANAGEMENT
 CREATE QUOTA	[]	GLOBAL	ACCESS MANAGEMENT
 ALTER QUOTA	[]	GLOBAL	ACCESS MANAGEMENT
 DROP QUOTA	[]	GLOBAL	ACCESS MANAGEMENT
@@ -79,7 +79,7 @@ ALTER SETTINGS PROFILE	['ALTER PROFILE']	GLOBAL	ACCESS MANAGEMENT
 DROP SETTINGS PROFILE	['DROP PROFILE']	GLOBAL	ACCESS MANAGEMENT
 SHOW USERS	['SHOW CREATE USER']	GLOBAL	SHOW ACCESS
 SHOW ROLES	['SHOW CREATE ROLE']	GLOBAL	SHOW ACCESS
-SHOW ROW POLICIES	['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY']	GLOBAL	SHOW ACCESS
+SHOW ROW POLICIES	['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY']	TABLE	SHOW ACCESS
 SHOW QUOTAS	['SHOW CREATE QUOTA']	GLOBAL	SHOW ACCESS
 SHOW SETTINGS PROFILES	['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE']	GLOBAL	SHOW ACCESS
 SHOW ACCESS	[]	\N	ACCESS MANAGEMENT

tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql

@@ -3,6 +3,6 @@ SELECT number FROM numbers(10) ORDER BY number DESC OFFSET 2 ROWS FETCH NEXT 3 R
 DROP TABLE IF EXISTS test_fetch;
 CREATE TABLE test_fetch(a Int32, b Int32) Engine = Memory;
 INSERT INTO test_fetch VALUES(1, 1), (2, 1), (3, 4), (3, 3), (5, 4), (0, 6), (5, 7);
-SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS ONLY;
+SELECT * FROM (SELECT * FROM test_fetch ORDER BY a, b OFFSET 1 ROW FETCH FIRST 3 ROWS ONLY) ORDER BY a, b;
 SELECT * FROM (SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS WITH TIES) ORDER BY a, b;
 DROP TABLE test_fetch;

tests/queries/0_stateless/01543_collate_in_tuple.sql

@@ -12,19 +12,19 @@ INSERT INTO collate_test1 VALUES (1, (1, 'Ё')), (1, (1, 'ё')), (1, (1, 'а')),
 INSERT INTO collate_test2 VALUES (1, (1, 'Ё')), (1, (1, 'ё')), (1, (1, 'а')), (2, (2, 'А')), (2, (1, 'я')), (2, (2, 'Я')), (1, (2, null)), (1, (3, 'я')), (1, (1, null)), (2, (2, null));
 INSERT INTO collate_test3 VALUES (1, (1, (1, ['Ё']))), (1, (2, (1, ['ё']))), (1, (1, (2, ['а']))), (2, (1, (1, ['А']))), (2, (2, (1, ['я']))), (2, (1, (1, ['Я']))), (1, (2, (1, ['ё','а']))), (1, (1, (2, ['ё', 'я']))), (2, (1, (1, ['ё', 'а', 'а'])));
 
-SELECT * FROM collate_test1 ORDER BY s COLLATE 'ru';
+SELECT * FROM collate_test1 ORDER BY s COLLATE 'ru', x;
 SELECT '';
 
 SELECT * FROM collate_test1 ORDER BY x, s COLLATE 'ru';
 SELECT '';
 
-SELECT * FROM collate_test2 ORDER BY s COLLATE 'ru';
+SELECT * FROM collate_test2 ORDER BY s COLLATE 'ru', x;
 SELECT '';
 
 SELECT * FROM collate_test2 ORDER BY x, s COLLATE 'ru';
 SELECT '';
 
-SELECT * FROM collate_test3 ORDER BY s COLLATE 'ru';
+SELECT * FROM collate_test3 ORDER BY s COLLATE 'ru', x;
 SELECT '';
 
 SELECT * FROM collate_test3 ORDER BY x, s COLLATE 'ru';

tests/queries/0_stateless/01591_window_functions.reference

@@ -39,7 +39,7 @@ select number, avg(number) over (order by number rows unbounded preceding) from
 8	4
 9	4.5
 -- no order by
-select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by value rows unbounded preceding) from numbers(10);
+select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) from numbers(10);
 0	0
 1	1
 2	1
@@ -51,7 +51,7 @@ select number, quantileExact(number) over (partition by intDiv(number, 3) AS val
 8	7
 9	9
 -- can add an alias after window spec
-select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by value rows unbounded preceding) q from numbers(10);
+select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) q from numbers(10);
 0	0
 1	1
 2	1
@@ -198,7 +198,7 @@ select sum(number) over w1, sum(number) over w2
 from numbers(10)
 window
     w1 as (rows unbounded preceding),
-    w2 as (partition by intDiv(number, 3) as value order by value rows unbounded preceding)
+    w2 as (partition by intDiv(number, 3) as value order by number rows unbounded preceding)
 ;
 0	0
 1	1
@@ -214,7 +214,7 @@ window
 -- EXPLAIN test for this.
 select
     sum(number) over w1,
-    sum(number) over (partition by intDiv(number, 3) as value order by value rows unbounded preceding)
+    sum(number) over (partition by intDiv(number, 3) as value order by number rows unbounded preceding)
 from numbers(10)
 window
     w1 as (partition by intDiv(number, 3) rows unbounded preceding)
@@ -240,118 +240,118 @@ select sum(number) over () from numbers(3);
 -- interesting corner cases.
 select number, intDiv(number, 3) p, mod(number, 2) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 5
 ;
-0	0	0	2
+0	0	0	1
 1	0	1	3
 2	0	0	2
-3	1	1	3
+3	1	1	2
 4	1	0	1
 5	1	1	3
-6	2	0	2
+6	2	0	1
 7	2	1	3
 8	2	0	2
-9	3	1	3
+9	3	1	2
 10	3	0	1
 11	3	1	3
-12	4	0	2
+12	4	0	1
 13	4	1	3
 14	4	0	2
-15	5	1	3
+15	5	1	2
 16	5	0	1
 17	5	1	3
-18	6	0	2
+18	6	0	1
 19	6	1	3
 20	6	0	2
-21	7	1	3
+21	7	1	2
 22	7	0	1
 23	7	1	3
-24	8	0	2
+24	8	0	1
 25	8	1	3
 26	8	0	2
-27	9	1	3
+27	9	1	2
 28	9	0	1
 29	9	1	3
 30	10	0	1
 select number, intDiv(number, 5) p, mod(number, 3) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 2
 ;
-0	0	0	2
-1	0	1	4
+0	0	0	1
+1	0	1	3
 2	0	2	5
 3	0	0	2
 4	0	1	4
-5	1	2	5
-6	1	0	2
+5	1	2	4
+6	1	0	1
 7	1	1	3
 8	1	2	5
 9	1	0	2
-10	2	1	3
-11	2	2	5
+10	2	1	2
+11	2	2	4
 12	2	0	1
 13	2	1	3
 14	2	2	5
-15	3	0	2
-16	3	1	4
+15	3	0	1
+16	3	1	3
 17	3	2	5
 18	3	0	2
 19	3	1	4
-20	4	2	5
-21	4	0	2
+20	4	2	4
+21	4	0	1
 22	4	1	3
 23	4	2	5
 24	4	0	2
-25	5	1	3
-26	5	2	5
+25	5	1	2
+26	5	2	4
 27	5	0	1
 28	5	1	3
 29	5	2	5
 30	6	0	1
 select number, intDiv(number, 5) p, mod(number, 2) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 3
 ;
-0	0	0	3
-1	0	1	5
-2	0	0	3
+0	0	0	1
+1	0	1	4
+2	0	0	2
 3	0	1	5
 4	0	0	3
-5	1	1	5
-6	1	0	2
-7	1	1	5
+5	1	1	3
+6	1	0	1
+7	1	1	4
 8	1	0	2
 9	1	1	5
-10	2	0	3
-11	2	1	5
-12	2	0	3
+10	2	0	1
+11	2	1	4
+12	2	0	2
 13	2	1	5
 14	2	0	3
-15	3	1	5
-16	3	0	2
-17	3	1	5
+15	3	1	3
+16	3	0	1
+17	3	1	4
 18	3	0	2
 19	3	1	5
-20	4	0	3
-21	4	1	5
-22	4	0	3
+20	4	0	1
+21	4	1	4
+22	4	0	2
 23	4	1	5
 24	4	0	3
-25	5	1	5
-26	5	0	2
-27	5	1	5
+25	5	1	3
+26	5	0	1
+27	5	1	4
 28	5	0	2
 29	5	1	5
 30	6	0	1
 select number, intDiv(number, 3) p, mod(number, 5) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 2
 ;
@@ -388,7 +388,7 @@ settings max_block_size = 2
 30	10	0	1
 select number, intDiv(number, 2) p, mod(number, 5) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 3
 ;
@@ -975,39 +975,39 @@ select number, p, o,
     row_number() over w
 from (select number, intDiv(number, 5) p, mod(number, 3) o
     from numbers(31) order by o, number) t
-window w as (partition by p order by o)
+window w as (partition by p order by o, number)
 order by p, o, number
 settings max_block_size = 2;
-0	0	0	2	1	1	1
-3	0	0	2	1	1	2
-1	0	1	4	3	2	3
-4	0	1	4	3	2	4
-2	0	2	5	5	3	5
-6	1	0	2	1	1	1
-9	1	0	2	1	1	2
-7	1	1	3	3	2	3
-5	1	2	5	4	3	4
-8	1	2	5	4	3	5
+0	0	0	1	1	1	1
+3	0	0	2	2	2	2
+1	0	1	3	3	3	3
+4	0	1	4	4	4	4
+2	0	2	5	5	5	5
+6	1	0	1	1	1	1
+9	1	0	2	2	2	2
+7	1	1	3	3	3	3
+5	1	2	4	4	4	4
+8	1	2	5	5	5	5
 12	2	0	1	1	1	1
-10	2	1	3	2	2	2
-13	2	1	3	2	2	3
-11	2	2	5	4	3	4
-14	2	2	5	4	3	5
-15	3	0	2	1	1	2
-18	3	0	2	1	1	1
-16	3	1	4	3	2	3
-19	3	1	4	3	2	4
-17	3	2	5	5	3	5
-21	4	0	2	1	1	1
-24	4	0	2	1	1	2
-22	4	1	3	3	2	3
-20	4	2	5	4	3	5
-23	4	2	5	4	3	4
+10	2	1	2	2	2	2
+13	2	1	3	3	3	3
+11	2	2	4	4	4	4
+14	2	2	5	5	5	5
+15	3	0	1	1	1	1
+18	3	0	2	2	2	2
+16	3	1	3	3	3	3
+19	3	1	4	4	4	4
+17	3	2	5	5	5	5
+21	4	0	1	1	1	1
+24	4	0	2	2	2	2
+22	4	1	3	3	3	3
+20	4	2	4	4	4	4
+23	4	2	5	5	5	5
 27	5	0	1	1	1	1
-25	5	1	3	2	2	2
-28	5	1	3	2	2	3
-26	5	2	5	4	3	4
-29	5	2	5	4	3	5
+25	5	1	2	2	2	2
+28	5	1	3	3	3	3
+26	5	2	4	4	4	4
+29	5	2	5	5	5	5
 30	6	0	1	1	1	1
 -- our replacement for lag/lead
 select
@@ -1153,7 +1153,7 @@ select count() over () where null;
 select number, count() over (w1 rows unbounded preceding) from numbers(10)
 window
     w0 as (partition by intDiv(number, 5) as p),
-    w1 as (w0 order by mod(number, 3) as o)
+    w1 as (w0 order by mod(number, 3) as o, number)
 order by p, o, number
 ;
 0	1

tests/queries/0_stateless/01591_window_functions.sql

@@ -13,10 +13,10 @@ select number, abs(number) over (partition by toString(intDiv(number, 3)) rows u
 select number, avg(number) over (order by number rows unbounded preceding) from numbers(10);
 
 -- no order by
-select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by value rows unbounded preceding) from numbers(10);
+select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) from numbers(10);
 
 -- can add an alias after window spec
-select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by value rows unbounded preceding) q from numbers(10);
+select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) q from numbers(10);
 
 -- can't reference it yet -- the window functions are calculated at the
 -- last stage of select, after all other functions.
@@ -81,14 +81,14 @@ select sum(number) over w1, sum(number) over w2
 from numbers(10)
 window
     w1 as (rows unbounded preceding),
-    w2 as (partition by intDiv(number, 3) as value order by value rows unbounded preceding)
+    w2 as (partition by intDiv(number, 3) as value order by number rows unbounded preceding)
 ;
 
 -- FIXME both functions should use the same window, but they don't. Add an
 -- EXPLAIN test for this.
 select
     sum(number) over w1,
-    sum(number) over (partition by intDiv(number, 3) as value order by value rows unbounded preceding)
+    sum(number) over (partition by intDiv(number, 3) as value order by number rows unbounded preceding)
 from numbers(10)
 window
     w1 as (partition by intDiv(number, 3) rows unbounded preceding)
@@ -103,35 +103,35 @@ select sum(number) over () from numbers(3);
 -- interesting corner cases.
 select number, intDiv(number, 3) p, mod(number, 2) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 5
 ;
 
 select number, intDiv(number, 5) p, mod(number, 3) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 2
 ;
 
 select number, intDiv(number, 5) p, mod(number, 2) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 3
 ;
 
 select number, intDiv(number, 3) p, mod(number, 5) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 2
 ;
 
 select number, intDiv(number, 2) p, mod(number, 5) o, count(number) over w as c
 from numbers(31)
-window w as (partition by p order by o range unbounded preceding)
+window w as (partition by p order by o, number range unbounded preceding)
 order by number
 settings max_block_size = 3
 ;
@@ -349,7 +349,7 @@ select number, p, o,
     row_number() over w
 from (select number, intDiv(number, 5) p, mod(number, 3) o
     from numbers(31) order by o, number) t
-window w as (partition by p order by o)
+window w as (partition by p order by o, number)
 order by p, o, number
 settings max_block_size = 2;
 
@@ -456,7 +456,7 @@ select count() over () where null;
 select number, count() over (w1 rows unbounded preceding) from numbers(10)
 window
     w0 as (partition by intDiv(number, 5) as p),
-    w1 as (w0 order by mod(number, 3) as o)
+    w1 as (w0 order by mod(number, 3) as o, number)
 order by p, o, number
 ;
 

tests/queries/0_stateless/01592_window_functions.sql

@@ -36,7 +36,7 @@ SELECT
   price,
 	rank() OVER (PARTITION BY group_name ORDER BY price) rank
 FROM products INNER JOIN product_groups USING (group_id)
-order by  group_name, rank, price;
+order by group_name, rank, price, product_name;
 
 select '---- Q3 ----';
 SELECT

tests/queries/0_stateless/01670_neighbor_lc_bug.sql

@@ -37,7 +37,7 @@ FROM
 (
     SELECT *
     FROM neighbor_test
-    ORDER BY val_string ASC
+    ORDER BY val_string, rowNr
 )
 ORDER BY rowNr, val_string, str_m1, str_p1, val_low, low_m1, low_p1
 format PrettyCompact;

tests/queries/0_stateless/01710_aggregate_projections.sh

@@ -14,8 +14,8 @@ $CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj group by
 $CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1"
 $CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2), intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc limit 5 settings allow_experimental_projection_optimization=1"
-$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2), intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings allow_experimental_projection_optimization=1"
+$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"
 
 $CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1"
 $CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read"

tests/queries/0_stateless/01780_column_sparse_full.reference

@@ -91,9 +91,9 @@ all_2_2_0	u	Default
 ======
 0	0	0
 0	0	0
-1	1	1
 1	0	
-2	2	2
+1	1	1
+2	0	
 ======
 0	0	0
 0	0	0

tests/queries/0_stateless/01780_column_sparse_full.sql

@@ -81,7 +81,7 @@ INNER JOIN t_sparse_full USING(u) ORDER BY id, u, s LIMIT 5;
 SELECT '======';
 
 SELECT id, u, s FROM (SELECT number * 2 AS u FROM numbers(10)) AS t1
-FULL JOIN t_sparse_full USING(u) ORDER BY id LIMIT 5;
+FULL JOIN t_sparse_full USING(u) ORDER BY id, u, s LIMIT 5;
 
 SELECT '======';
 

tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2

@@ -24,9 +24,9 @@ SET join_algorithm = 'partial_merge';
 SELECT '-- partial_merge --';
 
 SELECT '--';
-SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2;
+SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 ORDER BY t1.key, t1.key2;
 SELECT '--';
-SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2;
+SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2 ORDER BY t1.key, t1.key2;
 
 SELECT '--';
 SELECT t1.key FROM t1 INNER ANY JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2;

tests/queries/0_stateless/02104_overcommit_memory.reference

@@ -0,0 +1 @@
+OVERCOMMITED WITH USER LIMIT WAS KILLED

tests/queries/0_stateless/02104_overcommit_memory.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Tags: no-parallel
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u1 IDENTIFIED WITH no_password'
+$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u1'
+
+function overcommited()
+{
+    while true; do
+        $CLICKHOUSE_CLIENT -u u1 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_guaranteed_memory_usage=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo "OVERCOMMITED WITH USER LIMIT IS KILLED"
+    done
+}
+
+function expect_execution()
+{
+    while true; do
+        $CLICKHOUSE_CLIENT -u u1 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,max_guaranteed_memory_usage=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null
+    done
+}
+
+export -f overcommited
+export -f expect_execution
+
+function user_test()
+{
+    for _ in {1..10};
+    do
+        timeout 10 bash -c overcommited &
+        timeout 10 bash -c expect_execution &
+    done;
+
+    wait
+}
+
+output=$(user_test)
+
+if test -z "$output"
+then
+    echo "OVERCOMMITED WITH USER LIMIT WAS NOT KILLED"
+else
+    echo "OVERCOMMITED WITH USER LIMIT WAS KILLED"
+fi
+
+$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u1'

tests/queries/0_stateless/02180_group_by_lowcardinality.reference

@@ -0,0 +1,10 @@
+{"val":"1563.8","avg(toUInt32(val))":null}
+{"val":"891.4","avg(toUInt32(val))":null}
+{"val":"584.4","avg(toUInt32(val))":null}
+{"val":"269","avg(toUInt32(val))":269}
+{"val":"1233.4","avg(toUInt32(val))":null}
+{"val":"1833","avg(toUInt32(val))":1833}
+{"val":"1009.4","avg(toUInt32(val))":null}
+{"val":"1178.6","avg(toUInt32(val))":null}
+{"val":"372.6","avg(toUInt32(val))":null}
+{"val":"232.4","avg(toUInt32(val))":null}

tests/queries/0_stateless/02180_group_by_lowcardinality.sql

@@ -0,0 +1,10 @@
+create table if not exists t_group_by_lowcardinality(p_date Date, val LowCardinality(Nullable(String))) 
+engine=MergeTree() partition by p_date order by tuple();
+
+insert into t_group_by_lowcardinality select today() as p_date, toString(number/5) as val from numbers(10000);
+insert into t_group_by_lowcardinality select today() as p_date, Null as val from numbers(100);
+
+select val, avg(toUInt32(val)) from t_group_by_lowcardinality group by val limit 10 settings max_threads=1, max_rows_to_group_by=100, group_by_overflow_mode='any' format JSONEachRow;
+
+drop table if exists t_group_by_lowcardinality;
+

tests/queries/0_stateless/02187_insert_values_with_mv.reference

@@ -0,0 +1,12 @@
+VALUES	1
+TABLE	1
+VALUES	1
+VALUES	1
+VALUES	1
+VALUES	1
+VALUES	1
+TABLE	1
+TABLE	1
+TABLE	1
+TABLE	1
+TABLE	1