Merge branch 'master' into gyuton-DOCSUP-5601-Edit_and_translate_to_Russian

base/daemon/BaseDaemon.cpp

@@ -962,7 +962,7 @@ void BaseDaemon::setupWatchdog()
         if (WIFEXITED(status))
         {
             logger().information(fmt::format("Child process exited normally with code {}.", WEXITSTATUS(status)));
-            _exit(status);
+            _exit(WEXITSTATUS(status));
         }
 
         if (WIFSIGNALED(status))
@@ -980,7 +980,7 @@ void BaseDaemon::setupWatchdog()
                 logger().fatal(fmt::format("Child process was terminated by signal {}.", sig));
 
                 if (sig == SIGINT || sig == SIGTERM || sig == SIGQUIT)
-                    _exit(status);
+                    _exit(128 + sig);
             }
         }
         else

docker/test/coverage/run.sh

@@ -102,11 +102,11 @@ else
     echo "No failed tests"
 fi
 
-mkdir -p $COVERAGE_DIR
-mv /*.profraw $COVERAGE_DIR
+mkdir -p "$COVERAGE_DIR"
+mv /*.profraw "$COVERAGE_DIR"
 
-mkdir -p $SOURCE_DIR/obj-x86_64-linux-gnu
-cd $SOURCE_DIR/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /
-llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata
-llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov
-genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR}
+mkdir -p "$SOURCE_DIR"/obj-x86_64-linux-gnu
+cd "$SOURCE_DIR"/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /
+llvm-profdata-11 merge -sparse "${COVERAGE_DIR}"/* -o clickhouse.profdata
+llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex "$IGNORE" > output.lcov
+genhtml output.lcov --ignore-errors source --output-directory "${OUTPUT_DIR}"

docker/test/fasttest/run.sh

@@ -65,7 +65,7 @@ function start_server
 {
     set -m # Spawn server in its own process groups
     local opts=(
-        --config-file="$FASTTEST_DATA/config.xml"
+        --config-file "$FASTTEST_DATA/config.xml"
         --
         --path "$FASTTEST_DATA"
         --user_files_path "$FASTTEST_DATA/user_files"

docker/test/stateless/run.sh

@@ -55,9 +55,9 @@ function run_tests()
         ADDITIONAL_OPTIONS+=('00000_no_tests_to_skip')
     fi
 
-    for i in $(seq 1 $NUM_TRIES); do
+    for _ in $(seq 1 "$NUM_TRIES"); do
         clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt
-        if [ ${PIPESTATUS[0]} -ne "0" ]; then
+        if [ "${PIPESTATUS[0]}" -ne "0" ]; then
             break;
         fi
     done
@@ -65,4 +65,4 @@ function run_tests()
 
 export -f run_tests
 
-timeout $MAX_RUN_TIME bash -c run_tests ||:
+timeout "$MAX_RUN_TIME" bash -c run_tests ||:

docker/test/style/Dockerfile

@@ -8,4 +8,5 @@ CMD cd /ClickHouse/utils/check-style && \
     ./check-style -n | tee /test_output/style_output.txt && \
     ./check-typos | tee /test_output/typos_output.txt && \
     ./check-whitespaces -n | tee /test_output/whitespaces_output.txt && \
-    ./check-duplicate-includes.sh | tee /test_output/duplicate_output.txt
+    ./check-duplicate-includes.sh | tee /test_output/duplicate_output.txt && \
+    ./shellcheck-run.sh | tee /test_output/shellcheck_output.txt

docs/en/operations/settings/settings.md

@@ -726,7 +726,7 @@ log_queries=1
 
 ## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms}
 
-Minimal time for the query to run to get to the following tables:
+If enabled (non-zero), queries faster then the value of this setting will not be logged (you can think about this as a `long_query_time` for [MySQL Slow Query Log](https://dev.mysql.com/doc/refman/5.7/en/slow-query-log.html)), and this basically means that you will not find them in the following tables:
 
 - `system.query_log`
 - `system.query_thread_log`
@@ -2470,6 +2470,31 @@ Possible values:
 
 Default value: `0`.
 
+## union_default_mode {#union-default-mode}
+
+Sets a mode for combining `SELECT` query results. The setting is only used when shared with [UNION](../../sql-reference/statements/select/union.md) without explicitly specifying the `UNION ALL` or `UNION DISTINCT`.
+
+Possible values:
+
+-   `'DISTINCT'` — ClickHouse outputs rows as a result of combining queries removing duplicate rows.
+-   `'ALL'` — ClickHouse outputs all rows as a result of combining queries including duplicate rows.
+-   `''` — Clickhouse generates an exception when used with `UNION`.
+
+Default value: `''`.
+
+See examples in [UNION](../../sql-reference/statements/select/union.md).
+
+## data_type_default_nullable {#data_type_default_nullable}
+
+Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable).
+
+Possible values:
+
+- 1 — The data types in column definitions are set to `Nullable` by default.
+- 0 — The data types in column definitions are set to not `Nullable` by default.
+
+Default value: `0`.
+
 ## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold}
 
 Enables special logic to perform merges on replicas.

docs/en/operations/system-tables/errors.md

@@ -1,12 +1,12 @@
 # system.errors {#system_tables-errors}
 
-Contains error codes with number of times they have been triggered.
+Contains error codes with the number of times they have been triggered.
 
 Columns:
 
 -   `name` ([String](../../sql-reference/data-types/string.md)) — name of the error (`errorCodeToName`).
 -   `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error.
--   `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) - number of times this error has been happened.
+-   `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened.
 
 **Example**
 

docs/en/sql-reference/functions/hash-functions.md

@@ -157,14 +157,14 @@ Levels are the same as in URLHierarchy. This function is specific to Yandex.Metr
 
 ## farmHash64 {#farmhash64}
 
-Produces a 64-bit [FarmHash](https://github.com/google/farmhash) or Fingerprint value. Prefer `farmFingerprint64` for a stable and portable value.
+Produces a 64-bit [FarmHash](https://github.com/google/farmhash) or Fingerprint value. `farmFingerprint64` is preferred for a stable and portable value.
 
 ``` sql
 farmFingerprint64(par1, ...)
 farmHash64(par1, ...)
 ```
 
-These functions use the `Fingerprint64` and `Hash64` method respectively from all [available methods](https://github.com/google/farmhash/blob/master/src/farmhash.h).
+These functions use the `Fingerprint64` and `Hash64` methods respectively from all [available methods](https://github.com/google/farmhash/blob/master/src/farmhash.h).
 
 **Parameters**
 

docs/en/sql-reference/functions/string-functions.md

@@ -558,4 +558,46 @@ Result:
 └─────┘
 ```
 
+## encodeXMLComponent {#encode-xml-component}
+
+Escapes characters to place string into XML text node or attribute.
+
+The following five XML predefined entities will be replaced: `<`, `&`, `>`, `"`, `'`.
+
+**Syntax** 
+
+``` sql
+encodeXMLComponent(x)
+```
+
+**Parameters** 
+
+-   `x` — The sequence of characters. [String](../../sql-reference/data-types/string.md).
+
+**Returned value(s)**
+
+-   The sequence of characters with escape characters.
+
+Type: [String](../../sql-reference/data-types/string.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT encodeXMLComponent('Hello, "world"!');
+SELECT encodeXMLComponent('<123>');
+SELECT encodeXMLComponent('&clickhouse');
+SELECT encodeXMLComponent('\'foo\'');
+```
+
+Result:
+
+``` text
+Hello, &quot;world&quot;!
+&lt;123&gt;
+&amp;clickhouse
+&apos;foo&apos;
+```
+
 [Original article](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) <!--hide-->

docs/en/sql-reference/functions/string-search-functions.md

@@ -400,7 +400,8 @@ Result:
 └──────────────────────────────────────────────────────────────────────────────────────────┘
 ```
 
-**See also**
+**See Also**
+
 -   [extractAllGroupsVertical](#extractallgroups-vertical)
 
 ## extractAllGroupsVertical {#extractallgroups-vertical}
@@ -440,7 +441,8 @@ Result:
 └────────────────────────────────────────────────────────────────────────────────────────┘
 ```
 
-**See also**
+**See Also**
+
 -   [extractAllGroupsHorizontal](#extractallgroups-horizontal)
 
 ## like(haystack, pattern), haystack LIKE pattern operator {#function-like}
@@ -726,4 +728,51 @@ Result:
 
 Returns the number of regular expression matches for a `pattern` in a `haystack`.
 
+**Syntax**
+
+``` sql
+countMatches(haystack, pattern)
+```
+
+**Parameters**
+
+-   `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal).
+-   `pattern` — The regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). [String](../../sql-reference/data-types/string.md).
+
+**Returned value**
+
+-   The number of matches.
+
+Type: [UInt64](../../sql-reference/data-types/int-uint.md).
+
+**Examples**
+
+Query:
+
+``` sql
+SELECT countMatches('foobar.com', 'o+');
+```
+
+Result:
+
+``` text
+┌─countMatches('foobar.com', 'o+')─┐
+│                                2 │
+└──────────────────────────────────┘
+```
+
+Query:
+
+``` sql
+SELECT countMatches('aaaa', 'aa');
+```
+
+Result:
+
+``` text
+┌─countMatches('aaaa', 'aa')────┐
+│                             2 │
+└───────────────────────────────┘
+```
+
 [Original article](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) <!--hide-->

docs/en/sql-reference/statements/create/table.md

@@ -16,8 +16,8 @@ By default, tables are created only on the current server. Distributed DDL queri
 ``` sql
 CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 (
-    name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1],
-    name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2],
+    name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1],
+    name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2],
     ...
 ) ENGINE = engine
 ```
@@ -57,6 +57,14 @@ In all cases, if `IF NOT EXISTS` is specified, the query won’t return an error
 
 There can be other clauses after the `ENGINE` clause in the query. See detailed documentation on how to create tables in the descriptions of [table engines](../../../engines/table-engines/index.md#table_engines).
 
+## NULL Or NOT NULL Modifiers {#null-modifiers}
+
+`NULL` and `NOT NULL` modifiers after data type in column definition allow or do not allow it to be [Nullable](../../../sql-reference/data-types/nullable.md#data_type-nullable). 
+
+If the type is not `Nullable` and if `NULL` is specified, it will be treated as `Nullable`; if `NOT NULL` is specified, then no. For example, `INT NULL` is the same as `Nullable(INT)`. If the type is `Nullable` and `NULL` or `NOT NULL` modifiers are specified, the exception will be thrown.
+
+See also [data_type_default_nullable](../../../operations/settings/settings.md#data_type_default_nullable) setting.
+
 ## Default Values {#create-default-values}
 
 The column description can specify an expression for a default value, in one of the following ways: `DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`.

docs/en/sql-reference/statements/select/index.md

@@ -46,7 +46,7 @@ Specifics of each optional clause are covered in separate sections, which are li
 -   [SELECT clause](#select-clause)
 -   [DISTINCT clause](../../../sql-reference/statements/select/distinct.md)
 -   [LIMIT clause](../../../sql-reference/statements/select/limit.md)
--   [UNION clause](../../../sql-reference/statements/select/union-all.md)
+-   [UNION clause](../../../sql-reference/statements/select/union.md)
 -   [INTO OUTFILE clause](../../../sql-reference/statements/select/into-outfile.md)
 -   [FORMAT clause](../../../sql-reference/statements/select/format.md)
 

docs/en/sql-reference/statements/select/union-all.md

@@ -1,37 +0,0 @@
----
-toc_title: UNION
----
-
-# UNION ALL Clause {#union-all-clause}
-
-You can use `UNION ALL` to combine any number of `SELECT` queries by extending their results. Example:
-
-``` sql
-SELECT CounterID, 1 AS table, toInt64(count()) AS c
-    FROM test.hits
-    GROUP BY CounterID
-
-UNION ALL
-
-SELECT CounterID, 2 AS table, sum(Sign) AS c
-    FROM test.visits
-    GROUP BY CounterID
-    HAVING c > 0
-```
-
-Result columns are matched by their index (order inside `SELECT`). If column names do not match, names for the final result are taken from the first query.
-
-Type casting is performed for unions. For example, if two queries being combined have the same field with non-`Nullable` and `Nullable` types from a compatible type, the resulting `UNION ALL` has a `Nullable` type field.
-
-Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION ALL` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause.
-
-# UNION DISTINCT Clause {#union-distinct-clause}
-The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`.
-
-# UNION Clause {#union-clause}
-By default, `UNION` has the same behavior as `UNION DISTINCT`, but you can specify union mode by setting `union_default_mode`, values can be 'ALL', 'DISTINCT' or empty string. However, if you use `UNION` with setting `union_default_mode` to empty string, it will throw an exception.
-
-
-## Implementation Details {#implementation-details}
-
-Queries that are parts of `UNION/UNION ALL/UNION DISTINCT` can be run simultaneously, and their results can be mixed together.

docs/en/sql-reference/statements/select/union.md

@@ -0,0 +1,81 @@
+---
+toc_title: UNION
+---
+
+# UNION Clause {#union-clause}
+
+You can use `UNION` with explicitly specifying `UNION ALL` or `UNION DISTINCT`.
+
+If you don't specify `ALL` or `DISTINCT`, it will depend on the `union_default_mode` setting. The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`.
+
+You can use `UNION` to combine any number of `SELECT` queries by extending their results. Example:
+
+``` sql
+SELECT CounterID, 1 AS table, toInt64(count()) AS c
+    FROM test.hits
+    GROUP BY CounterID
+
+UNION ALL
+
+SELECT CounterID, 2 AS table, sum(Sign) AS c
+    FROM test.visits
+    GROUP BY CounterID
+    HAVING c > 0
+```
+
+Result columns are matched by their index (order inside `SELECT`). If column names do not match, names for the final result are taken from the first query.
+
+Type casting is performed for unions. For example, if two queries being combined have the same field with non-`Nullable` and `Nullable` types from a compatible type, the resulting `UNION` has a `Nullable` type field.
+
+Queries that are parts of `UNION` can be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause.
+
+If you use `UNION` without explicitly specifying `UNION ALL` or `UNION DISTINCT`, you can specify the union mode using the [union_default_mode](../../../operations/settings/settings.md#union-default-mode) setting. The setting values can be `ALL`, `DISTINCT` or an empty string. However, if you use `UNION` with `union_default_mode` setting to empty string, it will throw an exception. The following examples demonstrate the results of queries with different values setting.
+
+Query:
+
+```sql
+SET union_default_mode = 'DISTINCT';
+SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2;
+```
+
+Result:
+
+```text
+┌─1─┐
+│ 1 │
+└───┘
+┌─1─┐
+│ 2 │
+└───┘
+┌─1─┐
+│ 3 │
+└───┘
+```
+
+Query:
+
+```sql
+SET union_default_mode = 'ALL';
+SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2;
+```
+
+Result:
+
+```text
+┌─1─┐
+│ 1 │
+└───┘
+┌─1─┐
+│ 2 │
+└───┘
+┌─1─┐
+│ 2 │
+└───┘
+┌─1─┐
+│ 3 │
+└───┘
+```
+
+Queries that are parts of `UNION/UNION ALL/UNION DISTINCT` can be run simultaneously, and their results can be mixed together.
+
+[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/union/) <!-- hide -->

docs/es/sql-reference/statements/select/index.md

@@ -44,7 +44,7 @@ Los detalles de cada cláusula opcional se cubren en secciones separadas, que se
 -   [Cláusula HAVING](having.md)
 -   [Cláusula SELECT](#select-clause)
 -   [Cláusula LIMIT](limit.md)
--   [UNION ALL cláusula](union-all.md)
+-   [UNION ALL cláusula](union.md)
 
 ## SELECT Cláusula {#select-clause}
 

docs/es/sql-reference/statements/select/union.md

@@ -3,7 +3,7 @@ machine_translated: true
 machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
 ---
 
-# UNION ALL Cláusula {#union-all-clause}
+# UNION Cláusula {#union-clause}
 
 Usted puede utilizar `UNION ALL` combinar cualquier número de `SELECT` consultas extendiendo sus resultados. Ejemplo:
 

docs/fa/sql-reference/statements/select/index.md

@@ -44,7 +44,7 @@ SELECT [DISTINCT] expr_list
 -   [داشتن بند](having.md)
 -   [انتخاب بند](#select-clause)
 -   [بند محدود](limit.md)
--   [اتحادیه همه بند](union-all.md)
+-   [اتحادیه همه بند](union.md)
 
 ## انتخاب بند {#select-clause}
 

docs/fa/sql-reference/statements/select/union.md

@@ -3,7 +3,7 @@ machine_translated: true
 machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
 ---
 
-# اتحادیه همه بند {#union-all-clause}
+# اتحادیه همه بند {#union-clause}
 
 شما می توانید استفاده کنید `UNION ALL` برای ترکیب هر تعداد از `SELECT` نمایش داده شد با گسترش نتایج خود را. مثال:
 

docs/fr/sql-reference/statements/select/index.md

@@ -44,7 +44,7 @@ Spécificités de chaque clause facultative, sont couverts dans des sections dis
 -   [Clause HAVING](having.md)
 -   [Clause SELECT](#select-clause)
 -   [Clause LIMIT](limit.md)
--   [Clause UNION ALL](union-all.md)
+-   [Clause UNION ALL](union.md)
 
 ## Clause SELECT {#select-clause}
 

docs/fr/sql-reference/statements/select/union.md

@@ -3,7 +3,7 @@ machine_translated: true
 machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
 ---
 
-# Clause UNION ALL {#union-all-clause}
+# Clause UNION ALL {#union-clause}
 
 Vous pouvez utiliser `UNION ALL` à combiner `SELECT` requêtes en étendant leurs résultats. Exemple:
 

docs/ja/sql-reference/statements/select/union-all.md

@@ -1 +0,0 @@
-../../../../en/sql-reference/statements/select/union-all.md

docs/ja/sql-reference/statements/select/union.md

@@ -0,0 +1 @@
+../../../../en/sql-reference/statements/select/union.md

docs/ru/operations/settings/settings.md

@@ -408,11 +408,11 @@ INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102	2;
 
 -   `'best_effort'` — включает расширенный парсинг.
 
-        ClickHouse может парсить базовый формат `YYYY-MM-DD HH:MM:SS` и все форматы [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). Например, `'2018-06-08T01:02:03.000Z'`.
+ClickHouse может парсить базовый формат `YYYY-MM-DD HH:MM:SS` и все форматы [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). Например, `'2018-06-08T01:02:03.000Z'`.
 
 -   `'basic'` — используется базовый парсер.
 
-        ClickHouse может парсить только базовый формат `YYYY-MM-DD HH:MM:SS`. Например, `'2019-08-20 10:18:56'`.
+ClickHouse может парсить только базовый формат `YYYY-MM-DD HH:MM:SS` или `YYYY-MM-DD`. Например, `'2019-08-20 10:18:56'` или `2019-08-20`.
 
 Значение по умолчанию: `'basic'`.
 
@@ -691,6 +691,21 @@ ClickHouse использует этот параметр при чтении д
 log_queries=1
 ```
 
+## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms}
+
+Минимальное время выполнения запроса для логгирования в системные таблицы:
+
+- `system.query_log`
+- `system.query_thread_log`
+
+В случае ненулевого порога `log_queries_min_query_duration_ms`, в лог будут записываться лишь события об окончании выполнения запроса:
+
+- `QUERY_FINISH`
+- `EXCEPTION_WHILE_PROCESSING`
+
+-   Тип: milliseconds
+-   Значение по умолчанию: 0 (логгировать все запросы)
+
 ## log_queries_min_type {#settings-log-queries-min-type}
 
 Задаёт минимальный уровень логирования в `query_log`.
@@ -2324,6 +2339,20 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow;
 
 Значение по умолчанию: `0`.
 
+## union_default_mode {#union-default-mode}
+
+Устанавливает режим объединения результатов `SELECT` запросов. Настройка используется только при совместном использовании с [UNION](../../sql-reference/statements/select/union.md) без явного указания `UNION ALL` или `UNION DISTINCT`.
+
+Возможные значения:
+
+-   `'DISTINCT'` — ClickHouse выводит строки в результате объединения результатов запросов, удаляя повторяющиеся строки.
+-   `'ALL'` — ClickHouse выводит все строки в результате объединения результатов запросов, включая повторяющиеся строки.
+-   `''` — Clickhouse генерирует исключение при использовании с `UNION`.
+
+Значение по умолчанию: `''`.
+
+Смотрите примеры в разделе [UNION](../../sql-reference/statements/select/union.md).
+
 ## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold}
 
 Включает особую логику выполнения слияний на репликах.

docs/ru/operations/system-tables/errors.md

@@ -0,0 +1,23 @@
+# system.errors {#system_tables-errors}
+
+Содержит коды ошибок с указанием количества срабатываний.
+
+Столбцы:
+
+-   `name` ([String](../../sql-reference/data-types/string.md)) — название ошибки (`errorCodeToName`).
+-   `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — номер кода ошибки.
+-   `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — количество ошибок.
+
+**Пример**
+
+``` sql
+SELECT *
+FROM system.errors
+WHERE value > 0
+ORDER BY code ASC
+LIMIT 1
+
+┌─name─────────────┬─code─┬─value─┐
+│ CANNOT_OPEN_FILE │   76 │     1 │
+└──────────────────┴──────┴───────┘
+```

docs/ru/sql-reference/data-types/date.md

@@ -9,4 +9,39 @@ toc_title: Date
 
 Дата хранится без учёта часового пояса.
 
+## Примеры {#examples}
+
+**1.** Создание таблицы и добавление в неё данных:
+
+``` sql
+CREATE TABLE dt
+(
+    `timestamp` Date,
+    `event_id` UInt8
+)
+ENGINE = TinyLog;
+```
+
+``` sql
+INSERT INTO dt Values (1546300800, 1), ('2019-01-01', 2);
+```
+
+``` sql
+SELECT * FROM dt;
+```
+
+``` text
+┌──timestamp─┬─event_id─┐
+│ 2019-01-01 │        1 │
+│ 2019-01-01 │        2 │
+└────────────┴──────────┘
+```
+
+## Смотрите также {#see-also}
+
+-   [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md)
+-   [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime)
+-   [Тип данных `DateTime`](../../sql-reference/data-types/datetime.md)
+
+
 [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/date/) <!--hide-->

docs/ru/sql-reference/data-types/datetime.md

@@ -116,12 +116,14 @@ FROM dt
 
 ## See Also {#see-also}
 
--   [Функции преобразования типов](../../sql-reference/data-types/datetime.md)
--   [Функции для работы с датой и временем](../../sql-reference/data-types/datetime.md)
--   [Функции для работы с массивами](../../sql-reference/data-types/datetime.md)
--   [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format)
--   [Конфигурационный параметр сервера `timezone`](../../sql-reference/data-types/datetime.md#server_configuration_parameters-timezone)
--   [Операторы для работы с датой и временем](../../sql-reference/data-types/datetime.md#operators-datetime)
+-   [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md)
+-   [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md)
+-   [Функции для работы с массивами](../../sql-reference/functions/array-functions.md)
+-   [Настройка `date_time_input_format`](../../operations/settings/settings/#settings-date_time_input_format)
+-   [Настройка `date_time_output_format`](../../operations/settings/settings/)
+-   [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+-   [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime)
 -   [Тип данных `Date`](date.md)
+-   [Тип данных `DateTime64`](datetime64.md)
 
 [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/datetime/) <!--hide-->

docs/ru/sql-reference/data-types/datetime64.md

@@ -92,11 +92,12 @@ FROM dt
 
 ## See Also {#see-also}
 
--   [Функции преобразования типов](../../sql-reference/data-types/datetime64.md)
--   [Функции для работы с датой и временем](../../sql-reference/data-types/datetime64.md)
--   [Функции для работы с массивами](../../sql-reference/data-types/datetime64.md)
+-   [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md)
+-   [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md)
+-   [Функции для работы с массивами](../../sql-reference/functions/array-functions.md)
 -   [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format)
--   [Конфигурационный параметр сервера `timezone`](../../sql-reference/data-types/datetime64.md#server_configuration_parameters-timezone)
--   [Операторы для работы с датой и временем](../../sql-reference/data-types/datetime64.md#operators-datetime)
+-   [Настройка `date_time_output_format`](../../operations/settings/settings.md)
+-   [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+-   [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime)
 -   [Тип данных `Date`](date.md)
 -   [Тип данных `DateTime`](datetime.md)

docs/ru/sql-reference/functions/hash-functions.md

@@ -153,15 +153,18 @@ SELECT groupBitXor(cityHash64(*)) FROM table
 `URLHash(s, N)` - вычислить хэш от строки до N-го уровня в иерархии URL, без одного завершающего символа `/`, `?` или `#` на конце, если там такой есть.
 Уровни аналогичные URLHierarchy. Функция специфична для Яндекс.Метрики.
 
+## farmFingerprint64 {#farmfingerprint64}
+
 ## farmHash64 {#farmhash64}
 
-Генерирует 64-х битное значение [FarmHash](https://github.com/google/farmhash).
+Создает 64-битное значение [FarmHash](https://github.com/google/farmhash), независимое от платформы (архитектуры сервера), что важно, если значения сохраняются или используются для разбиения данных на группы.
 
 ``` sql
+farmFingerprint64(par1, ...)
 farmHash64(par1, ...)
 ```
 
-Из всех [доступных методов](https://github.com/google/farmhash/blob/master/src/farmhash.h) функция использует `Hash64`.
+Эти функции используют методы `Fingerprint64` и `Hash64` из всех [доступных методов](https://github.com/google/farmhash/blob/master/src/farmhash.h).
 
 **Параметры**
 

docs/ru/sql-reference/functions/other-functions.md

@@ -1686,6 +1686,26 @@ SELECT countDigits(toDecimal32(1, 9)), countDigits(toDecimal32(-1, 9)),
 10	10	19	19	39	39
 ```
 
+## errorCodeToName {#error-code-to-name}
+
+**Возвращаемое значение**
+
+-   Название переменной для кода ошибки.
+
+Тип: [LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md).
+
+**Синтаксис**
+
+``` sql
+errorCodeToName(1)
+```
+
+Результат:
+
+``` text
+UNSUPPORTED_METHOD
+```
+
 ## tcpPort {#tcpPort}
 
 Вовращает номер TCP порта, который использует сервер для [нативного протокола](../../interfaces/tcp.md).

docs/ru/sql-reference/functions/string-search-functions.md

@@ -521,6 +521,57 @@ SELECT * FROM Months WHERE ilike(name, '%j%')
 
 !!! note "Примечание"
     Для случая UTF-8 мы используем триграммное расстояние. Вычисление n-граммного расстояния не совсем честное. Мы используем 2-х байтные хэши для хэширования n-грамм, а затем вычисляем (не)симметрическую разность между хэш таблицами – могут возникнуть коллизии. В формате UTF-8 без учета регистра мы не используем честную функцию `tolower` – мы обнуляем 5-й бит (нумерация с нуля) каждого байта кодовой точки, а также первый бит нулевого байта, если байтов больше 1 – это работает для латиницы и почти для всех кириллических букв.
+	
+## countMatches(haystack, pattern) {#countmatcheshaystack-pattern}
+
+Возвращает количество совпадений, найденных в строке `haystack`, для регулярного выражения `pattern`.
+
+**Синтаксис**
+
+``` sql
+countMatches(haystack, pattern)
+```
+
+**Параметры**
+
+-   `haystack` — строка, по которой выполняется поиск. [String](../../sql-reference/syntax.md#syntax-string-literal).
+-   `pattern` — регулярное выражение, построенное по синтаксическим правилам [re2](https://github.com/google/re2/wiki/Syntax). [String](../../sql-reference/data-types/string.md).
+
+**Возвращаемое значение**
+
+-   Количество совпадений.
+
+Тип: [UInt64](../../sql-reference/data-types/int-uint.md).
+
+**Примеры**
+
+Запрос:
+
+``` sql
+SELECT countMatches('foobar.com', 'o+');
+```
+
+Результат:
+
+``` text
+┌─countMatches('foobar.com', 'o+')─┐
+│                                2 │
+└──────────────────────────────────┘
+```
+
+Запрос:
+
+``` sql
+SELECT countMatches('aaaa', 'aa');
+```
+
+Результат:
+
+``` text
+┌─countMatches('aaaa', 'aa')────┐
+│                             2 │
+└───────────────────────────────┘
+```
 
 ## countSubstrings {#countSubstrings}
 

docs/ru/sql-reference/statements/select/index.md

@@ -44,7 +44,7 @@ SELECT [DISTINCT] expr_list
 -   [Секция SELECT](#select-clause)
 -   [Секция DISTINCT](distinct.md)
 -   [Секция LIMIT](limit.md)
--   [Секция UNION ALL](union-all.md)
+-   [Секция UNION ALL](union.md)
 -   [Секция INTO OUTFILE](into-outfile.md)
 -   [Секция FORMAT](format.md)
 

docs/ru/sql-reference/statements/select/union-all.md

@@ -1,34 +0,0 @@
----
-toc_title: UNION ALL
----
-
-# Секция UNION ALL {#union-all-clause}
-
-Вы можете использовать `UNION ALL` чтобы объединить любое количество `SELECT` запросы путем расширения их результатов. Пример:
-
-``` sql
-SELECT CounterID, 1 AS table, toInt64(count()) AS c
-    FROM test.hits
-    GROUP BY CounterID
-
-UNION ALL
-
-SELECT CounterID, 2 AS table, sum(Sign) AS c
-    FROM test.visits
-    GROUP BY CounterID
-    HAVING c > 0
-```
-
-Результирующие столбцы сопоставляются по их индексу (порядку внутри `SELECT`). Если имена столбцов не совпадают, то имена для конечного результата берутся из первого запроса.
-
-При объединении выполняет приведение типов. Например, если два запроса имеют одно и то же поле с не-`Nullable` и `Nullable` совместимыми типами, полученные в результате `UNION ALL` данные будут иметь `Nullable` тип.
-
-Запросы, которые являются частью `UNION ALL` не могут быть заключен в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION ALL` запросы в подзапрос в секции [FROM](from.md).
-
-## Ограничения {#limitations}
-
-Поддерживается только `UNION ALL`. Обычный `UNION` (`UNION DISTINCT`) не поддерживается. Если вам это нужно `UNION DISTINCT`, вы можете написать `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`.
-
-## Детали реализации {#implementation-details}
-
-Запросы, которые являются частью `UNION ALL` выполняются параллельно, и их результаты могут быть смешаны вместе.

docs/ru/sql-reference/statements/select/union.md

@@ -0,0 +1,81 @@
+---
+toc_title: UNION
+---
+
+# Секция UNION {#union-clause}
+
+Вы можете использовать `UNION` в двух режимах: `UNION ALL` или `UNION DISTINCT`.
+
+Если `UNION` используется без указания `ALL` или `DISTINCT`, то его поведение определяется настройкой `union_default_mode`. Разница между `UNION ALL` и `UNION DISTINCT` в том, что `UNION DISTINCT` выполняет явное преобразование для результата объединения. Это равнозначно выражению `SELECT DISTINCT` из подзапроса, содержащего `UNION ALL`.
+
+Чтобы объединить любое количество `SELECT` запросов путем объединения их результатов, вы можете использовать `UNION`. Пример:
+
+``` sql
+SELECT CounterID, 1 AS table, toInt64(count()) AS c
+    FROM test.hits
+    GROUP BY CounterID
+
+UNION ALL
+
+SELECT CounterID, 2 AS table, sum(Sign) AS c
+    FROM test.visits
+    GROUP BY CounterID
+    HAVING c > 0
+```
+
+Результирующие столбцы сопоставляются по их индексу (порядку внутри `SELECT`). Если имена столбцов не совпадают, то имена для конечного результата берутся из первого запроса.
+
+При объединении выполняет приведение типов. Например, если два запроса имеют одно и то же поле с не-`Nullable` и `Nullable` совместимыми типами, полученные в результате `UNION` данные будут иметь `Nullable` тип.
+
+Запросы, которые являются частью `UNION`, могут быть заключены в круглые скобки. [ORDER BY](order-by.md) и [LIMIT](limit.md) применяются к отдельным запросам, а не к конечному результату. Если вам нужно применить преобразование к конечному результату, вы можете разместить все объединенные с помощью `UNION` запросы в подзапрос в секции [FROM](from.md).
+
+Если используете `UNION` без явного указания `UNION ALL` или `UNION DISTINCT`, то вы можете указать режим объединения с помощью настройки [union_default_mode](../../../operations/settings/settings.md#union-default-mode), значениями которой могут быть `ALL`, `DISTINCT` или пустая строка. Однако если вы используете `UNION` с настройкой `union_default_mode`, значением которой является пустая строка, то будет сгенерировано исключение. В следующих примерах продемонстрированы результаты запросов при разных значениях настройки.
+
+Запрос:
+
+```sql
+SET union_default_mode = 'DISTINCT';
+SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2;
+```
+
+Результат:
+
+```text
+┌─1─┐
+│ 1 │
+└───┘
+┌─1─┐
+│ 2 │
+└───┘
+┌─1─┐
+│ 3 │
+└───┘
+```
+
+Запрос:
+
+```sql
+SET union_default_mode = 'ALL';
+SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2;
+```
+
+Результат:
+
+```text
+┌─1─┐
+│ 1 │
+└───┘
+┌─1─┐
+│ 2 │
+└───┘
+┌─1─┐
+│ 2 │
+└───┘
+┌─1─┐
+│ 3 │
+└───┘
+```
+
+Запросы, которые являются частью `UNION/UNION ALL/UNION DISTINCT`, выполняются параллельно, и их результаты могут быть смешаны вместе.
+
+[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/union/) <!-- hide -->

docs/tr/sql-reference/statements/select/union-all.md

@@ -1 +0,0 @@
-../../../../en/sql-reference/statements/select/union-all.md

docs/tr/sql-reference/statements/select/union.md

@@ -0,0 +1 @@
+../../../../en/sql-reference/statements/select/union.md

docs/zh/interfaces/third-party/client-libraries.md

@@ -1,51 +1,66 @@
-# 第三方开发的库 {#di-san-fang-kai-fa-de-ku}
+---
+toc_priority: 26
+toc_title: 客户端开发库
+---
 
-!!! warning "放弃"
-    Yandex不维护下面列出的库，也没有进行任何广泛的测试以确保其质量。
+# 第三方开发库 {#client-libraries-from-third-party-developers}
+
+!!! warning "声明"
+Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试来确保它们的质量。
 
 -   Python
     -   [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)
-    -   [ﾂ环板driverｮﾂ嘉ｯﾂ偲](https://github.com/mymarilyn/clickhouse-driver)
-    -   [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://github.com/yurial/clickhouse-client)
+    -   [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
+    -   [clickhouse-client](https://github.com/yurial/clickhouse-client)
+    -   [aiochclient](https://github.com/maximdanilchenko/aiochclient)
 -   PHP
     -   [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
-    -   [8bitov/clickhouse-php客户端](https://packagist.org/packages/8bitov/clickhouse-php-client)
-    -   [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://packagist.org/packages/bozerkins/clickhouse-client)
-    -   [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://packagist.org/packages/simpod/clickhouse-client)
+    -   [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
+    -   [bozerkins/clickhouse-client](https://packagist.org/packages/bozerkins/clickhouse-client)
+    -   [simpod/clickhouse-client](https://packagist.org/packages/simpod/clickhouse-client)
     -   [seva-code/php-click-house-client](https://packagist.org/packages/seva-code/php-click-house-client)
-    -   [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://github.com/SeasX/SeasClick)
--   走吧
+    -   [SeasClick C++ client](https://github.com/SeasX/SeasClick)
+    -   [one-ck](https://github.com/lizhichao/one-ck)
+    -   [glushkovds/phpclickhouse-laravel](https://packagist.org/packages/glushkovds/phpclickhouse-laravel)
+-   Go
     -   [clickhouse](https://github.com/kshvakov/clickhouse/)
-    -   [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/roistat/go-clickhouse)
-    -   [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/mailru/go-clickhouse)
+    -   [go-clickhouse](https://github.com/roistat/go-clickhouse)
+    -   [mailrugo-clickhouse](https://github.com/mailru/go-clickhouse)
     -   [golang-clickhouse](https://github.com/leprosus/golang-clickhouse)
+-   Swift
+    -   [ClickHouseNIO](https://github.com/patrick-zippenfenig/ClickHouseNIO)
+    -   [ClickHouseVapor ORM](https://github.com/patrick-zippenfenig/ClickHouseVapor)
 -   NodeJs
-    -   [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人)](https://github.com/TimonKK/clickhouse)
-    -   [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/apla/node-clickhouse)
+    -   [clickhouse (NodeJs)](https://github.com/TimonKK/clickhouse)
+    -   [node-clickhouse](https://github.com/apla/node-clickhouse)
 -   Perl
     -   [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse)
     -   [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse)
-    -   [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://metacpan.org/release/AnyEvent-ClickHouse)
+    -   [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
 -   Ruby
-    -   [ﾂ暗ｪﾂ氾环催ﾂ団)](https://github.com/shlima/click_house)
-    -   [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/PNixx/clickhouse-activerecord)
+    -   [ClickHouse (Ruby)](https://github.com/shlima/click_house)
+    -   [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
 -   R
     -   [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r)
-    -   [RClickhouse](https://github.com/IMSMWU/RClickhouse)
+    -   [RClickHouse](https://github.com/IMSMWU/RClickHouse)
 -   Java
     -   [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java)
--   斯卡拉
-    -   [掳胫client-禄脢鹿脷露胫鲁隆鹿-client酶](https://github.com/crobox/clickhouse-scala-client)
+    -   [clickhouse-client](https://github.com/Ecwid/clickhouse-client)
+-   Scala
+    -   [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client)
 -   Kotlin
     -   [AORM](https://github.com/TanVD/AORM)
 -   C#
     -   [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient)
-    -   [克莱克豪斯Ado](https://github.com/killwort/ClickHouse-Net)
+    -   [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net)
+    -   [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client)
     -   [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net)
-    -   [克莱克豪斯客户](https://github.com/DarkWanderer/ClickHouse.Client)
--   仙丹
+-   Elixir
     -   [clickhousex](https://github.com/appodeal/clickhousex/)
--   尼姆
+    -   [pillar](https://github.com/sofakingworld/pillar)
+-   Nim
     -   [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse)
+-   Haskell
+    -   [hdbc-clickhouse](https://github.com/zaneli/hdbc-clickhouse)
 
-[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/client_libraries/) <!--hide-->
+[来源文章](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) <!--hide-->

docs/zh/interfaces/third-party/index.md

@@ -1,8 +1,16 @@
 ---
-machine_translated: true
-machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
-toc_folder_title: "\u7B2C\u4E09\u65B9"
+toc_folder_title: 第三方工具
 toc_priority: 24
 ---
 
+# 第三方工具 {#third-party-interfaces}
 
+这是第三方工具的链接集合，它们提供了一些ClickHouse的接口。它可以是可视化界面、命令行界面或API:
+
+-   [Client libraries](../../interfaces/third-party/client-libraries.md)
+-   [Integrations](../../interfaces/third-party/integrations.md)
+-   [GUI](../../interfaces/third-party/gui.md)
+-   [Proxies](../../interfaces/third-party/proxy.md)
+
+!!! note "注意"
+支持通用API的通用工具[ODBC](../../interfaces/odbc.md)或[JDBC](../../interfaces/jdbc.md)，通常也适用于ClickHouse，但这里没有列出，因为它们实在太多了。

docs/zh/interfaces/third-party/integrations.md

@@ -1,100 +1,108 @@
-# 第三方集成库 {#di-san-fang-ji-cheng-ku}
+---
+toc_priority: 27
+toc_title: 第三方集成库
+---
+
+# 第三方集成库 {#integration-libraries-from-third-party-developers}
 
 !!! warning "声明"
-    Yandex不维护下面列出的库，也没有进行任何广泛的测试以确保其质量。
+Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试来确保它们的质量。
 
-## 基建产品 {#ji-jian-chan-pin}
+## 基础设施 {#infrastructure-products}
 
--   关系数据库管理系统
+-   关系数据库
     -   [MySQL](https://www.mysql.com)
         -   [mysql2ch](https://github.com/long2ice/mysql2ch)
         -   [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support)
         -   [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader)
-        -   [horgh-复制器](https://github.com/larsnovikov/horgh-replicator)
+        -   [horgh-replicator](https://github.com/larsnovikov/horgh-replicator)
     -   [PostgreSQL](https://www.postgresql.org)
         -   [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw)
-        -   [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (使用 [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
+        -   [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
         -   [pg2ch](https://github.com/mkabilov/pg2ch)
+        -   [clickhouse_fdw](https://github.com/adjust/clickhouse_fdw)
     -   [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server)
-        -   [ClickHouseMightrator](https://github.com/zlzforever/ClickHouseMigrator)
+        -   [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator)
 -   消息队列
-    -   [卡夫卡](https://kafka.apache.org)
-        -   [clickhouse_sinker](https://github.com/housepower/clickhouse_sinker) (使用 [去客户](https://github.com/ClickHouse/clickhouse-go/))
+    -   [Kafka](https://kafka.apache.org)
+        -   [clickhouse_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/))
         -   [stream-loader-clickhouse](https://github.com/adform/stream-loader)
 -   流处理
     -   [Flink](https://flink.apache.org)
         -   [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink)
 -   对象存储
     -   [S3](https://en.wikipedia.org/wiki/Amazon_S3)
-    -   [ﾂ环板backupｮﾂ嘉ｯﾂ偲](https://github.com/AlexAkulov/clickhouse-backup)
+        -   [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup)
 -   容器编排
     -   [Kubernetes](https://kubernetes.io)
-    -   [clickhouse-操](https://github.com/Altinity/clickhouse-operator)
+        -   [clickhouse-operator](https://github.com/Altinity/clickhouse-operator)
 -   配置管理
-    -   [木偶](https://puppet.com)
-    -   [ﾂ环板/ｮﾂ嘉ｯﾂ偲](https://forge.puppet.com/innogames/clickhouse)
-    -   [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse)
--   监控
-    -   [石墨](https://graphiteapp.org)
+    -   [puppet](https://puppet.com)
+        -   [innogames/clickhouse](https://forge.puppet.com/innogames/clickhouse)
+        -   [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse)
+-   Monitoring
+    -   [Graphite](https://graphiteapp.org)
         -   [graphouse](https://github.com/yandex/graphouse)
-        -   [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/lomik/carbon-clickhouse) +
-        -   [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/lomik/graphite-clickhouse)
-        -   [石墨-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) -优化静态分区 [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) 如果从规则 [汇总配置](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) 可以应用
+        -   [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) +
+        -   [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse)
+        -   [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - optimizes staled partitions in [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) if rules from [rollup configuration](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) could be applied
     -   [Grafana](https://grafana.com/)
         -   [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana)
-    -   [普罗米修斯号](https://prometheus.io/)
+    -   [Prometheus](https://prometheus.io/)
         -   [clickhouse_exporter](https://github.com/f1yegor/clickhouse_exporter)
         -   [PromHouse](https://github.com/Percona-Lab/PromHouse)
-        -   [clickhouse_exporter](https://github.com/hot-wifi/clickhouse_exporter) （用途 [去客户](https://github.com/kshvakov/clickhouse/))
+        -   [clickhouse_exporter](https://github.com/hot-wifi/clickhouse_exporter) (uses [Go client](https://github.com/kshvakov/clickhouse/))
     -   [Nagios](https://www.nagios.org/)
         -   [check_clickhouse](https://github.com/exogroup/check_clickhouse/)
         -   [check_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py)
     -   [Zabbix](https://www.zabbix.com)
-        -   [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/Altinity/clickhouse-zabbix-template)
+        -   [clickhouse-zabbix-template](https://github.com/Altinity/clickhouse-zabbix-template)
     -   [Sematext](https://sematext.com/)
-        -   [clickhouse积分](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse)
--   记录
+        -   [clickhouse integration](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse)
+-   Logging
     -   [rsyslog](https://www.rsyslog.com/)
-        -   [鹿茫house omhousee酶](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html)
+        -   [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html)
     -   [fluentd](https://www.fluentd.org)
-        -   [loghouse](https://github.com/flant/loghouse) (对于 [Kubernetes](https://kubernetes.io))
-    -   [Sematext](https://www.sematext.com/logagent)
-        -   [logagent输出-插件-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/)
--   地理
+        -   [loghouse](https://github.com/flant/loghouse) (for [Kubernetes](https://kubernetes.io))
+    -   [logagent](https://www.sematext.com/logagent)
+        -   [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/)
+-   Geo
     -   [MaxMind](https://dev.maxmind.com/geoip/)
-        -   [ﾂ环板-ｮﾂ嘉ｯﾂ偲青clickｼﾂ氾ｶﾂ鉄ﾂ工ﾂ渉](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
+        -   [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
 
-## 编程语言生态系统 {#bian-cheng-yu-yan-sheng-tai-xi-tong}
+## 编程语言 {#programming-language-ecosystems}
 
 -   Python
     -   [SQLAlchemy](https://www.sqlalchemy.org)
-        -   [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/cloudflare/sqlalchemy-clickhouse) (使用 [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
-    -   [熊猫](https://pandas.pydata.org)
+        -   [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
+    -   [pandas](https://pandas.pydata.org)
         -   [pandahouse](https://github.com/kszucs/pandahouse)
 -   PHP
     -   [Doctrine](https://www.doctrine-project.org/)
         -   [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse)
 -   R
     -   [dplyr](https://db.rstudio.com/dplyr/)
-    -   [RClickhouse](https://github.com/IMSMWU/RClickhouse) (使用 [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/artpaul/clickhouse-cpp))
+        -   [RClickHouse](https://github.com/IMSMWU/RClickHouse) (uses [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp))
 -   Java
     -   [Hadoop](http://hadoop.apache.org)
-    -   [clickhouse-hdfs-装载机](https://github.com/jaykelin/clickhouse-hdfs-loader) (使用 [JDBC](../../sql-reference/table-functions/jdbc.md))
--   斯卡拉
+        -   [clickhouse-hdfs-loader](https://github.com/jaykelin/clickhouse-hdfs-loader) (uses [JDBC](../../sql-reference/table-functions/jdbc.md))
+-   Scala
     -   [Akka](https://akka.io)
-    -   [掳胫client-禄脢鹿脷露胫鲁隆鹿-client酶](https://github.com/crobox/clickhouse-scala-client)
+        -   [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client)
 -   C#
     -   [ADO.NET](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/ado-net-overview)
-    -   [克莱克豪斯Ado](https://github.com/killwort/ClickHouse-Net)
-    -   [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net)
-    -   [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations)
--   仙丹
+        -   [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net)
+        -   [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client)
+        -   [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net)
+        -   [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations)
+-   Elixir
     -   [Ecto](https://github.com/elixir-ecto/ecto)
-    -   [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto)
+        -   [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto)
 -   Ruby
     -   [Ruby on Rails](https://rubyonrails.org/)
         -   [activecube](https://github.com/bitquery/activecube)
+        -   [ActiveRecord](https://github.com/PNixx/clickhouse-activerecord)
     -   [GraphQL](https://github.com/graphql)
         -   [activecube-graphql](https://github.com/bitquery/activecube-graphql)
 
-[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/integrations/) <!--hide-->
+[源文章](https://clickhouse.tech/docs/en/interfaces/third-party/integrations/) <!--hide-->

docs/zh/interfaces/third-party/proxy.md

@@ -1,37 +1,44 @@
-# 来自第三方开发人员的代理服务器 {#lai-zi-di-san-fang-kai-fa-ren-yuan-de-dai-li-fu-wu-qi}
+---
+toc_priority: 29
+toc_title: 第三方代理
+---
 
-[chproxy](https://github.com/Vertamedia/chproxy) 是ClickHouse数据库的http代理和负载均衡器。
+# 第三方代理 {#proxy-servers-from-third-party-developers}
 
-特征
+## chproxy {#chproxy}
 
-*每用户路由和响应缓存。
-*灵活的限制。
-\*自动SSL证书续订。
+[chproxy](https://github.com/Vertamedia/chproxy), 是一个用于ClickHouse数据库的HTTP代理和负载均衡器。
 
-在Go中实现。
+特性:
+
+-   用户路由和响应缓存。
+-   灵活的限制。
+-   自动SSL证书续订。
+
+使用go语言实现。
 
 ## KittenHouse {#kittenhouse}
 
-[KittenHouse](https://github.com/VKCOM/kittenhouse) 设计为ClickHouse和应用程序服务器之间的本地代理，以防在应用程序端缓冲INSERT数据是不可能或不方便的。
+[KittenHouse](https://github.com/VKCOM/kittenhouse)被设计为ClickHouse和应用服务器之间的本地代理，以防不可能或不方便在应用程序端缓冲插入数据。
 
-特征：
+特性:
 
-*内存和磁盘数据缓冲。
-*每表路由。
-\*负载平衡和健康检查。
+-   内存和磁盘上的数据缓冲。
+-   表路由。
+-   负载平衡和运行状况检查。
 
-在Go中实现。
+使用go语言实现。
 
-## ﾂ环板-ｮﾂ嘉ｯﾂ偲 {#clickhouse-bulk}
+## ClickHouse-Bulk {#clickhouse-bulk}
 
-[ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/nikepan/clickhouse-bulk) 是一个简单的ClickHouse插入收集器。
+[ClickHouse-Bulk](https://github.com/nikepan/clickhouse-bulk)是一个简单的ClickHouse收集器。
 
-特征：
+特性:
 
-*分组请求并按阈值或间隔发送。
-*多个远程服务器。
-\*基本身份验证。
+-   按阈值或间隔对请求进行分组并发送。
+-   多个远程服务器。
+-   基本身份验证。
 
-在Go中实现。
+使用go语言实现。
 
-[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/proxy/) <!--hide-->
+[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/proxy/) <!--hide-->

docs/zh/sql-reference/statements/select/index.md

@@ -46,7 +46,7 @@ SELECT [DISTINCT] expr_list
 -   [SELECT 子句](#select-clause)
 -   [DISTINCT 子句](../../../sql-reference/statements/select/distinct.md)
 -   [LIMIT 子句](../../../sql-reference/statements/select/limit.md)
--   [UNION ALL 子句](../../../sql-reference/statements/select/union-all.md)
+-   [UNION ALL 子句](../../../sql-reference/statements/select/union.md)
 -   [INTO OUTFILE 子句](../../../sql-reference/statements/select/into-outfile.md)
 -   [FORMAT 子句](../../../sql-reference/statements/select/format.md)
 

docs/zh/sql-reference/statements/select/union.md

@@ -2,7 +2,7 @@
 toc_title: UNION ALL
 ---
 
-# UNION ALL子句 {#union-all-clause}
+# UNION ALL子句 {#union-clause}
 
 你可以使用 `UNION ALL` 结合任意数量的 `SELECT` 来扩展其结果。 示例:
 

docs/zh/whats-new/index.md

@@ -1,8 +1,8 @@
 ---
-machine_translated: true
-machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
-toc_folder_title: "\u65B0\u589E\u5185\u5BB9"
-toc_priority: 72
+toc_folder_title: ClickHouse事迹
+toc_priority: 82
 ---
 
+# ClickHouse变更及改动? {#whats-new-in-clickhouse}
 
+对于已经发布的版本，有一个[roadmap](../whats-new/roadmap.md)和一个详细的[changelog](../whats-new/changelog/index.md)。

docs/zh/whats-new/roadmap.md

@@ -1,17 +1,10 @@
 ---
 toc_priority: 74
-toc_title: 路线图
+toc_title: Roadmap
 ---
 
-# 路线图 {#roadmap}
+# Roadmap {#roadmap}
 
-## Q2 2020 {#q2-2020}
-
--  和外部认证服务集成
-
-## Q3 2020 {#q3-2020}
-
--   资源池，为用户提供更精准的集群资源分配
-
-{## [原始文档](https://clickhouse.tech/docs/en/roadmap/) ##}
+`2021年Roadmap`已公布供公开讨论查看[这里](https://github.com/ClickHouse/ClickHouse/issues/17623).
 
+{## [源文章](https://clickhouse.tech/docs/en/roadmap/) ##}

docs/zh/whats-new/security-changelog.md

@@ -1,41 +1,74 @@
-## 修复于 ClickHouse Release 18.12.13, 2018-09-10 {#xiu-fu-yu-clickhouse-release-18-12-13-2018-09-10}
+---
+toc_priority: 76
+toc_title: 安全更新日志
+---
+
+## 修复于ClickHouse Release 19.14.3.3, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10}
+
+### CVE-2019-15024 {#cve-2019-15024}
+
+对ZooKeeper具有写访问权限并且可以运行ClickHouse所在网络上可用的自定义服务器的攻击者可以创建一个自定义的恶意服务器，该服务器将充当ClickHouse副本并在ZooKeeper中注册。当另一个副本将从恶意副本获取数据部分时，它可以强制clickhouse服务器写入文件系统上的任意路径。
+
+作者：Yandex信息安全团队Eldar Zaitov
+
+### CVE-2019-16535 {#cve-2019-16535}
+
+解压算法中的OOB-read、OOB-write和整数下溢可以通过本机协议实现RCE或DoS。
+
+作者: Yandex信息安全团队Eldar Zaitov
+
+### CVE-2019-16536 {#cve-2019-16536}
+
+恶意的经过身份验证的客户端可能会触发导致DoS的堆栈溢出。
+
+作者: Yandex信息安全团队Eldar Zaitov
+
+## 修复于ClickHouse Release 19.13.6.1, 2019-09-20 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20}
+
+### CVE-2019-18657 {#cve-2019-18657}
+
+表函数`url`存在允许攻击者在请求中插入任意HTTP标头的漏洞。
+
+作者: [Nikita Tikhomirov](https://github.com/NSTikhomirov)
+
+## 修复于ClickHouse Release 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10}
 
 ### CVE-2018-14672 {#cve-2018-14672}
 
-加载CatBoost模型的功能，允许遍历路径并通过错误消息读取任意文件。
+加载CatBoost模型的函数允许路径遍历和通过错误消息读取任意文件。
 
-来源: Yandex信息安全团队的Andrey Krasichkov
+作者：Yandex信息安全团队Andrey Krasichkov
 
-## 修复于 ClickHouse Release 18.10.3, 2018-08-13 {#xiu-fu-yu-clickhouse-release-18-10-3-2018-08-13}
+## 修复于Release 18.10.3, 2018-08-13 {#fixed-in-clickhouse-release-18-10-3-2018-08-13}
 
 ### CVE-2018-14671 {#cve-2018-14671}
 
-unixODBC允许从文件系统加载任意共享对象，从而导致«远程执行代码»漏洞。
+unixODBC允许从文件系统加载任意共享对象，从而导致远程代码执行漏洞。
 
-来源：Yandex信息安全团队的Andrey Krasichkov和Evgeny Sidorov
+作者：Yandex信息安全团队Andrey Krasichkov和Evgeny Sidorov
 
-## 修复于 ClickHouse Release 1.1.54388, 2018-06-28 {#xiu-fu-yu-clickhouse-release-1-1-54388-2018-06-28}
+## 修复于ClickHouse Release 1.1.54388, 2018-06-28 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28}
 
 ### CVE-2018-14668 {#cve-2018-14668}
 
-远程表函数功能允许在 «user», «password» 及 «default_database» 字段中使用任意符号，从而导致跨协议请求伪造攻击。
+`remote`表函数允许在`user`，`password`和`default_database`字段中使用任意符号，从而导致跨协议请求伪造攻击。
 
-来源：Yandex信息安全团队的Andrey Krasichkov
+者：Yandex信息安全团队Andrey Krasichkov
 
-## 修复于 ClickHouse Release 1.1.54390, 2018-07-06 {#xiu-fu-yu-clickhouse-release-1-1-54390-2018-07-06}
+## 修复于ClickHouse Release 1.1.54390, 2018-07-06 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06}
 
 ### CVE-2018-14669 {#cve-2018-14669}
 
-ClickHouse MySQL客户端启用了 «LOAD DATA LOCAL INFILE» 功能，该功能允许恶意MySQL数据库从连接的ClickHouse服务器读取任意文件。
+ClickHouse MySQL客户端启用了`LOAD DATA LOCAL INFILE`功能，允许恶意MySQL数据库从连接的ClickHouse服务器读取任意文件。
 
-来源：Yandex信息安全团队的Andrey Krasichkov和Evgeny Sidorov
+作者：Yandex信息安全团队Andrey Krasichkov和Evgeny Sidorov
 
-## 修复于 ClickHouse Release 1.1.54131, 2017-01-10 {#xiu-fu-yu-clickhouse-release-1-1-54131-2017-01-10}
+## 修复于ClickHouse Release 1.1.54131, 2017-01-10 {#fixed-in-clickhouse-release-1-1-54131-2017-01-10}
 
 ### CVE-2018-14670 {#cve-2018-14670}
 
-deb软件包中的错误配置可能导致使用未经授权的数据库。
+deb包中的错误配置可能导致未经授权使用数据库。
 
-来源：英国国家网络安全中心（NCSC）
+作者：英国国家网络安全中心（NCSC）
 
-[来源文章](https://clickhouse.tech/docs/en/security_changelog/) <!--hide-->
+{## [Original article](https://clickhouse.tech/docs/en/security_changelog/) ##}

programs/client/QueryFuzzer.cpp

@@ -405,8 +405,8 @@ void QueryFuzzer::fuzz(ASTPtr & ast)
 
         if (fn->is_window_function)
         {
-            fuzzColumnLikeExpressionList(fn->window_partition_by);
-            fuzzOrderByList(fn->window_order_by);
+            fuzzColumnLikeExpressionList(fn->window_partition_by.get());
+            fuzzOrderByList(fn->window_order_by.get());
         }
 
         fuzz(fn->children);

programs/compressor/Compressor.cpp

@@ -6,8 +6,11 @@
 #include <Common/Exception.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/ReadBufferFromFileDescriptor.h>
+#include <IO/WriteBufferFromFile.h>
+#include <IO/ReadBufferFromFile.h>
 #include <Compression/CompressedWriteBuffer.h>
 #include <Compression/CompressedReadBuffer.h>
+#include <Compression/CompressedReadBufferFromFile.h>
 #include <IO/WriteHelpers.h>
 #include <IO/copyData.h>
 #include <Parsers/parseQuery.h>
@@ -58,34 +61,41 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out)
 
 }
 
-#pragma GCC diagnostic ignored "-Wunused-function"
-#pragma GCC diagnostic ignored "-Wmissing-declarations"
-
 int mainEntryClickHouseCompressor(int argc, char ** argv)
 {
     using namespace DB;
+    namespace po = boost::program_options;
 
-    boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
+    po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
     desc.add_options()
         ("help,h", "produce help message")
+        ("input", po::value<std::string>()->value_name("INPUT"), "input file")
+        ("output", po::value<std::string>()->value_name("OUTPUT"), "output file")
         ("decompress,d", "decompress")
-        ("block-size,b", boost::program_options::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
+        ("offset-in-compressed-file", po::value<size_t>()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)")
+        ("offset-in-decompressed-block", po::value<size_t>()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)")
+        ("block-size,b", po::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
         ("hc", "use LZ4HC instead of LZ4")
         ("zstd", "use ZSTD instead of LZ4")
-        ("codec", boost::program_options::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
-        ("level", boost::program_options::value<int>(), "compression level for codecs specified via flags")
+        ("codec", po::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
+        ("level", po::value<int>(), "compression level for codecs specified via flags")
         ("none", "use no compression instead of LZ4")
         ("stat", "print block statistics of compressed data")
     ;
 
-    boost::program_options::variables_map options;
-    boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
+    po::positional_options_description positional_desc;
+    positional_desc.add("input", 1);
+    positional_desc.add("output", 1);
+
+    po::variables_map options;
+    po::store(po::command_line_parser(argc, argv).options(desc).positional(positional_desc).run(), options);
 
     if (options.count("help"))
     {
-        std::cout << "Usage: " << argv[0] << " [options] < in > out" << std::endl;
+        std::cout << "Usage: " << argv[0] << " [options] < INPUT > OUTPUT" << std::endl;
+        std::cout << "Usage: " << argv[0] << " [options] INPUT OUTPUT" << std::endl;
         std::cout << desc << std::endl;
-        return 1;
+        return 0;
     }
 
     try
@@ -132,25 +142,52 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
             codec = CompressionCodecFactory::instance().get(method_family, level);
 
 
-        ReadBufferFromFileDescriptor rb(STDIN_FILENO);
-        WriteBufferFromFileDescriptor wb(STDOUT_FILENO);
+        std::unique_ptr<ReadBufferFromFileBase> rb;
+        std::unique_ptr<WriteBufferFromFileBase> wb;
+
+        if (options.count("input"))
+            rb = std::make_unique<ReadBufferFromFile>(options["input"].as<std::string>());
+        else
+            rb = std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
+
+        if (options.count("output"))
+            wb = std::make_unique<WriteBufferFromFile>(options["output"].as<std::string>());
+        else
+            wb = std::make_unique<WriteBufferFromFileDescriptor>(STDOUT_FILENO);
 
         if (stat_mode)
         {
             /// Output statistic for compressed file.
-            checkAndWriteHeader(rb, wb);
+            checkAndWriteHeader(*rb, *wb);
         }
         else if (decompress)
         {
             /// Decompression
-            CompressedReadBuffer from(rb);
-            copyData(from, wb);
+
+            size_t offset_in_compressed_file = options["offset-in-compressed-file"].as<size_t>();
+            size_t offset_in_decompressed_block = options["offset-in-decompressed-block"].as<size_t>();
+
+            if (offset_in_compressed_file || offset_in_decompressed_block)
+            {
+                if (!options.count("input"))
+                {
+                    throw DB::Exception("--offset-in-compressed-file/--offset-in-decompressed-block requires --input", DB::ErrorCodes::BAD_ARGUMENTS);
+                }
+                CompressedReadBufferFromFile compressed_file(options["input"].as<std::string>(), 0, 0, 0);
+                compressed_file.seek(offset_in_compressed_file, offset_in_decompressed_block);
+                copyData(compressed_file, *wb);
+            }
+            else
+            {
+                CompressedReadBuffer from(*rb);
+                copyData(from, *wb);
+            }
         }
         else
         {
             /// Compression
-            CompressedWriteBuffer to(wb, codec, block_size);
-            copyData(rb, to);
+            CompressedWriteBuffer to(*wb, codec, block_size);
+            copyData(*rb, to);
         }
     }
     catch (...)

programs/server/config.xml

@@ -324,6 +324,9 @@
                 auth_dn_prefix, auth_dn_suffix - prefix and suffix used to construct the DN to bind to.
                         Effectively, the resulting DN will be constructed as auth_dn_prefix + escape(user_name) + auth_dn_suffix string.
                         Note, that this implies that auth_dn_suffix should usually have comma ',' as its first non-space character.
+                verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed
+                         to be successfully authenticated for all consecutive requests without contacting the LDAP server.
+                        Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request.
                 enable_tls - flag to trigger use of secure connection to the LDAP server.
                         Specify 'no' for plain text (ldap://) protocol (not recommended).
                         Specify 'yes' for LDAP over SSL/TLS (ldaps://) protocol (recommended, the default).
@@ -343,6 +346,7 @@
                     <port>636</port>
                     <auth_dn_prefix>uid=</auth_dn_prefix>
                     <auth_dn_suffix>,ou=users,dc=example,dc=com</auth_dn_suffix>
+                    <verification_cooldown>300</verification_cooldown>
                     <enable_tls>yes</enable_tls>
                     <tls_minimum_protocol_version>tls1.2</tls_minimum_protocol_version>
                     <tls_require_cert>demand</tls_require_cert>

src/Access/Authentication.cpp

@@ -1,6 +1,5 @@
 #include <Access/Authentication.h>
 #include <Access/ExternalAuthenticators.h>
-#include <Access/LDAPClient.h>
 #include <Common/Exception.h>
 #include <Poco/SHA1Engine.h>
 
@@ -49,7 +48,7 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const
 }
 
 
-bool Authentication::isCorrectPassword(const String & password_, const String & user_, const ExternalAuthenticators & external_authenticators) const
+bool Authentication::isCorrectPassword(const String & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const
 {
     switch (type)
     {
@@ -81,14 +80,7 @@ bool Authentication::isCorrectPassword(const String & password_, const String &
         }
 
         case LDAP_SERVER:
-        {
-            auto ldap_server_params = external_authenticators.getLDAPServerParams(server_name);
-            ldap_server_params.user = user_;
-            ldap_server_params.password = password_;
-
-            LDAPSimpleAuthClient ldap_client(ldap_server_params);
-            return ldap_client.check();
-        }
+            return external_authenticators.checkLDAPCredentials(server_name, user_, password_);
 
         case MAX_TYPE:
             break;

src/Access/Authentication.h

@@ -88,8 +88,8 @@ public:
     void setServerName(const String & server_name_);
 
     /// Checks if the provided password is correct. Returns false if not.
-    /// User name and external authenticators' info are used only by some specific authentication type (e.g., LDAP_SERVER).
-    bool isCorrectPassword(const String & password_, const String & user_, const ExternalAuthenticators & external_authenticators) const;
+    /// User name and external authenticators are used by the specific authentication types only (e.g., LDAP_SERVER).
+    bool isCorrectPassword(const String & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const;
 
     friend bool operator ==(const Authentication & lhs, const Authentication & rhs) { return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash); }
     friend bool operator !=(const Authentication & lhs, const Authentication & rhs) { return !(lhs == rhs); }

src/Access/ExternalAuthenticators.cpp

@@ -1,9 +1,13 @@
 #include <Access/ExternalAuthenticators.h>
+#include <Access/LDAPClient.h>
 #include <Common/Exception.h>
 #include <Common/quoteString.h>
 #include <Poco/Util/AbstractConfiguration.h>
 #include <boost/algorithm/string/case_conv.hpp>
 
+#include <optional>
+#include <utility>
+
 
 namespace DB
 {
@@ -29,6 +33,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
     const bool has_port = config.has(ldap_server_config + ".port");
     const bool has_auth_dn_prefix = config.has(ldap_server_config + ".auth_dn_prefix");
     const bool has_auth_dn_suffix = config.has(ldap_server_config + ".auth_dn_suffix");
+    const bool has_verification_cooldown = config.has(ldap_server_config + ".verification_cooldown");
     const bool has_enable_tls = config.has(ldap_server_config + ".enable_tls");
     const bool has_tls_minimum_protocol_version = config.has(ldap_server_config + ".tls_minimum_protocol_version");
     const bool has_tls_require_cert = config.has(ldap_server_config + ".tls_require_cert");
@@ -52,6 +57,9 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
     if (has_auth_dn_suffix)
         params.auth_dn_suffix = config.getString(ldap_server_config + ".auth_dn_suffix");
 
+    if (has_verification_cooldown)
+        params.verification_cooldown = std::chrono::seconds{config.getUInt64(ldap_server_config + ".verification_cooldown")};
+
     if (has_enable_tls)
     {
         String enable_tls_lc_str = config.getString(ldap_server_config + ".enable_tls");
@@ -130,16 +138,28 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
     return params;
 }
 
-void parseAndAddLDAPServers(ExternalAuthenticators & external_authenticators, const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
+}
+
+void ExternalAuthenticators::reset()
 {
+    std::scoped_lock lock(mutex);
+    ldap_server_params.clear();
+    ldap_server_caches.clear();
+}
+
+void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
+{
+    std::scoped_lock lock(mutex);
+
+    reset();
+
     Poco::Util::AbstractConfiguration::Keys ldap_server_names;
     config.keys("ldap_servers", ldap_server_names);
-
     for (const auto & ldap_server_name : ldap_server_names)
     {
         try
         {
-            external_authenticators.setLDAPServerParams(ldap_server_name, parseLDAPServer(config, ldap_server_name));
+            ldap_server_params.insert_or_assign(ldap_server_name, parseLDAPServer(config, ldap_server_name));
         }
         catch (...)
         {
@@ -148,35 +168,100 @@ void parseAndAddLDAPServers(ExternalAuthenticators & external_authenticators, co
     }
 }
 
-}
-
-void ExternalAuthenticators::reset()
+bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const String & user_name, const String & password) const
 {
-    std::scoped_lock lock(mutex);
-    ldap_server_params.clear();
-}
+    std::optional<LDAPServerParams> params;
+    std::size_t params_hash = 0;
 
-void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
-{
-    std::scoped_lock lock(mutex);
-    reset();
-    parseAndAddLDAPServers(*this, config, log);
-}
+    {
+        std::scoped_lock lock(mutex);
 
-void ExternalAuthenticators::setLDAPServerParams(const String & server, const LDAPServerParams & params)
-{
-    std::scoped_lock lock(mutex);
-    ldap_server_params.erase(server);
-    ldap_server_params[server] = params;
-}
+        // Retrieve the server parameters.
+        const auto pit = ldap_server_params.find(server);
+        if (pit == ldap_server_params.end())
+            throw Exception("LDAP server '" + server + "' is not configured", ErrorCodes::BAD_ARGUMENTS);
 
-LDAPServerParams ExternalAuthenticators::getLDAPServerParams(const String & server) const
-{
-    std::scoped_lock lock(mutex);
-    auto it = ldap_server_params.find(server);
-    if (it == ldap_server_params.end())
-        throw Exception("LDAP server '" + server + "' is not configured", ErrorCodes::BAD_ARGUMENTS);
-    return it->second;
+        params = pit->second;
+        params->user = user_name;
+        params->password = password;
+        params_hash = params->getCoreHash();
+
+        // Check the cache, but only if the caching is enabled at all.
+        if (params->verification_cooldown > std::chrono::seconds{0})
+        {
+            const auto cit = ldap_server_caches.find(server);
+            if (cit != ldap_server_caches.end())
+            {
+                auto & cache = cit->second;
+
+                const auto eit = cache.find(user_name);
+                if (eit != cache.end())
+                {
+                    const auto & entry = eit->second;
+                    const auto last_check_period = std::chrono::steady_clock::now() - entry.last_successful_authentication_timestamp;
+
+                    if (
+                        // Forbid the initial values explicitly.
+                        entry.last_successful_params_hash != 0 &&
+                        entry.last_successful_authentication_timestamp != std::chrono::steady_clock::time_point{} &&
+
+                        // Check if we can safely "reuse" the result of the previous successful password verification.
+                        entry.last_successful_params_hash == params_hash &&
+                        last_check_period >= std::chrono::seconds{0} &&
+                        last_check_period <= params->verification_cooldown
+                    )
+                    {
+                        return true;
+                    }
+
+                    // Erase the entry, if expired.
+                    if (last_check_period > params->verification_cooldown)
+                        cache.erase(eit);
+                }
+
+                // Erase the cache, if empty.
+                if (cache.empty())
+                    ldap_server_caches.erase(cit);
+            }
+        }
+    }
+
+    LDAPSimpleAuthClient client(params.value());
+    const auto result = client.check();
+    const auto current_check_timestamp = std::chrono::steady_clock::now();
+
+    // Update the cache, but only if this is the latest check and the server is still configured in a compatible way.
+    if (result)
+    {
+        std::scoped_lock lock(mutex);
+
+        // If the server was removed from the config while we were checking the password, we discard the current result.
+        const auto pit = ldap_server_params.find(server);
+        if (pit == ldap_server_params.end())
+            return false;
+
+        auto new_params = pit->second;
+        new_params.user = user_name;
+        new_params.password = password;
+
+        // If the critical server params have changed while we were checking the password, we discard the current result.
+        if (params_hash != new_params.getCoreHash())
+            return false;
+
+        auto & entry = ldap_server_caches[server][user_name];
+        if (entry.last_successful_authentication_timestamp < current_check_timestamp)
+        {
+            entry.last_successful_params_hash = params_hash;
+            entry.last_successful_authentication_timestamp = current_check_timestamp;
+        }
+        else if (entry.last_successful_params_hash != params_hash)
+        {
+            // Somehow a newer check with different params/password succeeded, so the current result is obsolete and we discard it.
+            return false;
+        }
+    }
+
+    return result;
 }
 
 }

src/Access/ExternalAuthenticators.h

@@ -3,9 +3,10 @@
 #include <Access/LDAPParams.h>
 #include <common/types.h>
 
+#include <chrono>
 #include <map>
-#include <memory>
 #include <mutex>
+#include <unordered_map>
 
 
 namespace Poco
@@ -27,13 +28,23 @@ class ExternalAuthenticators
 public:
     void reset();
     void setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log);
+    bool checkLDAPCredentials(const String & server, const String & user_name, const String & password) const;
 
-    void setLDAPServerParams(const String & server, const LDAPServerParams & params);
-    LDAPServerParams getLDAPServerParams(const String & server) const;
+private:
+    struct LDAPCacheEntry
+    {
+        std::size_t last_successful_params_hash = 0;
+        std::chrono::steady_clock::time_point last_successful_authentication_timestamp;
+    };
+
+    using LDAPServerCache = std::unordered_map<String, LDAPCacheEntry>; // user name   -> cache entry
+    using LDAPServerCaches = std::map<String, LDAPServerCache>;         // server name -> cache
+    using LDAPServersParams = std::map<String, LDAPServerParams>;       // server name -> params
 
 private:
     mutable std::recursive_mutex mutex;
-    std::map<String, LDAPServerParams> ldap_server_params;
+    LDAPServersParams ldap_server_params;
+    mutable LDAPServerCaches ldap_server_caches;
 };
 
 }

src/Access/IAccessStorage.cpp

@@ -463,7 +463,7 @@ UUID IAccessStorage::loginImpl(
 
 bool IAccessStorage::isPasswordCorrectImpl(const User & user, const String & password, const ExternalAuthenticators & external_authenticators) const
 {
-    return user.authentication.isCorrectPassword(password, user.getName(), external_authenticators);
+    return user.authentication.isCorrectPassword(user.getName(), password, external_authenticators);
 }
 
 

src/Access/LDAPParams.h

@@ -2,6 +2,8 @@
 
 #include <common/types.h>
 
+#include <boost/container_hash/hash.hpp>
+
 #include <chrono>
 
 
@@ -68,10 +70,26 @@ struct LDAPServerParams
     String user;
     String password;
 
+    std::chrono::seconds verification_cooldown{0};
+
     std::chrono::seconds operation_timeout{40};
     std::chrono::seconds network_timeout{30};
     std::chrono::seconds search_timeout{20};
     std::uint32_t search_limit = 100;
+
+    std::size_t getCoreHash() const
+    {
+        std::size_t seed = 0;
+
+        boost::hash_combine(seed, host);
+        boost::hash_combine(seed, port);
+        boost::hash_combine(seed, auth_dn_prefix);
+        boost::hash_combine(seed, auth_dn_suffix);
+        boost::hash_combine(seed, user);
+        boost::hash_combine(seed, password);
+
+        return seed;
+    }
 };
 
 }

src/Common/ThreadStatus.cpp

@@ -96,7 +96,7 @@ ThreadStatus::~ThreadStatus()
     catch (const DB::Exception &)
     {
         /// It's a minor tracked memory leak here (not the memory itself but it's counter).
-        /// We've already allocated a little bit more then the limit and cannot track it in the thread memory tracker or its parent.
+        /// We've already allocated a little bit more than the limit and cannot track it in the thread memory tracker or its parent.
     }
 
     if (deleter)

src/Core/ExternalResultDescription.cpp

@@ -4,6 +4,7 @@
 #include <DataTypes/DataTypeDateTime64.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeFixedString.h>
 #include <DataTypes/DataTypeUUID.h>
 #include <DataTypes/DataTypesDecimal.h>
 #include <DataTypes/DataTypesNumber.h>
@@ -76,6 +77,8 @@ void ExternalResultDescription::init(const Block & sample_block_)
             types.emplace_back(ValueType::vtDecimal128, is_nullable);
         else if (typeid_cast<const DataTypeDecimal<Decimal256> *>(type))
             types.emplace_back(ValueType::vtDecimal256, is_nullable);
+        else if (typeid_cast<const DataTypeFixedString *>(type))
+            types.emplace_back(ValueType::vtFixedString, is_nullable);
         else
             throw Exception{"Unsupported type " + type->getName(), ErrorCodes::UNKNOWN_TYPE};
     }

src/Core/ExternalResultDescription.h

@@ -30,7 +30,8 @@ struct ExternalResultDescription
         vtDecimal32,
         vtDecimal64,
         vtDecimal128,
-        vtDecimal256
+        vtDecimal256,
+        vtFixedString
     };
 
     Block sample_block;

src/Core/Settings.h

@@ -417,7 +417,7 @@ class IColumn;
     M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing. Will be removed after 2021-03-31", 0) \
     M(Bool, enable_debug_queries, false, "Enabled debug queries, but now is obsolete", 0) \
     M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing. Will be removed after 2021-02-12", 0) \
-    M(UnionMode, union_default_mode, UnionMode::DISTINCT, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) \
+    M(UnionMode, union_default_mode, UnionMode::Unspecified, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) \
     M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \
     M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \
 

src/Dictionaries/IPAddressDictionary.cpp

@@ -518,7 +518,7 @@ void IPAddressDictionary::loadData()
     {
         /// We format key attribute values here instead of filling with data from key_column
         /// because string representation can be normalized if bits beyond mask are set.
-        /// Also all IPv4 will be displayed as mapped IPv6 if threre are any IPv6.
+        /// Also all IPv4 will be displayed as mapped IPv6 if there are any IPv6.
         /// It's consistent with representation in table created with `ENGINE = Dictionary` from this dictionary.
         char str_buffer[48];
         if (has_ipv6)

src/Formats/MySQLBlockInputStream.cpp

@@ -8,6 +8,7 @@
 #    include <Columns/ColumnString.h>
 #    include <Columns/ColumnsNumber.h>
 #    include <Columns/ColumnDecimal.h>
+#    include <Columns/ColumnFixedString.h>
 #    include <DataTypes/IDataType.h>
 #    include <DataTypes/DataTypeNullable.h>
 #    include <IO/ReadHelpers.h>
@@ -110,6 +111,9 @@ namespace
                 data_type.deserializeAsWholeText(column, buffer, FormatSettings{});
                 break;
             }
+            case ValueType::vtFixedString:
+                assert_cast<ColumnFixedString &>(column).insertData(value.data(), value.size());
+                break;
         }
     }
 

src/Functions/ExtractString.h

@@ -20,8 +20,7 @@ namespace DB
 // includes extracting ASCII ngram, UTF8 ngram, ASCII word and UTF8 word
 struct ExtractStringImpl
 {
-    // read a ASCII word
-    static ALWAYS_INLINE inline const UInt8 * readOneASCIIWord(const UInt8 *& pos, const UInt8 * end)
+    static ALWAYS_INLINE inline const UInt8 * readOneWord(const UInt8 *& pos, const UInt8 * end)
     {
         // jump separators
         while (pos < end && isUTF8Sep(*pos))
@@ -35,22 +34,6 @@ struct ExtractStringImpl
         return word_start;
     }
 
-    // read one UTF8 word from pos to word
-    static ALWAYS_INLINE inline const UInt8 * readOneUTF8Word(const UInt8 *& pos, const UInt8 * end)
-    {
-        // jump UTF8 separator
-        while (pos < end && isUTF8Sep(*pos))
-            ++pos;
-
-        // UTF8 word's character number
-        const UInt8 * word_start = pos;
-
-        while (pos < end && !isUTF8Sep(*pos))
-            readOneUTF8Code(pos, end);
-
-        return word_start;
-    }
-
     // we use ASCII non-alphanum character as UTF8 separator
     static ALWAYS_INLINE inline bool isUTF8Sep(const UInt8 c) { return c < 128 && !isAlphaNumericASCII(c); }
 

src/Functions/FunctionsStringHash.cpp

@@ -249,13 +249,7 @@ struct SimHashImpl
         // get first word shingle
         while (start < end && words.size() < shingle_size)
         {
-            const UInt8 * word_start;
-
-            if constexpr (UTF8)
-                word_start = ExtractStringImpl::readOneUTF8Word(start, end);
-            else
-                word_start = ExtractStringImpl::readOneASCIIWord(start, end);
-
+            const UInt8 * word_start = ExtractStringImpl::readOneWord(start, end);
             size_t length = start - word_start;
 
             if (length >= min_word_size)
@@ -271,13 +265,7 @@ struct SimHashImpl
         size_t offset = 0;
         while (start < end)
         {
-            const UInt8 * word_start;
-
-            if constexpr (UTF8)
-                word_start = ExtractStringImpl::readOneUTF8Word(start, end);
-            else
-                word_start = ExtractStringImpl::readOneASCIIWord(start, end);
-
+            const UInt8 * word_start = ExtractStringImpl::readOneWord(start, end);
             size_t length = start - word_start;
 
             if (length < min_word_size)
@@ -340,7 +328,7 @@ struct MinHashImpl
 {
     static constexpr size_t min_word_size = 4;
 
-    template<typename Comp>
+    template <typename Comp>
     struct Heap
     {
         void update(UInt64 hash, BytesRef ref, size_t limit)
@@ -478,13 +466,7 @@ struct MinHashImpl
         // get first word shingle
         while (start < end && words.size() < shingle_size)
         {
-            const UInt8 * word_start;
-
-            if constexpr (UTF8)
-                word_start = ExtractStringImpl::readOneUTF8Word(start, end);
-            else
-                word_start = ExtractStringImpl::readOneASCIIWord(start, end);
-
+            const UInt8 * word_start = ExtractStringImpl::readOneWord(start, end);
             size_t length = start - word_start;
 
             if (length >= min_word_size)
@@ -506,12 +488,7 @@ struct MinHashImpl
         size_t offset = 0;
         while (start < end)
         {
-            const UInt8 * word_start;
-
-            if constexpr (UTF8)
-                word_start = ExtractStringImpl::readOneUTF8Word(start, end);
-            else
-                word_start = ExtractStringImpl::readOneASCIIWord(start, end);
+            const UInt8 * word_start = ExtractStringImpl::readOneWord(start, end);
 
             size_t length = start - word_start;
 

src/Functions/FunctionsStringHash.h

@@ -77,7 +77,7 @@ public:
         {
             if constexpr (is_simhash)
                 throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION,
-                                "Function {} expect no more then two arguments (text, shingle size), got {}",
+                                "Function {} expect no more than two arguments (text, shingle size), got {}",
                                 getName(), arguments.size());
 
             if (!isUnsignedInteger(arguments[2].type))
@@ -95,7 +95,7 @@ public:
         if (arguments.size() > 3)
         {
             throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION,
-                            "Function {} expect no more then three arguments (text, shingle size, num hashes), got {}",
+                            "Function {} expect no more than three arguments (text, shingle size, num hashes), got {}",
                             getName(), arguments.size());
         }
 

src/Interpreters/ActionsVisitor.cpp

@@ -738,15 +738,26 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
     if (node.is_window_function)
     {
         // Also add columns from PARTITION BY and ORDER BY of window functions.
-        // Requiring a constant reference to a shared pointer to non-const AST
-        // doesn't really look sane, but the visitor does indeed require it.
         if (node.window_partition_by)
         {
-            visit(node.window_partition_by->clone(), data);
+            visit(node.window_partition_by, data);
         }
         if (node.window_order_by)
         {
-            visit(node.window_order_by->clone(), data);
+            visit(node.window_order_by, data);
+        }
+
+        // Also manually add columns for arguments of the window function itself.
+        // ActionVisitor is written in such a way that this method must itself
+        // descend into all needed function children. Window functions can't have
+        // any special functions as argument, so the code below that handles
+        // special arguments is not needed. This is analogous to the
+        // appendWindowFunctionsArguments() in SelectQueryExpressionAnalyzer and
+        // partially duplicates its code. Probably we can remove most of the
+        // logic from that function, but I don't yet have it all figured out...
+        for (const auto & arg : node.arguments->children)
+        {
+            visit(arg, data);
         }
 
         // Don't need to do anything more for window functions here -- the

src/Interpreters/ExpressionAnalyzer.cpp

@@ -970,7 +970,9 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(
     ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
 
     // 1) Add actions for window functions and their arguments;
-    // 2) Mark the columns that are really required.
+    // 2) Mark the columns that are really required. We have to mark them as
+    //    required because we finish the expression chain before processing the
+    //    window functions.
     for (const auto & [_, w] : window_descriptions)
     {
         for (const auto & f : w.window_functions)
@@ -981,41 +983,14 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(
             getRootActionsNoMakeSet(f.function_node->clone(),
                 true /* no_subqueries */, step.actions());
 
-            // 1.2) result of window function: an empty INPUT.
-            // It is an aggregate function, so it won't be added by getRootActions.
-            // This is something of a hack. Other options:
-            //  a] do it like aggregate function -- break the chain of actions
-            //     and manually add window functions to the starting list of
-            //     input columns. Logically this is similar to what we're doing
-            //     now, but would require to split the window function processing
-            //     into a full-fledged step after plain functions. This would be
-            //     somewhat cumbersome. With INPUT hack we can avoid a separate
-            //     step and pretend that window functions are almost "normal"
-            //     select functions. The limitation of both these ways is that
-            //     we can't reference window functions in other SELECT
-            //     expressions.
-            //  b] add a WINDOW action type, then sort, then split the chain on
-            //     each WINDOW action and insert the Window pipeline between the
-            //     Expression pipelines. This is a "proper" way that would allow
-            //     us to depend on window functions in other functions. But it's
-            //     complicated so I avoid doing it for now.
-            ColumnWithTypeAndName col;
-            col.type = f.aggregate_function->getReturnType();
-            col.column = col.type->createColumn();
-            col.name = f.column_name;
-
-            step.actions()->addInput(col);
-
+            // 2.1) function arguments;
             for (const auto & a : f.function_node->arguments->children)
             {
-                // 2.1) function arguments;
                 step.required_output.push_back(a->getColumnName());
             }
-            // 2.2) function result;
-            step.required_output.push_back(f.column_name);
         }
 
-        // 2.3) PARTITION BY and ORDER BY columns.
+        // 2.1) PARTITION BY and ORDER BY columns.
         for (const auto & c : w.full_sort_description)
         {
             step.required_output.push_back(c.column_name);
@@ -1048,6 +1023,15 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain,
 
     for (const auto & child : select_query->select()->children)
     {
+        if (const auto * function = typeid_cast<const ASTFunction *>(child.get());
+            function
+            && function->is_window_function)
+        {
+            // Skip window function columns here -- they are calculated after
+            // other SELECT expressions by a special step.
+            continue;
+        }
+
         step.required_output.push_back(child->getColumnName());
     }
 }
@@ -1421,11 +1405,54 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
         /// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers.
         query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage));
 
-        query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage);
+        // Window functions are processed in a separate expression chain after
+        // the main SELECT, similar to what we do for aggregate functions.
+        if (has_window)
+        {
+            query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage);
+
+            // Build a list of output columns of the window step.
+            // 1) We need the columns that are the output of ExpressionActions.
+            for (const auto & x : chain.getLastActions()->getNamesAndTypesList())
+            {
+                query_analyzer.columns_after_window.push_back(x);
+            }
+            // 2) We also have to manually add the output of the window function
+            // to the list of the output columns of the window step, because the
+            // window functions are not in the ExpressionActions.
+            for (const auto & [_, w] : query_analyzer.window_descriptions)
+            {
+                for (const auto & f : w.window_functions)
+                {
+                    query_analyzer.columns_after_window.push_back(
+                        {f.column_name, f.aggregate_function->getReturnType()});
+                }
+            }
+
+            before_window = chain.getLastActions();
+            finalize_chain(chain);
+
+            auto & step = chain.lastStep(query_analyzer.columns_after_window);
+
+            // The output of this expression chain is the result of
+            // SELECT (before "final projection" i.e. renaming the columns), so
+            // we have to mark the expressions that are required in the output,
+            // again. We did it for the previous expression chain ("select w/o
+            // window functions") earlier, in appendSelect(). But that chain also
+            // produced the expressions required to calculate window functions.
+            // They are not needed in the final SELECT result. Knowing the correct
+            // list of columns is important when we apply SELECT DISTINCT later.
+            const auto * select_query = query_analyzer.getSelectQuery();
+            for (const auto & child : select_query->select()->children)
+            {
+                step.required_output.push_back(child->getColumnName());
+            }
+        }
 
         selected_columns = chain.getLastStep().required_output;
+
         has_order_by = query.orderBy() != nullptr;
-        before_order_and_select = query_analyzer.appendOrderBy(
+        before_order_by = query_analyzer.appendOrderBy(
                 chain,
                 only_types || (need_aggregate ? !second_stage : !first_stage),
                 optimize_read_in_order,
@@ -1572,9 +1599,9 @@ std::string ExpressionAnalysisResult::dump() const
         ss << "before_window " << before_window->dumpDAG() << "\n";
     }
 
-    if (before_order_and_select)
+    if (before_order_by)
     {
-        ss << "before_order_and_select " << before_order_and_select->dumpDAG() << "\n";
+        ss << "before_order_by " << before_order_by->dumpDAG() << "\n";
     }
 
     if (before_limit_by)
@@ -1587,6 +1614,20 @@ std::string ExpressionAnalysisResult::dump() const
         ss << "final_projection " << final_projection->dumpDAG() << "\n";
     }
 
+    if (!selected_columns.empty())
+    {
+        ss << "selected_columns ";
+        for (size_t i = 0; i < selected_columns.size(); i++)
+        {
+            if (i > 0)
+            {
+                ss << ", ";
+            }
+            ss << backQuote(selected_columns[i]);
+        }
+        ss << "\n";
+    }
+
     return ss.str();
 }
 

src/Interpreters/ExpressionAnalyzer.h

@@ -55,6 +55,8 @@ struct ExpressionAnalyzerData
     NamesAndTypesList columns_after_join;
     /// Columns after ARRAY JOIN, JOIN, and/or aggregation.
     NamesAndTypesList aggregated_columns;
+    /// Columns after window functions.
+    NamesAndTypesList columns_after_window;
 
     bool has_aggregation = false;
     NamesAndTypesList aggregation_keys;
@@ -202,11 +204,12 @@ struct ExpressionAnalysisResult
     ActionsDAGPtr before_aggregation;
     ActionsDAGPtr before_having;
     ActionsDAGPtr before_window;
-    ActionsDAGPtr before_order_and_select;
+    ActionsDAGPtr before_order_by;
     ActionsDAGPtr before_limit_by;
     ActionsDAGPtr final_projection;
 
-    /// Columns from the SELECT list, before renaming them to aliases.
+    /// Columns from the SELECT list, before renaming them to aliases. Used to
+    /// perform SELECT DISTINCT.
     Names selected_columns;
 
     /// Columns will be removed after prewhere actions execution.

src/Interpreters/ExtractExpressionInfoVisitor.cpp

@@ -22,15 +22,22 @@ void ExpressionInfoMatcher::visit(const ASTFunction & ast_function, const ASTPtr
     {
         data.is_array_join = true;
     }
-    // "is_aggregate_function" doesn't mean much by itself. Apparently here it is
-    // used to move filters from HAVING to WHERE, and probably for this purpose
-    // an aggregate function calculated as a window function is not relevant.
+    // "is_aggregate_function" is used to determine whether we can move a filter
+    // (1) from HAVING to WHERE or (2) from WHERE of a parent query to HAVING of
+    // a subquery.
+    // For aggregate functions we can't do (1) but can do (2).
+    // For window functions both don't make sense -- they are not allowed in
+    // WHERE or HAVING.
     else if (!ast_function.is_window_function
         && AggregateFunctionFactory::instance().isAggregateFunctionName(
             ast_function.name))
     {
         data.is_aggregate_function = true;
     }
+    else if (ast_function.is_window_function)
+    {
+        data.is_window_function = true;
+    }
     else
     {
         const auto & function = FunctionFactory::instance().tryGet(ast_function.name, data.context);
@@ -75,15 +82,26 @@ bool ExpressionInfoMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &)
     return !node->as<ASTSubquery>();
 }
 
-bool hasStatefulFunction(const ASTPtr & node, const Context & context)
+bool hasNonRewritableFunction(const ASTPtr & node, const Context & context)
 {
     for (const auto & select_expression : node->children)
     {
         ExpressionInfoVisitor::Data expression_info{.context = context, .tables = {}};
         ExpressionInfoVisitor(expression_info).visit(select_expression);
 
-        if (expression_info.is_stateful_function)
+        if (expression_info.is_stateful_function
+            || expression_info.is_window_function)
+        {
+            // If an outer query has a WHERE on window function, we can't move
+            // it into the subquery, because window functions are not allowed in
+            // WHERE and HAVING. Example:
+            // select * from (
+            //     select number,
+            //          count(*) over (partition by intDiv(number, 3)) c
+            //     from numbers(3)
+            // ) where c > 1;
             return true;
+        }
     }
 
     return false;

src/Interpreters/ExtractExpressionInfoVisitor.h

@@ -21,6 +21,7 @@ struct ExpressionInfoMatcher
         bool is_array_join = false;
         bool is_stateful_function = false;
         bool is_aggregate_function = false;
+        bool is_window_function = false;
         bool is_deterministic_function = true;
         std::unordered_set<size_t> unique_reference_tables_pos = {};
     };
@@ -36,6 +37,6 @@ struct ExpressionInfoMatcher
 
 using ExpressionInfoVisitor = ConstInDepthNodeVisitor<ExpressionInfoMatcher, true>;
 
-bool hasStatefulFunction(const ASTPtr & node, const Context & context);
+bool hasNonRewritableFunction(const ASTPtr & node, const Context & context);
 
 }

src/Interpreters/GetAggregatesVisitor.h

@@ -33,11 +33,14 @@ public:
             return false;
         if (auto * func = node->as<ASTFunction>())
         {
-            if (isAggregateFunction(*func)
-                || func->is_window_function)
+            if (isAggregateFunction(*func))
             {
                 return false;
             }
+
+            // Window functions can contain aggregation results as arguments
+            // to the window functions, or columns of PARTITION BY or ORDER BY
+            // of the window.
         }
         return true;
     }

src/Interpreters/InterpreterSelectQuery.cpp

@@ -211,6 +211,18 @@ static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & table
     JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query);
 }
 
+/// Returns true if we should ignore quotas and limits for a specified table in the system database.
+static bool shouldIgnoreQuotaAndLimits(const StorageID & table_id)
+{
+    if (table_id.database_name == DatabaseCatalog::SYSTEM_DATABASE)
+    {
+        static const boost::container::flat_set<String> tables_ignoring_quota{"quotas", "quota_limits", "quota_usage", "quotas_usage", "one"};
+        if (tables_ignoring_quota.count(table_id.table_name))
+            return true;
+    }
+    return false;
+}
+
 InterpreterSelectQuery::InterpreterSelectQuery(
     const ASTPtr & query_ptr_,
     const Context & context_,
@@ -255,14 +267,18 @@ InterpreterSelectQuery::InterpreterSelectQuery(
 
     JoinedTables joined_tables(getSubqueryContext(*context), getSelectQuery());
 
+    bool got_storage_from_query = false;
     if (!has_input && !storage)
+    {
         storage = joined_tables.getLeftTableStorage();
+        got_storage_from_query = true;
+    }
 
     if (storage)
     {
         table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout);
         table_id = storage->getStorageID();
-        if (metadata_snapshot == nullptr)
+        if (!metadata_snapshot)
             metadata_snapshot = storage->getInMemoryMetadataPtr();
     }
 
@@ -280,9 +296,10 @@ InterpreterSelectQuery::InterpreterSelectQuery(
         if (storage && joined_tables.isLeftTableSubquery())
         {
             /// Rewritten with subquery. Free storage locks here.
-            storage = {};
+            storage = nullptr;
             table_lock.reset();
             table_id = StorageID::createEmpty();
+            metadata_snapshot = nullptr;
         }
     }
 
@@ -445,16 +462,14 @@ InterpreterSelectQuery::InterpreterSelectQuery(
     if (query.prewhere() && !query.where())
         analysis_result.prewhere_info->need_filter = true;
 
-    const StorageID & left_table_id = joined_tables.leftTableID();
-
-    if (left_table_id)
-        context->checkAccess(AccessType::SELECT, left_table_id, required_columns);
-
-    /// Remove limits for some tables in the `system` database.
-    if (left_table_id.database_name == "system")
+    if (table_id && got_storage_from_query && !joined_tables.isLeftTableFunction())
     {
-        static const boost::container::flat_set<String> system_tables_ignoring_quota{"quotas", "quota_limits", "quota_usage", "quotas_usage", "one"};
-        if (system_tables_ignoring_quota.count(left_table_id.table_name))
+        /// The current user should have the SELECT privilege.
+        /// If this table_id is for a table function we don't check access rights here because in this case they have been already checked in ITableFunction::execute().
+        context->checkAccess(AccessType::SELECT, table_id, required_columns);
+
+        /// Remove limits for some tables in the `system` database.
+        if (shouldIgnoreQuotaAndLimits(table_id) && (joined_tables.tablesCount() <= 1))
         {
             options.ignore_quota = true;
             options.ignore_limits = true;
@@ -538,7 +553,10 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
     if (options.to_stage == QueryProcessingStage::Enum::WithMergeableState)
     {
         if (!analysis_result.need_aggregate)
-            return analysis_result.before_order_and_select->getResultColumns();
+        {
+            // What's the difference with selected_columns?
+            return analysis_result.before_order_by->getResultColumns();
+        }
 
         Block header = analysis_result.before_aggregation->getResultColumns();
 
@@ -564,7 +582,8 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
 
     if (options.to_stage == QueryProcessingStage::Enum::WithMergeableStateAfterAggregation)
     {
-        return analysis_result.before_order_and_select->getResultColumns();
+        // What's the difference with selected_columns?
+        return analysis_result.before_order_by->getResultColumns();
     }
 
     return analysis_result.final_projection->getResultColumns();
@@ -958,8 +977,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
             }
             else
             {
-                executeExpression(query_plan, expressions.before_order_and_select, "Before ORDER BY and SELECT");
+                executeExpression(query_plan, expressions.before_window, "Before window functions");
                 executeWindow(query_plan);
+                executeExpression(query_plan, expressions.before_order_by, "Before ORDER BY");
                 executeDistinct(query_plan, true, expressions.selected_columns, true);
             }
 
@@ -1005,8 +1025,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
                 else if (expressions.hasHaving())
                     executeHaving(query_plan, expressions.before_having);
 
-                executeExpression(query_plan, expressions.before_order_and_select, "Before ORDER BY and SELECT");
+                executeExpression(query_plan, expressions.before_window,
+                    "Before window functions");
                 executeWindow(query_plan);
+                executeExpression(query_plan, expressions.before_order_by, "Before ORDER BY");
                 executeDistinct(query_plan, true, expressions.selected_columns, true);
 
             }
@@ -1029,10 +1051,23 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
             /** Optimization - if there are several sources and there is LIMIT, then first apply the preliminary LIMIT,
               * limiting the number of rows in each up to `offset + limit`.
               */
+            bool has_withfill = false;
+            if (query.orderBy())
+            {
+                SortDescription order_descr = getSortDescription(query, *context);
+                for (auto & desc : order_descr)
+                    if (desc.with_fill)
+                    {
+                        has_withfill = true;
+                        break;
+                    }
+            }
+
             bool has_prelimit = false;
             if (!to_aggregation_stage &&
                 query.limitLength() && !query.limit_with_ties && !hasWithTotalsInAnySubqueryInFromClause(query) &&
-                !query.arrayJoinExpressionList() && !query.distinct && !expressions.hasLimitBy() && !settings.extremes)
+                !query.arrayJoinExpressionList() && !query.distinct && !expressions.hasLimitBy() && !settings.extremes &&
+                !has_withfill)
             {
                 executePreLimit(query_plan, false);
                 has_prelimit = true;
@@ -1745,6 +1780,11 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific
 
 void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description)
 {
+    if (!expression)
+    {
+        return;
+    }
+
     auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), expression);
 
     expression_step->setStepDescription(description);

src/Interpreters/JoinedTables.cpp

@@ -161,6 +161,7 @@ StoragePtr JoinedTables::getLeftTableStorage()
     if (isLeftTableFunction())
         return context.getQueryContext().executeTableFunction(left_table_expression);
 
+    StorageID table_id = StorageID::createEmpty();
     if (left_db_and_table)
     {
         table_id = context.resolveStorageID(StorageID(left_db_and_table->database, left_db_and_table->table, left_db_and_table->uuid));

src/Interpreters/JoinedTables.h

@@ -43,8 +43,6 @@ public:
     bool isLeftTableFunction() const;
     size_t tablesCount() const { return table_expressions.size(); }
 
-    const StorageID & leftTableID() const { return table_id; }
-
     void rewriteDistributedInAndJoins(ASTPtr & query);
 
     std::unique_ptr<InterpreterSelectWithUnionQuery> makeLeftTableSubquery(const SelectQueryOptions & select_options);
@@ -57,9 +55,6 @@ private:
     /// Legacy (duplicated left table values)
     ASTPtr left_table_expression;
     std::optional<DatabaseAndTableWithAlias> left_db_and_table;
-
-    /// left_db_and_table or 'system.one'
-    StorageID table_id = StorageID::createEmpty();
 };
 
 }

src/Interpreters/PredicateExpressionsOptimizer.cpp

@@ -90,8 +90,12 @@ std::vector<ASTs> PredicateExpressionsOptimizer::extractTablesPredicates(const A
         ExpressionInfoVisitor::Data expression_info{.context = context, .tables = tables_with_columns};
         ExpressionInfoVisitor(expression_info).visit(predicate_expression);
 
-        if (expression_info.is_stateful_function || !expression_info.is_deterministic_function)
-            return {};   /// Not optimized when predicate contains stateful function or indeterministic function
+        if (expression_info.is_stateful_function
+            || !expression_info.is_deterministic_function
+            || expression_info.is_window_function)
+        {
+            return {};   /// Not optimized when predicate contains stateful function or indeterministic function or window functions
+        }
 
         if (!expression_info.is_array_join)
         {
@@ -190,6 +194,12 @@ bool PredicateExpressionsOptimizer::tryMovePredicatesFromHavingToWhere(ASTSelect
         if (expression_info.is_stateful_function)
             return false;
 
+        if (expression_info.is_window_function)
+        {
+            // Window functions are not allowed in either HAVING or WHERE.
+            return false;
+        }
+
         if (expression_info.is_aggregate_function)
             having_predicates.emplace_back(moving_predicate);
         else

src/Interpreters/PredicateRewriteVisitor.cpp

@@ -88,7 +88,7 @@ bool PredicateRewriteVisitorData::rewriteSubquery(ASTSelectQuery & subquery, con
         || (!optimize_with && subquery.with())
         || subquery.withFill()
         || subquery.limitBy() || subquery.limitLength()
-        || hasStatefulFunction(subquery.select(), context))
+        || hasNonRewritableFunction(subquery.select(), context))
         return false;
 
     for (const auto & predicate : predicates)

src/Interpreters/QueryNormalizer.cpp

@@ -148,9 +148,9 @@ void QueryNormalizer::visit(ASTSelectQuery & select, const ASTPtr &, Data & data
 /// Don't go into select query. It processes children itself.
 /// Do not go to the left argument of lambda expressions, so as not to replace the formal parameters
 ///  on aliases in expressions of the form 123 AS x, arrayMap(x -> 1, [2]).
-void QueryNormalizer::visitChildren(const ASTPtr & node, Data & data)
+void QueryNormalizer::visitChildren(IAST * node, Data & data)
 {
-    if (const auto * func_node = node->as<ASTFunction>())
+    if (auto * func_node = node->as<ASTFunction>())
     {
         if (func_node->tryGetQueryArgument())
         {
@@ -176,6 +176,16 @@ void QueryNormalizer::visitChildren(const ASTPtr & node, Data & data)
                     visit(child, data);
             }
         }
+
+        if (func_node->window_partition_by)
+        {
+            visitChildren(func_node->window_partition_by.get(), data);
+        }
+
+        if (func_node->window_order_by)
+        {
+            visitChildren(func_node->window_order_by.get(), data);
+        }
     }
     else if (!node->as<ASTSelectQuery>())
     {
@@ -221,7 +231,7 @@ void QueryNormalizer::visit(ASTPtr & ast, Data & data)
     if (ast.get() != initial_ast.get())
         visit(ast, data);
     else
-        visitChildren(ast, data);
+        visitChildren(ast.get(), data);
 
     current_asts.erase(initial_ast.get());
     current_asts.erase(ast.get());

src/Interpreters/QueryNormalizer.h

@@ -69,7 +69,7 @@ private:
     static void visit(ASTTablesInSelectQueryElement &, const ASTPtr &, Data &);
     static void visit(ASTSelectQuery &, const ASTPtr &, Data &);
 
-    static void visitChildren(const ASTPtr &, Data & data);
+    static void visitChildren(IAST * node, Data & data);
 };
 
 }

src/Interpreters/TreeRewriter.cpp

@@ -29,6 +29,7 @@
 #include <DataTypes/DataTypeNullable.h>
 
 #include <IO/WriteHelpers.h>
+#include <IO/WriteBufferFromOStream.h>
 #include <Storages/IStorage.h>
 
 #include <AggregateFunctions/AggregateFunctionFactory.h>
@@ -445,6 +446,8 @@ std::vector<const ASTFunction *> getAggregates(ASTPtr & query, const ASTSelectQu
             for (auto & arg : node->arguments->children)
             {
                 assertNoAggregates(arg, "inside another aggregate function");
+                // We also can't have window functions inside aggregate functions,
+                // because the window functions are calculated later.
                 assertNoWindows(arg, "inside an aggregate function");
             }
         }
@@ -454,7 +457,9 @@ std::vector<const ASTFunction *> getAggregates(ASTPtr & query, const ASTSelectQu
 
 std::vector<const ASTFunction *> getWindowFunctions(ASTPtr & query, const ASTSelectQuery & select_query)
 {
-    /// There can not be window functions inside the WHERE and PREWHERE.
+    /// There can not be window functions inside the WHERE, PREWHERE and HAVING
+    if (select_query.having())
+        assertNoWindows(select_query.having(), "in HAVING");
     if (select_query.where())
         assertNoWindows(select_query.where(), "in WHERE");
     if (select_query.prewhere())
@@ -463,17 +468,34 @@ std::vector<const ASTFunction *> getWindowFunctions(ASTPtr & query, const ASTSel
     GetAggregatesVisitor::Data data;
     GetAggregatesVisitor(data).visit(query);
 
-    /// There can not be other window functions within the aggregate functions.
+    /// Window functions cannot be inside aggregates or other window functions.
+    /// Aggregate functions can be inside window functions because they are
+    /// calculated earlier.
     for (const ASTFunction * node : data.window_functions)
     {
         if (node->arguments)
         {
             for (auto & arg : node->arguments->children)
             {
-                assertNoAggregates(arg, "inside a window function");
                 assertNoWindows(arg, "inside another window function");
             }
         }
+
+        if (node->window_partition_by)
+        {
+            for (auto & arg : node->window_partition_by->children)
+            {
+                assertNoWindows(arg, "inside PARTITION BY of a window");
+            }
+        }
+
+        if (node->window_order_by)
+        {
+            for (auto & arg : node->window_order_by->children)
+            {
+                assertNoWindows(arg, "inside ORDER BY of a window");
+            }
+        }
     }
 
     return data.window_functions;

src/Parsers/ASTFunction.cpp

@@ -39,6 +39,16 @@ void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const
             (*it)->appendColumnName(ostr);
         }
     writeChar(')', ostr);
+
+    if (is_window_function)
+    {
+        writeCString(" OVER (", ostr);
+        FormatSettings settings{ostr, true /* one_line */};
+        FormatState state;
+        FormatStateStacked frame;
+        appendWindowDescription(settings, state, frame);
+        writeCString(")", ostr);
+    }
 }
 
 /** Get the text that identifies this element. */
@@ -57,17 +67,20 @@ ASTPtr ASTFunction::clone() const
 
     if (window_name)
     {
-        res->set(res->window_name, window_name->clone());
+        res->window_name = window_name->clone();
+        res->children.push_back(res->window_name);
     }
 
     if (window_partition_by)
     {
-        res->set(res->window_partition_by, window_partition_by->clone());
+        res->window_partition_by = window_partition_by->clone();
+        res->children.push_back(res->window_partition_by);
     }
 
     if (window_order_by)
     {
-        res->set(res->window_order_by, window_order_by->clone());
+        res->window_order_by = window_order_by->clone();
+        res->children.push_back(res->window_order_by);
     }
 
     return res;

src/Parsers/ASTFunction.h

@@ -21,9 +21,25 @@ public:
     ASTPtr parameters;
 
     bool is_window_function = false;
-    ASTIdentifier * window_name;
-    ASTExpressionList * window_partition_by;
-    ASTExpressionList * window_order_by;
+
+    // We have to make these fields ASTPtr because this is what the visitors
+    // expect. Some of them take const ASTPtr & (makes no sense), and some
+    // take ASTPtr & and modify it. I don't understand how the latter is
+    // compatible with also having an owning `children` array -- apparently it
+    // leads to some dangling children that are not referenced by the fields of
+    // the AST class itself. Some older code hints at the idea of having
+    // ownership in `children` only, and making the class fields to be raw
+    // pointers of proper type (see e.g. IAST::set), but this is not compatible
+    // with the visitor interface.
+
+    // ASTIdentifier
+    ASTPtr window_name;
+
+    // ASTExpressionList
+    ASTPtr window_partition_by;
+
+    // ASTExpressionList of
+    ASTPtr window_order_by;
 
     /// do not print empty parentheses if there are no args - compatibility with new AST for data types and engine names.
     bool no_empty_args = false;

src/Parsers/ExpressionElementParsers.cpp

@@ -419,7 +419,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         ParserIdentifier window_name_parser;
         if (window_name_parser.parse(pos, window_name_ast, expected))
         {
-            function->set(function->window_name, window_name_ast);
+            function->children.push_back(window_name_ast);
+            function->window_name = window_name_ast;
             return true;
         }
         else
@@ -442,7 +443,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         ASTPtr partition_by_ast;
         if (columns_partition_by.parse(pos, partition_by_ast, expected))
         {
-            function->set(function->window_partition_by, partition_by_ast);
+            function->children.push_back(partition_by_ast);
+            function->window_partition_by = partition_by_ast;
         }
         else
         {
@@ -455,7 +457,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         ASTPtr order_by_ast;
         if (columns_order_by.parse(pos, order_by_ast, expected))
         {
-            function->set(function->window_order_by, order_by_ast);
+            function->children.push_back(order_by_ast);
+            function->window_order_by = order_by_ast;
         }
         else
         {

src/Processors/LimitTransform.cpp

@@ -237,7 +237,7 @@ LimitTransform::Status LimitTransform::preparePair(PortsData & data)
             previous_row_chunk = makeChunkWithPreviousRow(data.current_chunk, data.current_chunk.getNumRows() - 1);
     }
     else
-        /// This function may be heavy to execute in prepare. But it happens no more then twice, and make code simpler.
+        /// This function may be heavy to execute in prepare. But it happens no more than twice, and make code simpler.
         splitChunk(data);
 
     bool may_need_more_data_for_ties = previous_row_chunk || rows_read - rows <= offset + limit;

src/Processors/Merges/Algorithms/MergedData.h

@@ -82,7 +82,7 @@ public:
         if (need_flush)
             return true;
 
-        /// Never return more then max_block_size.
+        /// Never return more than max_block_size.
         if (merged_rows >= max_block_size)
             return true;
 

src/Processors/Pipe.h

@@ -71,8 +71,8 @@ public:
     enum class StreamType
     {
         Main = 0, /// Stream for query data. There may be several streams of this type.
-        Totals,  /// Stream for totals. No more then one.
-        Extremes, /// Stream for extremes. No more then one.
+        Totals,  /// Stream for totals. No more than one.
+        Extremes, /// Stream for extremes. No more than one.
     };
 
     using ProcessorGetter = std::function<ProcessorPtr(const Block & header)>;

src/Processors/QueryPlan/IQueryPlanStep.cpp

@@ -46,6 +46,8 @@ static void doDescribeHeader(const Block & header, size_t count, IQueryPlanStep:
 
         first = false;
         elem.dumpNameAndType(settings.out);
+        settings.out << ": ";
+        elem.dumpStructure(settings.out);
         settings.out << '\n';
     }
 }

src/Processors/QueryPlan/QueryPlan.cpp

@@ -247,6 +247,15 @@ static void explainStep(
         step.describeActions(settings);
 }
 
+std::string debugExplainStep(const IQueryPlanStep & step)
+{
+    WriteBufferFromOwnString out;
+    IQueryPlanStep::FormatSettings settings{.out = out};
+    QueryPlan::ExplainPlanOptions options{.actions = true};
+    explainStep(step, settings, options);
+    return out.str();
+}
+
 void QueryPlan::explainPlan(WriteBuffer & buffer, const ExplainPlanOptions & options)
 {
     checkInitialized();
@@ -488,6 +497,7 @@ static bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node *
 {
     auto & parent = parent_node->step;
     auto & child = child_node->step;
+
     /// TODO: FilterStep
     auto * parent_expr = typeid_cast<ExpressionStep *>(parent.get());
     auto * child_expr = typeid_cast<ExpressionStep *>(child.get());

src/Processors/QueryPlan/QueryPlan.h

@@ -97,4 +97,6 @@ private:
     std::vector<std::shared_ptr<Context>> interpreter_context;
 };
 
+std::string debugExplainStep(const IQueryPlanStep & step);
+
 }

src/Processors/Transforms/WindowTransform.cpp

@@ -77,6 +77,11 @@ void WindowTransform::transform(Chunk & chunk)
         ws.argument_columns.clear();
         for (const auto column_index : ws.argument_column_indices)
         {
+            // Aggregate functions can't work with constant columns, so we have to
+            // materialize them like the Aggregator does.
+            columns[column_index]
+                = std::move(columns[column_index])->convertToFullColumnIfConst();
+
             ws.argument_columns.push_back(columns[column_index].get());
         }
 

src/Storages/MergeTree/MergeTreeIOSettings.h

@@ -46,7 +46,7 @@ struct MergeTreeWriterSettings
     bool rewrite_primary_key;
     bool blocks_are_granules_size;
 
-    /// Used for AIO threshold comparsion
+    /// Used for AIO threshold comparison
     /// FIXME currently doesn't work because WriteBufferAIO contain obscure bug(s)
     size_t estimated_size = 0;
 };

src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.cpp

@@ -26,7 +26,7 @@ void ReplicatedMergeTreeAltersSequence::addMetadataAlter(
     int alter_version, std::lock_guard<std::mutex> & /*state_lock*/)
 {
     /// Data alter (mutation) always added before. See ReplicatedMergeTreeQueue::pullLogsToQueue.
-    /// So mutation alredy added to this sequence or doesn't exist.
+    /// So mutation already added to this sequence or doesn't exist.
     if (!queue_state.count(alter_version))
         queue_state.emplace(alter_version, AlterState{.metadata_finished=false, .data_finished=true});
     else

src/Storages/StorageReplicatedMergeTree.cpp

@@ -6155,7 +6155,7 @@ bool StorageReplicatedMergeTree::dropPart(
         /// DROP_RANGE with detach will move this part together with source parts to `detached/` dir.
         entry.type = LogEntry::DROP_RANGE;
         entry.source_replica = replica_name;
-        entry.new_part_name = drop_part_info.getPartName();
+        entry.new_part_name = getPartNamePossiblyFake(format_version, drop_part_info);
         entry.detach = detach;
         entry.create_time = time(nullptr);
 

tests/integration/test_materialize_mysql_database/materialize_with_ddl.py

@@ -48,16 +48,15 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam
                      "/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */"
                      "unsigned_float FLOAT UNSIGNED, _float FLOAT, "
                      "unsigned_double DOUBLE UNSIGNED, _double DOUBLE, "
-                     "_varchar VARCHAR(10), _char CHAR(10), "
+                     "_varchar VARCHAR(10), _char CHAR(10), binary_col BINARY(8), "
                      "/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */"
                      "_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;")
 
     # it already has some data
     mysql_node.query("""
-        INSERT INTO test_database.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 
+        INSERT INTO test_database.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary',
         '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', true);
         """)
-
     clickhouse_node.query(
         "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(
             service_name))
@@ -65,51 +64,51 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam
     assert "test_database" in clickhouse_node.query("SHOW DATABASES")
 
     check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV",
-                "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t"
+                "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t"
                 "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n")
 
     mysql_node.query("""
-        INSERT INTO test_database.test_table_1 VALUES(2, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 
+        INSERT INTO test_database.test_table_1 VALUES(2, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary',
         '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', false);
         """)
 
     check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV",
-                "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t"
+                "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t"
                 "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t"
-                "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n")
+                "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n")
 
     mysql_node.query("UPDATE test_database.test_table_1 SET unsigned_tiny_int = 2 WHERE `key` = 1")
 
     check_query(clickhouse_node, """
         SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int,
          small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, 
-         unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, 
+         unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col,
          _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ 
          _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV
         """,
-        "1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t"
+        "1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t"
         "2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t"
-        "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n")
+        "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n")
 
     # update primary key
     mysql_node.query("UPDATE test_database.test_table_1 SET `key` = 3 WHERE `unsigned_tiny_int` = 2")
 
     check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int,"
                                  " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, "
-                                 " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, "
+                                 " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, "
                                  " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ "
                                  " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV",
                 "2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t"
-                "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t"
-                "4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t1\n")
+                "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t"
+                "4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t1\n")
 
     mysql_node.query('DELETE FROM test_database.test_table_1 WHERE `key` = 2')
     check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int,"
                                  " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, "
-                                 " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, "
+                                 " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, "
                                  " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ "
                                  " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV",
-                "3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t"
+                "3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t"
                 "2020-01-01 00:00:00\t1\n")
 
     mysql_node.query('DELETE FROM test_database.test_table_1 WHERE `unsigned_tiny_int` = 2')

tests/integration/test_merge_tree_empty_parts/configs/cleanup_thread.xml

@@ -0,0 +1,6 @@
+<yandex>
+    <merge_tree>
+        <cleanup_delay_period>0</cleanup_delay_period>
+        <cleanup_delay_period_random_add>0</cleanup_delay_period_random_add>
+    </merge_tree>
+</yandex>

tests/integration/test_merge_tree_empty_parts/configs/remote_servers.xml

@@ -0,0 +1,13 @@
+<yandex>
+    <remote_servers>
+        <test_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>node1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster>
+    </remote_servers>
+</yandex>

tests/integration/test_merge_tree_empty_parts/test.py

@@ -0,0 +1,38 @@
+import pytest
+import helpers.client
+import helpers.cluster
+from helpers.test_tools import assert_eq_with_retry
+
+
+cluster = helpers.cluster.ClickHouseCluster(__file__)
+node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/cleanup_thread.xml'], with_zookeeper=True)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+def test_empty_parts_alter_delete(started_cluster):
+    node1.query("CREATE TABLE empty_parts_delete (d Date, key UInt64, value String) \
+        ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1', d, key, 8192)")
+
+    node1.query("INSERT INTO empty_parts_delete VALUES (toDate('2020-10-10'), 1, 'a')")
+    node1.query("ALTER TABLE empty_parts_delete DELETE WHERE 1 SETTINGS mutations_sync = 2")
+
+    print(node1.query("SELECT count() FROM empty_parts_delete"))
+    assert_eq_with_retry(node1, "SELECT count() FROM system.parts WHERE table = 'empty_parts_delete' AND active", "0")
+
+def test_empty_parts_summing(started_cluster):
+    node1.query("CREATE TABLE empty_parts_summing (d Date, key UInt64, value Int64) \
+        ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1', d, key, 8192)")
+
+    node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, 1)")
+    node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, -1)")
+    node1.query("OPTIMIZE TABLE empty_parts_summing FINAL")
+
+    assert_eq_with_retry(node1, "SELECT count() FROM system.parts WHERE table = 'empty_parts_summing' AND active", "0")

tests/integration/test_select_access_rights/test.py

@@ -0,0 +1,157 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance('instance')
+
+
+@pytest.fixture(scope="module", autouse=True)
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+@pytest.fixture(autouse=True)
+def cleanup_after_test():
+    instance.query("CREATE USER OR REPLACE A")
+    yield
+    instance.query("DROP TABLE IF EXISTS table1")
+    instance.query("DROP TABLE IF EXISTS table2")
+
+
+def test_select_single_column():
+    instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d")
+
+    select_query = "SELECT a FROM table1"
+    assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(a) ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+    instance.query("REVOKE SELECT(a) ON default.table1 FROM A")
+    assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+
+def test_select_single_column_with_table_grant():
+    instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d")
+
+    select_query = "SELECT a FROM table1"
+    assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+    instance.query("REVOKE SELECT(a) ON default.table1 FROM A")
+    assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+
+def test_select_all_columns():
+    instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d")
+
+    select_query = "SELECT * FROM table1"
+    assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(d) ON default.table1 TO A")
+    assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(a) ON default.table1 TO A")
+    assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(b) ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+
+def test_select_all_columns_with_table_grant():
+    instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d")
+
+    select_query = "SELECT * FROM table1"
+    assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+
+def test_alias():
+    instance.query("CREATE TABLE table1(x Int32, y Int32) ENGINE = MergeTree ORDER BY tuple()")
+
+    select_query = "SELECT x, y, x + y AS s FROM table1"
+    assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(x, y) ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+
+def test_alias_columns():
+    instance.query("CREATE TABLE table1(x Int32, y Int32, s Int32 ALIAS x + y) ENGINE = MergeTree ORDER BY tuple()")
+
+    select_query = "SELECT * FROM table1"
+    assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(x,y) ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+    select_query = "SELECT s FROM table1"
+    assert "it's necessary to have grant SELECT(s) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+    
+    instance.query("GRANT SELECT(s) ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+    instance.query("REVOKE SELECT(x,y) ON default.table1 FROM A")
+    assert instance.query(select_query, user = 'A') == ""
+
+
+def test_materialized_columns():
+    instance.query("CREATE TABLE table1(x Int32, y Int32, p Int32 MATERIALIZED x * y) ENGINE = MergeTree ORDER BY tuple()")
+
+    select_query = "SELECT * FROM table1"
+    assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(x,y) ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+    select_query = "SELECT p FROM table1"
+    assert "it's necessary to have grant SELECT(p) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+    
+    instance.query("GRANT SELECT(p) ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+    instance.query("REVOKE SELECT(x,y) ON default.table1 FROM A")
+    assert instance.query(select_query, user = 'A') == ""
+
+
+def test_select_join():
+    instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d")
+    instance.query("CREATE TABLE table2(d DATE, x UInt32, y UInt8) ENGINE = MergeTree ORDER BY d")
+
+    select_query = "SELECT * FROM table1 JOIN table2 USING(d)"
+    assert "it's necessary to have grant SELECT(d, x, y) ON default.table2" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(d, x, y) ON default.table2 TO A")
+    assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(d, a, b) ON default.table1 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+    instance.query("REVOKE SELECT ON default.table2 FROM A")
+    assert "it's necessary to have grant SELECT(d, x, y) ON default.table2" in instance.query_and_get_error(select_query, user = 'A')
+
+
+def test_select_union():
+    instance.query("CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY tuple()")
+    instance.query("CREATE TABLE table2(a String, b UInt8) ENGINE = MergeTree ORDER BY tuple()")
+
+    select_query = "SELECT * FROM table1 UNION ALL SELECT * FROM table2"
+    assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(a, b) ON default.table1 TO A")
+    assert "it's necessary to have grant SELECT(a, b) ON default.table2" in instance.query_and_get_error(select_query, user = 'A')
+
+    instance.query("GRANT SELECT(a, b) ON default.table2 TO A")
+    assert instance.query(select_query, user = 'A') == ""
+
+    instance.query("REVOKE SELECT ON default.table1 FROM A")
+    assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A')

tests/integration/test_storage_mysql/test.py

@@ -148,6 +148,13 @@ def test_table_function(started_cluster):
     assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '60000'
     conn.close()
 
+def test_binary_type(started_cluster):
+    conn = get_mysql_conn()
+    with conn.cursor() as cursor:
+        cursor.execute("CREATE TABLE clickhouse.binary_type (id INT PRIMARY KEY, data BINARY(16) NOT NULL)")
+    table_function = "mysql('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('binary_type')
+    node1.query("INSERT INTO {} VALUES (42, 'clickhouse')".format('TABLE FUNCTION ' + table_function))
+    assert node1.query("SELECT * FROM {}".format(table_function)) == '42\tclickhouse\\0\\0\\0\\0\\0\\0\n'
 
 def test_enum_type(started_cluster):
     table_name = 'test_enum_type'

tests/performance/window_functions.xml

@@ -0,0 +1,38 @@
+<test>
+    <preconditions>
+        <table_exists>hits_100m_single</table_exists>
+    </preconditions>
+
+    <settings>
+        <allow_experimental_window_functions>1</allow_experimental_window_functions>
+    </settings>
+
+    <!--
+        For some counters, find top 10 users by the numer of records.
+        First with LIMIT BY, next with window functions.
+    -->
+    <query><![CDATA[
+        select CounterID, UserID, count(*) user_hits
+        from hits_100m_single
+        where CounterID < 10000
+        group by CounterID, UserID
+        order by user_hits desc
+        limit 10 by CounterID
+        format Null
+    ]]></query>
+
+    <query><![CDATA[
+        select *
+        from (
+            select CounterID, UserID, count(*) user_hits,
+                count() over (partition by CounterID order by user_hits desc)
+                    user_rank
+            from hits_100m_single
+            where CounterID < 10000
+            group by CounterID, UserID
+        )
+        where user_rank <= 10
+        format Null
+    ]]></query>
+
+</test>

tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh