Merge branch 'master' into rocksdb_metacache

.clang-format

@@ -12,6 +12,7 @@ BraceWrapping:
     AfterUnion: true
     BeforeCatch: true
     BeforeElse: true
+    BeforeLambdaBody: true
     IndentBraces: false
 BreakConstructorInitializersBeforeComma: false
 Cpp11BracedListStyle: true

.github/workflows/nightly.yml

@@ -0,0 +1,73 @@
+name: NightlyBuilds
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+"on":
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  DockerHubPushAarch64:
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix aarch64 --all
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
+  DockerHubPushAmd64:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix amd64 --all
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
+  DockerHubPush:
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download changed aarch64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}
+      - name: Download changed amd64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/changed_images.json

contrib/rocksdb-cmake/CMakeLists.txt

@@ -127,11 +127,6 @@ endif()
 
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_definitions(-DOS_MACOSX)
-  if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
-    add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
-    # no debug info for IOS, that will make our library big
-    add_definitions(-DNDEBUG)
-  endif()
 elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
   add_definitions(-DOS_LINUX)
 elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")

docker/images.json

@@ -32,6 +32,7 @@
         "dependent": []
     },
     "docker/test/pvs": {
+        "only_amd64": true,
         "name": "clickhouse/pvs-test",
         "dependent": []
     },
@@ -72,6 +73,7 @@
         "dependent": []
     },
     "docker/test/integration/runner": {
+        "only_amd64": true,
         "name": "clickhouse/integration-tests-runner",
         "dependent": []
     },
@@ -124,6 +126,7 @@
         "dependent": []
     },
     "docker/test/integration/kerberos_kdc": {
+        "only_amd64": true,
         "name": "clickhouse/kerberos-kdc",
         "dependent": []
     },
@@ -137,6 +140,7 @@
          ]
     },
     "docker/test/integration/kerberized_hadoop": {
+        "only_amd64": true,
         "name": "clickhouse/kerberized-hadoop",
         "dependent": []
     },

docker/test/integration/kerberized_hadoop/Dockerfile

@@ -20,4 +20,4 @@ RUN cd /tmp && \
 	cd commons-daemon-1.0.15-src/src/native/unix && \
 	./configure && \
 	make && \
-	cp ./jsvc /usr/local/hadoop/sbin
+	cp ./jsvc /usr/local/hadoop-2.7.0/sbin

docker/test/integration/runner/Dockerfile

@@ -58,9 +58,7 @@ RUN apt-get update \
 
 RUN dockerd --version; docker --version
 
-ARG TARGETARCH
+RUN python3 -m pip install \
-# FIXME: psycopg2-binary is not available for aarch64, we skip it for now
-RUN test x$TARGETARCH = xarm64 || ( python3 -m pip install \
     PyMySQL \
     aerospike==4.0.0 \
     avro==1.10.2 \
@@ -90,7 +88,7 @@ RUN test x$TARGETARCH = xarm64 || ( python3 -m pip install \
     urllib3 \
     requests-kerberos \
     pyhdfs \
-    azure-storage-blob )
+    azure-storage-blob
 
 COPY modprobe.sh /usr/local/bin/modprobe
 COPY dockerd-entrypoint.sh /usr/local/bin/

docker/test/performance-comparison/Dockerfile

@@ -1,5 +1,5 @@
 # docker build -t clickhouse/performance-comparison .
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"

docker/test/pvs/Dockerfile

@@ -4,11 +4,7 @@
 ARG FROM_TAG=latest
 FROM clickhouse/binary-builder:$FROM_TAG
 
-# PVS studio doesn't support aarch64/arm64, so there is a check for it everywhere
+RUN apt-get update --yes \
-# We'll produce an empty image for arm64
-ARG TARGETARCH
-
-RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \
     && apt-get install \
         bash \
         wget \
@@ -21,7 +17,7 @@ RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \
         libprotoc-dev \
         libgrpc++-dev \
         libc-ares-dev \
-        --yes --no-install-recommends )
+        --yes --no-install-recommends
 
 #RUN wget -nv -O - http://files.viva64.com/etc/pubkey.txt | sudo apt-key add -
 #RUN sudo wget -nv -O /etc/apt/sources.list.d/viva64.list http://files.viva64.com/etc/viva64.list
@@ -33,7 +29,7 @@ RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \
 
 ENV PKG_VERSION="pvs-studio-latest"
 
-RUN test x$TARGETARCH = xarm64 || ( set -x \
+RUN set -x \
     && export PUBKEY_HASHSUM="ad369a2e9d8b8c30f5a9f2eb131121739b79c78e03fef0f016ea51871a5f78cd4e6257b270dca0ac3be3d1f19d885516" \
     && wget -nv https://files.viva64.com/etc/pubkey.txt -O /tmp/pubkey.txt \
     && echo "${PUBKEY_HASHSUM} /tmp/pubkey.txt" | sha384sum -c \
@@ -41,7 +37,7 @@ RUN test x$TARGETARCH = xarm64 || ( set -x \
     && wget -nv "https://files.viva64.com/${PKG_VERSION}.deb" \
     && { debsig-verify ${PKG_VERSION}.deb \
     || echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \
-    && dpkg -i "${PKG_VERSION}.deb" )
+    && dpkg -i "${PKG_VERSION}.deb"
 
 ENV CCACHE_DIR=/test_output/ccache
 

docker/test/testflows/runner/Dockerfile

@@ -43,24 +43,27 @@ RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0
 ENV DOCKER_CHANNEL stable
 ENV DOCKER_VERSION 20.10.6
 
-RUN set -eux; \
+# Architecture of the image when BuildKit/buildx is used
-  \
+ARG TARGETARCH
-# this "case" statement is generated via "update.sh"
+
-  \
+# Install docker
-  if ! wget -nv -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \
+RUN arch=${TARGETARCH:-amd64} \
-    echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \
+  && case $arch in \
-    exit 1; \
+      amd64) rarch=x86_64 ;; \
-  fi; \
+      arm64) rarch=aarch64 ;; \
-  \
+    esac \
-  tar --extract \
+  && set -eux \
+  && if ! wget -nv -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${rarch}/docker-${DOCKER_VERSION}.tgz"; then \
+    echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${rarch}'" \
+    && exit 1; \
+  fi \
+  && tar --extract \
     --file docker.tgz \
     --strip-components 1 \
     --directory /usr/local/bin/ \
-  ; \
+  && rm docker.tgz \
-  rm docker.tgz; \
+  && dockerd --version \
-  \
+  && docker --version
-  dockerd --version; \
-  docker --version
 
 COPY modprobe.sh /usr/local/bin/modprobe
 COPY dockerd-entrypoint.sh /usr/local/bin/

docs/en/engines/table-engines/mergetree-family/mergetree.md

@@ -886,3 +886,12 @@ S3 disk can be configured as `main` or `cold` storage:
 ```
 
 In case of `cold` option a data can be moved to S3 if local disk free size will be smaller than `move_factor * disk_size` or by TTL move rule.
+
+## Virtual Columns {#virtual-columns}
+
+-   `_part` — Name of a part.
+-   `_part_index` — Sequential index of the part in the query result.
+-   `_partition_id` — Name of a partition.
+-   `_part_uuid` — Unique part identifier (if enabled MergeTree setting `assign_part_uuids`).
+-   `_partition_value` — Values (a tuple) of a `partition by` expression.
+-   `_sample_factor` — Sample factor (from the query).

docs/en/operations/settings/settings.md

@@ -2304,7 +2304,7 @@ Possible values:
 -   1 — Enabled.
 -   0 — Disabled.
 
-Default value: `0`.
+Default value: `1`.
 
 ## output_format_parallel_formatting {#output-format-parallel-formatting}
 
@@ -2315,7 +2315,7 @@ Possible values:
 -   1 — Enabled.
 -   0 — Disabled.
 
-Default value: `0`.
+Default value: `1`.
 
 ## min_chunk_bytes_for_parallel_parsing {#min-chunk-bytes-for-parallel-parsing}
 

docs/en/sql-reference/functions/functions-for-nulls.md

@@ -120,7 +120,7 @@ The `mail` and `phone` fields are of type String, but the `icq` field is `UInt32
 Get the first available contact method for the customer from the contact list:
 
 ``` sql
-SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook;
+SELECT name, coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook;
 ```
 
 ``` text

docs/en/sql-reference/functions/tuple-functions.md

@@ -22,7 +22,7 @@ tuple(x, y, …)
 ## tupleElement {#tupleelement}
 
 A function that allows getting a column from a tuple.
-‘N’ is the column index, starting from 1. N must be a constant. ‘N’ must be a constant. ‘N’ must be a strict postive integer no greater than the size of the tuple.
+‘N’ is the column index, starting from 1. ‘N’ must be a constant. ‘N’ must be a strict postive integer no greater than the size of the tuple.
 There is no cost to execute the function.
 
 The function implements the operator `x.N`.

docs/en/sql-reference/operators/in.md

@@ -216,6 +216,17 @@ This is more optimal than using the normal IN. However, keep the following point
 
 It also makes sense to specify a local table in the `GLOBAL IN` clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers.
 
+### Distributed Subqueries and max_rows_in_set
+
+You can use [`max_rows_in_set`](../../operations/settings/query-complexity.md#max-rows-in-set) and [`max_bytes_in_set`](../../operations/settings/query-complexity.md#max-rows-in-set) to control how much data is tranferred during distributed queries. 
+
+This is specially important if the  `global in` query returns a large amount of data. Consider the following sql - 
+```sql
+select * from table1 where col1 global in (select col1 from table2 where <some_predicate>)
+```
+ 
+If `some_predicate` is not selective enough, it will return large amount of data and cause performance issues. In such cases, it is wise to limit the data transfer over the network. Also, note that [`set_overflow_mode`](../../operations/settings/query-complexity.md#set_overflow_mode) is set to `throw` (by default) meaning that an exception is raised when these thresholds are met.
+
 ### Distributed Subqueries and max_parallel_replicas {#max_parallel_replica-subqueries}
 
 When max_parallel_replicas is greater than 1, distributed queries are further transformed. For example, the following:

docs/en/whats-new/roadmap.md

@@ -5,6 +5,6 @@ toc_title: Roadmap
 
 # Roadmap {#roadmap}
 
-The roadmap for the year 2021 is published for open discussion [here](https://github.com/ClickHouse/ClickHouse/issues/17623).
+The roadmap for the year 2022 is published for open discussion [here](https://github.com/ClickHouse/ClickHouse/issues/32513).
 
 {## [Original article](https://clickhouse.com/docs/en/roadmap/) ##}

docs/ru/engines/table-engines/mergetree-family/mergetree.md

@@ -872,3 +872,13 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd'
 ```
 
 Если диск сконфигурирован как `cold`, данные будут переноситься в S3 при срабатывании правил TTL или когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`.
+
+## Виртуальные столбцы {#virtual-columns}
+
+-   `_part` — Имя куска.
+-   `_part_index` — Номер куска по порядку в результате запроса.
+-   `_partition_id` — Имя партиции.
+-   `_part_uuid` — Уникальный идентификатор куска (если включена MergeTree настройка `assign_part_uuids`).
+-   `_partition_value` — Значения (кортеж) выражения `partition by`.
+-   `_sample_factor` — Коэффициент сэмплирования (из запроса).
+

docs/ru/operations/settings/settings.md

@@ -2119,7 +2119,7 @@ ClickHouse генерирует исключение:
 -   1 — включен режим параллельного разбора.
 -   0 — отключен режим параллельного разбора.
 
-Значение по умолчанию: `0`.
+Значение по умолчанию: `1`.
 
 ## output_format_parallel_formatting {#output-format-parallel-formatting}
 
@@ -2130,7 +2130,7 @@ ClickHouse генерирует исключение:
 -   1 — включен режим параллельного форматирования.
 -   0 — отключен режим параллельного форматирования.
 
-Значение по умолчанию: `0`.
+Значение по умолчанию: `1`.
 
 ## min_chunk_bytes_for_parallel_parsing {#min-chunk-bytes-for-parallel-parsing}
 

docs/zh/engines/database-engines/postgresql.md

@@ -24,6 +24,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cac
 -   `database` — 远程数据库名次
 -   `user` — PostgreSQL用户名称
 -   `password` — PostgreSQL用户密码
+-   `schema` - PostgreSQL 模式
 -   `use_table_cache` —  定义数据库表结构是否已缓存或不进行。可选的。默认值： `0`.
 
 ## 支持的数据类型 {#data_types-support}

docs/zh/engines/database-engines/replicated.md

@@ -31,6 +31,7 @@ CREATE DATABASE testdb ENGINE = Replicated('zoo_path', 'shard_name', 'replica_na
 
 当创建数据库的新副本时，该副本会自己创建表。如果副本已经不可用很长一段时间，并且已经滞后于复制日志-它用ZooKeeper中的当前元数据检查它的本地元数据，将带有数据的额外表移动到一个单独的非复制数据库(以免意外地删除任何多余的东西)，创建缺失的表，如果表名已经被重命名，则更新表名。数据在`ReplicatedMergeTree`级别被复制，也就是说，如果表没有被复制，数据将不会被复制(数据库只负责元数据)。
 
+允许[`ALTER TABLE ATTACH|FETCH|DROP|DROP DETACHED|DETACH PARTITION|PART`](../../sql-reference/statements/alter/partition.md)查询，但不允许复制。数据库引擎将只向当前副本添加/获取/删除分区/部件。但是，如果表本身使用了Replicated表引擎，那么数据将在使用`ATTACH`后被复制。
 ## 使用示例 {#usage-example}
 
 创建三台主机的集群:

docs/zh/operations/system-tables/query_thread_log.md

@@ -1,67 +1,62 @@
----
-machine_translated: true
-machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3
----
-
 # 系统。query_thread_log {#system_tables-query_thread_log}
 
 包含有关执行查询的线程的信息，例如，线程名称、线程开始时间、查询处理的持续时间。
 
-开始记录:
+开启日志功能:
 
-1.  在配置参数 [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) 科。
+1.  在配置参数 [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) 部分。
-2.  设置 [log_query_threads](../../operations/settings/settings.md#settings-log-query-threads) 到1。
+2.  设置 [log_query_threads](../../operations/settings/settings.md#settings-log-query-threads) 为1。
 
-数据的冲洗周期设置在 `flush_interval_milliseconds` 的参数 [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) 服务器设置部分。 要强制冲洗，请使用 [SYSTEM FLUSH LOGS](../../sql-reference/statements/system.md#query_language-system-flush_logs) 查询。
+数据从缓存写入数据表周期时间参数 `flush_interval_milliseconds` 位于 [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) 服务器设置部分。如果需要强制从缓存写入数据表，请使用 [SYSTEM FLUSH LOGS](../../sql-reference/statements/system.md#query_language-system-flush_logs) 查询请求。
 
-ClickHouse不会自动从表中删除数据。 看 [导言](../../operations/system-tables/index.md#system-tables-introduction) 欲了解更多详情。
+ClickHouse不会自动从表中删除数据。 欲了解更多详情，请参照 [介绍](../../operations/system-tables/index.md#system-tables-introduction)。
 
 列:
 
--   `event_date` ([日期](../../sql-reference/data-types/date.md)) — The date when the thread has finished execution of the query.
+-   `event_date` ([日期](../../sql-reference/data-types/date.md)) — 该查询线程执行完成的日期。
--   `event_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — The date and time when the thread has finished execution of the query.
+-   `event_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — 该查询线程执行完成的时间。
--   `query_start_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — Start time of query execution.
+-   `query_start_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — 查询的开始时间。
--   `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution.
+-   `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 查询执行持续的时间。
--   `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read rows.
+-   `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 读取的行数。
--   `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read bytes.
+-   `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 读取的字节数。
--   `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` 查询，写入的行数。 对于其他查询，列值为0。
+-   `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 对于 `INSERT` 查询，写入的行数。 对于其他查询，为0。
--   `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` 查询时，写入的字节数。 对于其他查询，列值为0。
+-   `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 对于 `INSERT` 查询，写入的字节数。 对于其他查询，为0。
--   `memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — The difference between the amount of allocated and freed memory in context of this thread.
+-   `memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — 在线程上下文，分配的内存和空闲内存之差。
--   `peak_memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — The maximum difference between the amount of allocated and freed memory in context of this thread.
+-   `peak_memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — 在线程上下文，分配的内存和空闲内存之差的最大值。
--   `thread_name` ([字符串](../../sql-reference/data-types/string.md)) — Name of the thread.
+-   `thread_name` ([字符串](../../sql-reference/data-types/string.md)) — 线程名。
--   `thread_number` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Internal thread ID.
+-   `thread_number` ([UInt32](../../sql-reference/data-types/int-uint.md)) — 内部线程ID。
--   `thread_id` ([Int32](../../sql-reference/data-types/int-uint.md)) — thread ID.
+-   `thread_id` ([Int32](../../sql-reference/data-types/int-uint.md)) — 线程ID。
--   `master_thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — OS initial ID of initial thread.
+-   `master_thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — OS初始线程的初始ID。
--   `query` ([字符串](../../sql-reference/data-types/string.md)) — Query string.
+-   `query` ([字符串](../../sql-reference/data-types/string.md)) — 查询语句。
--   `is_initial_query` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Query type. Possible values:
+-   `is_initial_query` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 查询类型，可能的值：
-    -   1 — Query was initiated by the client.
+    -   1 — 由用户发起的查询。
-    -   0 — Query was initiated by another query for distributed query execution.
+    -   0 — 由其他查询发起的分布式查询。
--   `user` ([字符串](../../sql-reference/data-types/string.md)) — Name of the user who initiated the current query.
+-   `user` ([字符串](../../sql-reference/data-types/string.md)) — 发起查询的用户名。
--   `query_id` ([字符串](../../sql-reference/data-types/string.md)) — ID of the query.
+-   `query_id` ([字符串](../../sql-reference/data-types/string.md)) — 查询的ID。
--   `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query.
+-   `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起查询的IP地址。
--   `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the query.
+-   `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的端口。
--   `initial_user` ([字符串](../../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution).
+-   `initial_user` ([字符串](../../sql-reference/data-types/string.md)) — 首次发起查询的用户名（对于分布式查询）。
--   `initial_query_id` ([字符串](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
+-   `initial_query_id` ([字符串](../../sql-reference/data-types/string.md)) — 首次发起查询的ID（对于分布式查询）。
--   `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
+-   `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起该查询的父查询IP地址。
--   `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the parent query.
+-   `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起该查询的父查询端口。
--   `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Interface that the query was initiated from. Possible values:
+-   `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的界面，可能的值:
     -   1 — TCP.
     -   2 — HTTP.
--   `os_user` ([字符串](../../sql-reference/data-types/string.md)) — OS's username who runs [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md).
+-   `os_user` ([字符串](../../sql-reference/data-types/string.md)) — 使用 [clickhouse-client](../../interfaces/cli.md) 的系统用户名。
--   `client_hostname` ([字符串](../../sql-reference/data-types/string.md)) — Hostname of the client machine where the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或者运行另一个TCP客户端。
+-   `client_hostname` ([字符串](../../sql-reference/data-types/string.md)) — 运行 [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的主机名。
--   `client_name` ([字符串](../../sql-reference/data-types/string.md)) — The [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端名称。
+-   `client_name` ([字符串](../../sql-reference/data-types/string.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的名称。
--   `client_revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Revision of the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端。
+-   `client_revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的修订号。
--   `client_version_major` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Major version of the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端。
+-   `client_version_major` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的主版本号。
--   `client_version_minor` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Minor version of the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端。
+-   `client_version_minor` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的次版本号。
--   `client_version_patch` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Patch component of the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端版本。
+-   `client_version_patch` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的补丁版本号。
--   `http_method` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — HTTP method that initiated the query. Possible values:
+-   `http_method` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的HTTP方法，可能的值：
-    -   0 — The query was launched from the TCP interface.
+    -   0 — 查询通过TCP界面发起。
     -   1 — `GET` 方法被使用。
     -   2 — `POST` 方法被使用。
--   `http_user_agent` ([字符串](../../sql-reference/data-types/string.md)) — The `UserAgent` http请求中传递的标头。
+-   `http_user_agent` ([字符串](../../sql-reference/data-types/string.md)) — `UserAgent` HTTP请求中传递的UA表头。
--   `quota_key` ([字符串](../../sql-reference/data-types/string.md)) — The “quota key” 在指定 [配额](../../operations/quotas.md) 设置（见 `keyed`).
+-   `quota_key` ([字符串](../../sql-reference/data-types/string.md)) —  “quota key” 在 [配额](../../operations/quotas.md) 设置内（详见 `keyed`).
--   `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
+-   `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse 修订版本号.
--   `ProfileEvents` ([数组（字符串, UInt64)](../../sql-reference/data-types/array.md)) — Counters that measure different metrics for this thread. The description of them could be found in the table [系统。活动](#system_tables-events).
+-   `ProfileEvents` ([数组（字符串, UInt64)](../../sql-reference/data-types/array.md)) — 对于该线程的多个指标计数器。这一项可以参考 [system.events](#system_tables-events).
 
 **示例**
 
@@ -113,4 +108,5 @@ ProfileEvents:        {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'Compr
 
 **另请参阅**
 
--   [系统。query_log](../../operations/system-tables/query_log.md#system_tables-query_log) — Description of the `query_log` 系统表，其中包含有关查询执行的公共信息。
+-   [system.query_log](../../operations/system-tables/query_log.md#system_tables-query_log) — `query_log` 系统表描述，其中包含有关查询执行的公共信息。
+-   [system.query_views_log](../../operations/system-tables/query_views_log.md#system_tables-query_views_log) — 这个表包含在查询线程中使用的各个视图的信息。

docs/zh/sql-reference/data-types/int-uint.md

@@ -1,17 +1,41 @@
-# UInt8,UInt16,UInt32,UInt64,Int8,Int16,Int32,Int64 {#uint8-uint16-uint32-uint64-int8-int16-int32-int64}
+---
+toc_priority: 40
+toc_title: UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256
+---
+
+# UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256
 
 固定长度的整型，包括有符号整型或无符号整型。
 
+创建表时，可以为整数设置类型参数 (例如. `TINYINT(8)`, `SMALLINT(16)`, `INT(32)`, `BIGINT(64)`), 但 ClickHouse 会忽略它们.
+
+
 ## 整型范围 {#int-ranges}
 
--   Int8-\[-128:127\]
+
--   Int16-\[-32768:32767\]
+-   `Int8` — \[-128 : 127\]
--   Int32-\[-2147483648:2147483647\]
+-   `Int16` — \[-32768 : 32767\]
--   Int64-\[-9223372036854775808:9223372036854775807\]
+-   `Int32` — \[-2147483648 : 2147483647\]
+-   `Int64` — \[-9223372036854775808 : 9223372036854775807\]
+-   `Int128` — \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\]
+-   `Int256` — \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\]
+
+别名：
+
+-   `Int8` — `TINYINT`, `BOOL`, `BOOLEAN`, `INT1`.
+-   `Int16` — `SMALLINT`, `INT2`.
+-   `Int32` — `INT`, `INT4`, `INTEGER`.
+-   `Int64` — `BIGINT`.
 
 ## 无符号整型范围 {#uint-ranges}
 
--   UInt8-\[0:255\]
+
--   UInt16-\[0:65535\]
+-   `UInt8` — \[0 : 255\]
--   UInt32-\[0:4294967295\]
+-   `UInt16` — \[0 : 65535\]
--   UInt64-\[0:18446744073709551615\]
+-   `UInt32` — \[0 : 4294967295\]
+-   `UInt64` — \[0 : 18446744073709551615\]
+-   `UInt128` — \[0 : 340282366920938463463374607431768211455\]
+-   `UInt256` — \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
+
+
+[源文档](https://clickhouse.com/docs/en/data_types/int_uint/) <!--hide-->

docs/zh/whats-new/roadmap.md

@@ -5,6 +5,6 @@ toc_title: Roadmap
 
 # Roadmap {#roadmap}
 
-`2021年Roadmap`已公布供公开讨论查看[这里](https://github.com/ClickHouse/ClickHouse/issues/17623).
+`2022年Roadmap`已公布供公开讨论查看 [这里](https://github.com/ClickHouse/ClickHouse/issues/32513).
 
 {## [源文章](https://clickhouse.com/docs/en/roadmap/) ##}

programs/client/Client.cpp

@@ -481,7 +481,19 @@ catch (...)
 
 void Client::connect()
 {
-    connection_parameters = ConnectionParameters(config());
+    UInt16 default_port = ConnectionParameters::getPortFromConfig(config());
+    connection_parameters = ConnectionParameters(config(), hosts_ports[0].host,
+                                                 hosts_ports[0].port.value_or(default_port));
+
+    String server_name;
+    UInt64 server_version_major = 0;
+    UInt64 server_version_minor = 0;
+    UInt64 server_version_patch = 0;
+
+    for (size_t attempted_address_index = 0; attempted_address_index < hosts_ports.size(); ++attempted_address_index)
+    {
+        connection_parameters.host = hosts_ports[attempted_address_index].host;
+        connection_parameters.port = hosts_ports[attempted_address_index].port.value_or(default_port);
 
         if (is_interactive)
             std::cout << "Connecting to "
@@ -490,11 +502,6 @@ void Client::connect()
                       << connection_parameters.host << ":" << connection_parameters.port
                       << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "") << "." << std::endl;
 
-    String server_name;
-    UInt64 server_version_major = 0;
-    UInt64 server_version_minor = 0;
-    UInt64 server_version_patch = 0;
-
         try
         {
             connection = Connection::createConnection(connection_parameters, global_context);
@@ -507,10 +514,14 @@ void Client::connect()
 
             connection->getServerVersion(
                 connection_parameters.timeouts, server_name, server_version_major, server_version_minor, server_version_patch, server_revision);
+            config().setString("host", connection_parameters.host);
+            config().setInt("port", connection_parameters.port);
+            break;
         }
         catch (const Exception & e)
         {
             /// It is typical when users install ClickHouse, type some password and instantly forget it.
+            /// This problem can't be fixed with reconnection so it is not attempted
             if ((connection_parameters.user.empty() || connection_parameters.user == "default")
                 && e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED)
             {
@@ -520,10 +531,27 @@ void Client::connect()
                           << "and deleting this file will reset the password." << std::endl
                           << "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed." << std::endl
                           << std::endl;
-        }
-
                 throw;
             }
+            else
+            {
+                if (attempted_address_index == hosts_ports.size() - 1)
+                    throw;
+
+                if (is_interactive)
+                {
+                    std::cerr << "Connection attempt to database at "
+                              << connection_parameters.host << ":" << connection_parameters.port
+                              << " resulted in failure"
+                              << std::endl
+                              << getExceptionMessage(e, false)
+                              << std::endl
+                              << "Attempting connection to the next provided address"
+                              << std::endl;
+                }
+            }
+        }
+    }
 
     server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch);
     load_suggestions = is_interactive && (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false));
@@ -966,8 +994,11 @@ void Client::addOptions(OptionsDescription & options_description)
     /// Main commandline options related to client functionality and all parameters from Settings.
     options_description.main_description->add_options()
         ("config,c", po::value<std::string>(), "config-file path (another shorthand)")
-        ("host,h", po::value<std::string>()->default_value("localhost"), "server host")
+        ("host,h", po::value<std::vector<HostPort>>()->multitoken()->default_value({{"localhost"}}, "localhost"),
-        ("port", po::value<int>()->default_value(9000), "server port")
+         "list of server hosts with optionally assigned port to connect. List elements are separated by a space."
+         "Every list element looks like '<host>[:<port>]'. If port isn't assigned, connection is made by port from '--port' param"
+         "Example of usage: '-h host1:1 host2 host3:3'")
+        ("port", po::value<int>()->default_value(9000), "server port, which is default port for every host from '--host' param")
         ("secure,s", "Use TLS connection")
         ("user,u", po::value<std::string>()->default_value("default"), "user")
         /** If "--password [value]" is used but the value is omitted, the bad argument exception will be thrown.
@@ -1074,8 +1105,8 @@ void Client::processOptions(const OptionsDescription & options_description,
 
     if (options.count("config"))
         config().setString("config-file", options["config"].as<std::string>());
-    if (options.count("host") && !options["host"].defaulted())
+    if (options.count("host"))
-        config().setString("host", options["host"].as<std::string>());
+        hosts_ports = options["host"].as<std::vector<HostPort>>();
     if (options.count("interleave-queries-file"))
         interleave_queries_files = options["interleave-queries-file"].as<std::vector<std::string>>();
     if (options.count("port") && !options["port"].defaulted())

programs/server/config.xml

@@ -217,13 +217,12 @@
         <!-- The following file is used only if ssl_require_client_auth=1 -->
         <ssl_ca_cert_file>/path/to/ssl_ca_cert_file</ssl_ca_cert_file>
 
-        <!-- Default compression algorithm (applied if client doesn't specify another algorithm, see result_compression in QueryInfo).
+        <!-- Default transport compression type (can be overridden by client, see the transport_compression_type field in QueryInfo).
              Supported algorithms: none, deflate, gzip, stream_gzip -->
-        <compression>deflate</compression>
+        <transport_compression_type>none</transport_compression_type>
 
-        <!-- Default compression level (applied if client doesn't specify another level, see result_compression in QueryInfo).
+        <!-- Default transport compression level. Supported levels: 0..3 -->
-             Supported levels: none, low, medium, high -->
+        <transport_compression_level>0</transport_compression_level>
-        <compression_level>medium</compression_level>
 
         <!-- Send/receive message size limits in bytes. -1 means unlimited -->
         <max_send_message_size>-1</max_send_message_size>

src/Access/Common/AccessType.h

@@ -86,7 +86,7 @@ enum class AccessType
     M(CREATE_DICTIONARY, "", DICTIONARY, CREATE) /* allows to execute {CREATE|ATTACH} DICTIONARY */\
     M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables;
                                                      implicitly enabled by the grant CREATE_TABLE on any table */ \
-    M(CREATE_FUNCTION, "", DATABASE, CREATE) /* allows to execute CREATE FUNCTION */ \
+    M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \
     M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \
     \
     M(DROP_DATABASE, "", DATABASE, DROP) /* allows to execute {DROP|DETACH} DATABASE */\
@@ -94,7 +94,7 @@ enum class AccessType
     M(DROP_VIEW, "", VIEW, DROP) /* allows to execute {DROP|DETACH} TABLE for views;
                                     implicitly enabled by the grant DROP_TABLE */\
     M(DROP_DICTIONARY, "", DICTIONARY, DROP) /* allows to execute {DROP|DETACH} DICTIONARY */\
-    M(DROP_FUNCTION, "", DATABASE, DROP) /* allows to execute DROP FUNCTION */\
+    M(DROP_FUNCTION, "", GLOBAL, DROP) /* allows to execute DROP FUNCTION */\
     M(DROP, "", GROUP, ALL) /* allows to execute {DROP|DETACH} */\
     \
     M(TRUNCATE, "TRUNCATE TABLE", TABLE, ALL) \

src/Access/ContextAccess.cpp

@@ -425,6 +425,7 @@ bool ContextAccess::checkAccessImplHelper(const AccessFlags & flags, const Args
             | AccessType::TRUNCATE;
 
         const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY;
+        const AccessFlags function_ddl = AccessType::CREATE_FUNCTION | AccessType::DROP_FUNCTION;
         const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
         const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
         const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS;
@@ -432,7 +433,7 @@ bool ContextAccess::checkAccessImplHelper(const AccessFlags & flags, const Args
         const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY;
         const AccessFlags not_readonly_1_flags = AccessType::CREATE_TEMPORARY_TABLE;
 
-        const AccessFlags ddl_flags = table_ddl | dictionary_ddl;
+        const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl;
         const AccessFlags introspection_flags = AccessType::INTROSPECTION;
     };
     static const PrecalculatedFlags precalc;

src/Access/tests/gtest_access_rights_ops.cpp

@@ -45,7 +45,7 @@ TEST(AccessRights, Union)
     lhs.grant(AccessType::INSERT);
     rhs.grant(AccessType::ALL, "db1");
     lhs.makeUnion(rhs);
-    ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, CREATE FUNCTION, DROP, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
+    ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
 }
 
 

src/AggregateFunctions/ReservoirSampler.h

@@ -239,6 +239,7 @@ private:
 
     UInt64 genRandom(size_t lim)
     {
+        assert(lim > 0);
         /// With a large number of values, we will generate random numbers several times slower.
         if (lim <= static_cast<UInt64>(rng.max()))
             return static_cast<UInt32>(rng()) % static_cast<UInt32>(lim);

src/Client/ClientBase.cpp

@@ -1317,7 +1317,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
         if (insert && insert->select)
             insert->tryFindInputFunction(input_function);
 
-        bool is_async_insert = global_context->getSettings().async_insert && insert && insert->hasInlinedData();
+        bool is_async_insert = global_context->getSettingsRef().async_insert && insert && insert->hasInlinedData();
 
         /// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately.
         if (insert && (!insert->select || input_function) && !insert->watch && !is_async_insert)
@@ -1929,7 +1929,7 @@ void ClientBase::init(int argc, char ** argv)
 
     /// Output of help message.
     if (options.count("help")
-        || (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help.
+        || (options.count("host") && options["host"].as<std::vector<HostPort>>()[0].host == "elp")) /// If user writes -help instead of --help.
     {
         printHelpMessage(options_description);
         exit(0);

src/Client/ClientBase.h

@@ -5,6 +5,7 @@
 #include <Common/InterruptListener.h>
 #include <Common/ShellCommand.h>
 #include <Common/Stopwatch.h>
+#include <Common/DNSResolver.h>
 #include <Core/ExternalTable.h>
 #include <Poco/Util/Application.h>
 #include <Interpreters/Context.h>
@@ -243,6 +244,25 @@ protected:
     } profile_events;
 
     QueryProcessingStage::Enum query_processing_stage;
+
+    struct HostPort
+    {
+        String host;
+        std::optional<UInt16> port{};
+        friend std::istream & operator>>(std::istream & in, HostPort & hostPort)
+        {
+            String host_with_port;
+            in >> host_with_port;
+            DB::DNSResolver & resolver = DB::DNSResolver::instance();
+            std::pair<Poco::Net::IPAddress, std::optional<UInt16>>
+                host_and_port = resolver.resolveHostOrAddress(host_with_port);
+            hostPort.host = host_and_port.first.toString();
+            hostPort.port = host_and_port.second;
+
+            return in;
+        }
+    };
+    std::vector<HostPort> hosts_ports{};
 };
 
 }

src/Client/ConnectionParameters.cpp

@@ -23,15 +23,13 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
-ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfiguration & config)
+ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfiguration & config,
+                                           std::string connection_host,
+                                           int connection_port) : host(connection_host), port(connection_port)
 {
     bool is_secure = config.getBool("secure", false);
     security = is_secure ? Protocol::Secure::Enable : Protocol::Secure::Disable;
 
-    host = config.getString("host", "localhost");
-    port = config.getInt(
-        "port", config.getInt(is_secure ? "tcp_port_secure" : "tcp_port", is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT));
-
     default_database = config.getString("database", "");
 
     /// changed the default value to "default" to fix the issue when the user in the prompt is blank
@@ -69,4 +67,17 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
             Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0),
             Poco::Timespan(config.getInt("tcp_keep_alive_timeout", 0), 0));
 }
+
+ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfiguration & config)
+    : ConnectionParameters(config, config.getString("host", "localhost"), getPortFromConfig(config))
+{
+}
+
+int ConnectionParameters::getPortFromConfig(const Poco::Util::AbstractConfiguration & config)
+{
+    bool is_secure = config.getBool("secure", false);
+    return config.getInt("port",
+        config.getInt(is_secure ? "tcp_port_secure" : "tcp_port",
+            is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT));
+}
 }

src/Client/ConnectionParameters.h

@@ -24,6 +24,9 @@ struct ConnectionParameters
 
     ConnectionParameters() {}
     ConnectionParameters(const Poco::Util::AbstractConfiguration & config);
+    ConnectionParameters(const Poco::Util::AbstractConfiguration & config, std::string host, int port);
+
+    static int getPortFromConfig(const Poco::Util::AbstractConfiguration & config);
 };
 
 }

src/Columns/ColumnArray.cpp

@@ -50,12 +50,12 @@ ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr &&
     if (!offsets_concrete)
         throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::LOGICAL_ERROR);
 
-    if (!offsets_concrete->empty() && nested_column)
+    if (!offsets_concrete->empty() && data)
     {
         Offset last_offset = offsets_concrete->getData().back();
 
         /// This will also prevent possible overflow in offset.
-        if (nested_column->size() != last_offset)
+        if (data->size() != last_offset)
             throw Exception("offsets_column has data inconsistent with nested_column", ErrorCodes::LOGICAL_ERROR);
     }
 

src/Common/ArenaUtils.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <string.h>
+#include <string>
+#include <base/StringRef.h>
+
+/** Copy string value into Arena.
+  * Arena should support method:
+  * char * alloc(size_t size).
+  */
+template <typename Arena>
+inline StringRef copyStringInArena(Arena & arena, StringRef value)
+{
+    size_t key_size = value.size;
+    char * place_for_key = arena.alloc(key_size);
+    memcpy(reinterpret_cast<void *>(place_for_key), reinterpret_cast<const void *>(value.data), key_size);
+    StringRef result{place_for_key, key_size};
+
+    return result;
+}

src/Common/DNSResolver.cpp

@@ -202,6 +202,45 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, U
     return  Poco::Net::SocketAddress(impl->cache_host(host).front(), port);
 }
 
+std::pair<Poco::Net::IPAddress, std::optional<UInt16>> DNSResolver::resolveHostOrAddress(const std::string & host_and_port)
+{
+    Poco::Net::IPAddress ip;
+
+    size_t number_of_colons = std::count(host_and_port.begin(), host_and_port.end(), ':');
+    if (number_of_colons > 1)
+    {
+        /// IPv6 host
+        if (host_and_port.starts_with('['))
+        {
+            size_t close_bracket_pos = host_and_port.find(']');
+            assert(close_bracket_pos != std::string::npos);
+            ip = resolveHost(host_and_port.substr(0, close_bracket_pos));
+
+            if (close_bracket_pos == host_and_port.size() - 1)
+                return {ip, std::nullopt};
+            if (host_and_port[close_bracket_pos + 1] != ':')
+                throw Exception("Missing delimiter between host and port", ErrorCodes::BAD_ARGUMENTS);
+
+            unsigned int port;
+            if (!Poco::NumberParser::tryParseUnsigned(host_and_port.substr(close_bracket_pos + 2), port))
+                throw Exception("Port must be numeric", ErrorCodes::BAD_ARGUMENTS);
+            if (port > 0xFFFF)
+                throw Exception("Port must be less 0xFFFF", ErrorCodes::BAD_ARGUMENTS);
+            return {ip, port};
+        }
+        return {resolveHost(host_and_port), std::nullopt};
+    }
+    else if (number_of_colons == 1)
+    {
+        /// IPv4 host with port
+        Poco::Net::SocketAddress socket = resolveAddress(host_and_port);
+        return {socket.host(), socket.port()};
+    }
+
+    /// IPv4 host
+    return {resolveHost(host_and_port), std::nullopt};
+}
+
 String DNSResolver::reverseResolve(const Poco::Net::IPAddress & address)
 {
     if (impl->disable_cache)

src/Common/DNSResolver.h

@@ -34,6 +34,10 @@ public:
 
     Poco::Net::SocketAddress resolveAddress(const std::string & host, UInt16 port);
 
+    /// Accepts host names like 'example.com'/'example.com:port' or '127.0.0.1'/'127.0.0.1:port' or '::1'/'[::1]:port'
+    /// and resolves its IP and port, if port is set
+    std::pair<Poco::Net::IPAddress, std::optional<UInt16>> resolveHostOrAddress(const std::string & host_and_port);
+
     /// Accepts host IP and resolves its host name
     String reverseResolve(const Poco::Net::IPAddress & address);
 

src/Common/ProfileEvents.cpp

@@ -289,6 +289,9 @@
     M(MergeTreeMetadataCacheHit, "Number of times the read of meta file was done from MergeTree metadata cache") \
     M(MergeTreeMetadataCacheMiss, "Number of times the read of meta file was not done from MergeTree metadata cache") \
     \
+    M(ScalarSubqueriesGlobalCacheHit, "Number of times a read from a scalar subquery was done using the global cache") \
+    M(ScalarSubqueriesLocalCacheHit, "Number of times a read from a scalar subquery was done using the local cache") \
+    M(ScalarSubqueriesCacheMiss, "Number of times a read from a scalar subquery was not cached and had to be calculated completely")
 
 namespace ProfileEvents
 {

src/Coordination/SnapshotableHashTable.h

@@ -2,6 +2,7 @@
 #include <base/StringRef.h>
 #include <Common/HashTable/HashMap.h>
 #include <Common/ArenaWithFreeLists.h>
+#include <Common/ArenaUtils.h>
 #include <unordered_map>
 #include <list>
 #include <atomic>
@@ -36,6 +37,8 @@ private:
     /// Allows to avoid additional copies in updateValue function
     size_t snapshot_up_to_size = 0;
     ArenaWithFreeLists arena;
+    /// Collect invalid iterators to avoid traversing the whole list
+    std::vector<Mapped> snapshot_invalid_iters;
 
     uint64_t approximate_data_size{0};
 
@@ -113,17 +116,6 @@ private:
         }
     }
 
-    StringRef copyStringInArena(const std::string & value_to_copy)
-    {
-        size_t value_to_copy_size = value_to_copy.size();
-        char * place_for_key = arena.alloc(value_to_copy_size);
-        memcpy(reinterpret_cast<void *>(place_for_key), reinterpret_cast<const void *>(value_to_copy.data()), value_to_copy_size);
-        StringRef updated_value{place_for_key, value_to_copy_size};
-
-        return updated_value;
-    }
-
-
 public:
 
     using iterator = typename List::iterator;
@@ -137,7 +129,7 @@ public:
 
         if (!it)
         {
-            ListElem elem{copyStringInArena(key), value, true};
+            ListElem elem{copyStringInArena(arena, key), value, true};
             auto itr = list.insert(list.end(), elem);
             bool inserted;
             map.emplace(itr->key, it, inserted, hash_value);
@@ -159,7 +151,7 @@ public:
 
         if (it == map.end())
         {
-            ListElem elem{copyStringInArena(key), value, true};
+            ListElem elem{copyStringInArena(arena, key), value, true};
             auto itr = list.insert(list.end(), elem);
             bool inserted;
             map.emplace(itr->key, it, inserted, hash_value);
@@ -175,6 +167,7 @@ public:
                 list_itr->active_in_map = false;
                 auto new_list_itr = list.insert(list.end(), elem);
                 it->getMapped() = new_list_itr;
+                snapshot_invalid_iters.push_back(list_itr);
             }
             else
             {
@@ -195,6 +188,7 @@ public:
         if (snapshot_mode)
         {
             list_itr->active_in_map = false;
+            snapshot_invalid_iters.push_back(list_itr);
             list_itr->free_key = true;
             map.erase(it->getKey());
         }
@@ -235,6 +229,7 @@ public:
             {
                 auto elem_copy = *(list_itr);
                 list_itr->active_in_map = false;
+                snapshot_invalid_iters.push_back(list_itr);
                 updater(elem_copy.value);
                 auto itr = list.insert(list.end(), elem_copy);
                 it->getMapped() = itr;
@@ -274,23 +269,15 @@ public:
 
     void clearOutdatedNodes()
     {
-        auto start = list.begin();
+        for (auto & itr: snapshot_invalid_iters)
-        auto end = list.end();
-        for (auto itr = start; itr != end;)
-        {
-            if (!itr->active_in_map)
         {
+            assert(!itr->active_in_map);
             updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size, itr->value.sizeInBytes(), 0);
             if (itr->free_key)
                 arena.free(const_cast<char *>(itr->key.data), itr->key.size);
-                itr = list.erase(itr);
+            list.erase(itr);
-            }
-            else
-            {
-                assert(!itr->free_key);
-                itr++;
-            }
         }
+        snapshot_invalid_iters.clear();
     }
 
     void clear()
@@ -310,7 +297,6 @@ public:
 
     void disableSnapshotMode()
     {
-
         snapshot_mode = false;
         snapshot_up_to_size = 0;
     }

src/Core/Settings.h

@@ -70,7 +70,9 @@ class IColumn;
     M(UInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \
     M(UInt64, distributed_connections_pool_size, 1024, "Maximum number of connections with one remote server in the pool.", 0) \
     M(UInt64, connections_with_failover_max_tries, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, "The maximum number of attempts to connect to replicas.", 0) \
-    M(UInt64, s3_min_upload_part_size, 32*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \
+    M(UInt64, s3_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \
+    M(UInt64, s3_upload_part_size_multiply_factor, 2, "Multiply s3_min_upload_part_size by this factor each time s3_multiply_parts_count_threshold parts were uploaded from a single write to S3.", 0) \
+    M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, 1000, "Each time this number of parts was uploaded to S3 s3_min_upload_part_size multiplied by s3_upload_part_size_multiply_factor.", 0) \
     M(UInt64, s3_max_single_part_upload_size, 32*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \
     M(UInt64, s3_max_single_read_retries, 4, "The maximum number of retries during single S3 read.", 0) \
     M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \
@@ -262,6 +264,7 @@ class IColumn;
     M(UInt64, http_max_fields, 1000000, "Maximum number of fields in HTTP header", 0) \
     M(UInt64, http_max_field_name_size, 1048576, "Maximum length of field name in HTTP header", 0) \
     M(UInt64, http_max_field_value_size, 1048576, "Maximum length of field value in HTTP header", 0) \
+    M(Bool, http_skip_not_found_url_for_globs, true, "Skip url's for globs with HTTP_NOT_FOUND error", 0) \
     M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \
     M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \
     M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \
@@ -426,6 +429,7 @@ class IColumn;
     M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \
     \
     M(DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic, "Default database engine.", 0) \
+    M(DefaultTableEngine, default_table_engine, DefaultTableEngine::None, "Default table engine used when ENGINE is not set in CREATE statement.",0) \
     M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \
     M(Bool, database_atomic_wait_for_drop_and_detach_synchronously, false, "When executing DROP or DETACH TABLE in Atomic database, wait for table data to be finally dropped or detached.", 0) \
     M(Bool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \
@@ -479,7 +483,6 @@ class IColumn;
     M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \
     M(Bool, optimize_skip_merged_partitions, false, "Skip partitions with one part with level > 0 in optimize final", 0) \
     M(Bool, optimize_on_insert, true, "Do the same transformation for inserted block of data as if merge was done on this block.", 0) \
-    M(Bool, allow_experimental_projection_optimization, false, "Enable projection optimization when processing SELECT queries", 0) \
     M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \
     M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \
     M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \
@@ -579,6 +582,7 @@ class IColumn;
     MAKE_OBSOLETE(M, UInt64, merge_tree_clear_old_parts_interval_seconds, 1) \
     MAKE_OBSOLETE(M, UInt64, partial_merge_join_optimizations, 0) \
     MAKE_OBSOLETE(M, MaxThreads, max_alter_threads, 0) \
+    MAKE_OBSOLETE(M, Bool, allow_experimental_projection_optimization, true) \
     /** The section above is for obsolete settings. Do not add anything there. */
 
 

src/Core/SettingsEnums.cpp

@@ -93,6 +93,16 @@ IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultDatabaseEngine, ErrorCodes::BAD_ARGUME
     {{"Ordinary", DefaultDatabaseEngine::Ordinary},
      {"Atomic",   DefaultDatabaseEngine::Atomic}})
 
+IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultTableEngine, ErrorCodes::BAD_ARGUMENTS,
+    {{"None", DefaultTableEngine::None},
+     {"Log", DefaultTableEngine::Log},
+     {"StripeLog", DefaultTableEngine::StripeLog},
+     {"MergeTree", DefaultTableEngine::MergeTree},
+     {"ReplacingMergeTree", DefaultTableEngine::ReplacingMergeTree},
+     {"ReplicatedMergeTree", DefaultTableEngine::ReplicatedMergeTree},
+     {"ReplicatedReplacingMergeTree", DefaultTableEngine::ReplicatedReplacingMergeTree},
+     {"Memory", DefaultTableEngine::Memory}})
+
 IMPLEMENT_SETTING_MULTI_ENUM(MySQLDataTypesSupport, ErrorCodes::UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL,
     {{"decimal",    MySQLDataTypesSupport::DECIMAL},
      {"datetime64", MySQLDataTypesSupport::DATETIME64}})

src/Core/SettingsEnums.h

@@ -120,6 +120,19 @@ enum class DefaultDatabaseEngine
 
 DECLARE_SETTING_ENUM(DefaultDatabaseEngine)
 
+enum class DefaultTableEngine
+{
+    None = 0, /// Disable. Need to use ENGINE =
+    Log,
+    StripeLog,
+    MergeTree,
+    ReplacingMergeTree,
+    ReplicatedMergeTree,
+    ReplicatedReplacingMergeTree,
+    Memory,
+};
+
+DECLARE_SETTING_ENUM(DefaultTableEngine)
 
 enum class MySQLDataTypesSupport
 {

src/Databases/DatabaseOnDisk.cpp

@@ -77,6 +77,10 @@ std::pair<String, StoragePtr> createTableFromAST(
         /// - the code is simpler, since the query is already brought to a suitable form.
         if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns)
         {
+            if (!ast_create_query.storage || !ast_create_query.storage->engine)
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid storage definition in metadata file: "
+                                                           "it's a bug or result of manual intervention in metadata files");
+
             if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(ast_create_query.storage->engine->name))
                 throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
             /// Leave columns empty.

src/Databases/MySQL/MaterializedMySQLSyncThread.cpp

@@ -316,7 +316,7 @@ getTableOutput(const String & database_name, const String & table_name, ContextM
     return std::move(res.pipeline);
 }
 
-static inline String reWriteMysqlQueryColumn(mysqlxx::Pool::Entry & connection, const String & database_name, const String & table_name, const Settings & global_settings)
+static inline String rewriteMysqlQueryColumn(mysqlxx::Pool::Entry & connection, const String & database_name, const String & table_name, const Settings & global_settings)
 {
     Block tables_columns_sample_block
             {
@@ -376,7 +376,7 @@ static inline void dumpDataForTables(
 
             auto pipeline = getTableOutput(database_name, table_name, query_context);
             StreamSettings mysql_input_stream_settings(context->getSettingsRef());
-            String mysql_select_all_query = "SELECT " + reWriteMysqlQueryColumn(connection, mysql_database_name, table_name, context->getSettings()) + " FROM "
+            String mysql_select_all_query = "SELECT " + rewriteMysqlQueryColumn(connection, mysql_database_name, table_name, context->getSettingsRef()) + " FROM "
                     + backQuoteIfNeed(mysql_database_name) + "." + backQuoteIfNeed(table_name);
             LOG_INFO(&Poco::Logger::get("MaterializedMySQLSyncThread(" + database_name + ")"), "mysql_select_all_query is {}", mysql_select_all_query);
             auto input = std::make_unique<MySQLSource>(connection, mysql_select_all_query, pipeline.getHeader(), mysql_input_stream_settings);

src/Dictionaries/CacheDictionaryStorage.h

@@ -8,10 +8,10 @@
 #include <Common/randomSeed.h>
 #include <Common/Arena.h>
 #include <Common/ArenaWithFreeLists.h>
+#include <Common/ArenaUtils.h>
 #include <Common/HashTable/LRUHashMap.h>
 #include <Dictionaries/DictionaryStructure.h>
 #include <Dictionaries/ICacheDictionaryStorage.h>
-#include <Dictionaries/DictionaryHelpers.h>
 
 
 namespace DB

src/Dictionaries/DictionaryHelpers.h

@@ -623,17 +623,6 @@ void mergeBlockWithPipe(
     }
 }
 
-template <typename Arena>
-static StringRef copyStringInArena(Arena & arena, StringRef value)
-{
-    size_t key_size = value.size;
-    char * place_for_key = arena.alloc(key_size);
-    memcpy(reinterpret_cast<void *>(place_for_key), reinterpret_cast<const void *>(value.data), key_size);
-    StringRef result{place_for_key, key_size};
-
-    return result;
-}
-
 /**
  * Returns ColumnVector data as PaddedPodArray.
 

src/Dictionaries/ExecutablePoolDictionarySource.cpp

@@ -197,7 +197,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory)
 
         size_t max_command_execution_time = config.getUInt64(settings_config_prefix + ".max_command_execution_time", 10);
 
-        size_t max_execution_time_seconds = static_cast<size_t>(context->getSettings().max_execution_time.totalSeconds());
+        size_t max_execution_time_seconds = static_cast<size_t>(context->getSettingsRef().max_execution_time.totalSeconds());
         if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds)
             max_command_execution_time = max_execution_time_seconds;
 

src/Dictionaries/FlatDictionary.cpp

@@ -3,6 +3,7 @@
 #include <Core/Defines.h>
 #include <Common/HashTable/HashMap.h>
 #include <Common/HashTable/HashSet.h>
+#include <Common/ArenaUtils.h>
 
 #include <DataTypes/DataTypesDecimal.h>
 #include <IO/WriteHelpers.h>
@@ -13,7 +14,7 @@
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <Processors/Executors/PullingPipelineExecutor.h>
 
-#include <Dictionaries//DictionarySource.h>
+#include <Dictionaries/DictionarySource.h>
 #include <Dictionaries/DictionaryFactory.h>
 #include <Dictionaries/HierarchyDictionariesUtils.h>
 

src/Dictionaries/HashedArrayDictionary.cpp

@@ -1,5 +1,6 @@
 #include "HashedArrayDictionary.h"
 
+#include <Common/ArenaUtils.h>
 #include <Core/Defines.h>
 #include <DataTypes/DataTypesDecimal.h>
 #include <Columns/ColumnsNumber.h>

src/Dictionaries/HashedDictionary.cpp

@@ -1,5 +1,6 @@
 #include "HashedDictionary.h"
 
+#include <Common/ArenaUtils.h>
 #include <Core/Defines.h>
 #include <DataTypes/DataTypesDecimal.h>
 #include <Columns/ColumnsNumber.h>

src/Dictionaries/RangeHashedDictionary.cpp

@@ -1,5 +1,7 @@
 #include <Dictionaries/RangeHashedDictionary.h>
 
+#include <Common/ArenaUtils.h>
+
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypesDecimal.h>
 #include <DataTypes/DataTypeEnum.h>

src/Dictionaries/SSDCacheDictionaryStorage.h

@@ -16,6 +16,7 @@
 #include <Common/randomSeed.h>
 #include <Common/Arena.h>
 #include <Common/ArenaWithFreeLists.h>
+#include <Common/ArenaUtils.h>
 #include <Common/MemorySanitizer.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/HashTable/HashMap.h>

src/Disks/S3/DiskS3.cpp

@@ -283,6 +283,8 @@ std::unique_ptr<WriteBufferFromFileBase> DiskS3::writeFile(const String & path,
         bucket,
         metadata.remote_fs_root_path + s3_path,
         settings->s3_min_upload_part_size,
+        settings->s3_upload_part_size_multiply_factor,
+        settings->s3_upload_part_size_multiply_parts_count_threshold,
         settings->s3_max_single_part_upload_size,
         std::move(object_metadata),
         buf_size,
@@ -338,6 +340,8 @@ void DiskS3::createFileOperationObject(const String & operation_name, UInt64 rev
         bucket,
         remote_fs_root_path + key,
         settings->s3_min_upload_part_size,
+        settings->s3_upload_part_size_multiply_factor,
+        settings->s3_upload_part_size_multiply_parts_count_threshold,
         settings->s3_max_single_part_upload_size,
         metadata);
 
@@ -417,6 +421,8 @@ void DiskS3::saveSchemaVersion(const int & version)
         bucket,
         remote_fs_root_path + SCHEMA_VERSION_OBJECT,
         settings->s3_min_upload_part_size,
+        settings->s3_upload_part_size_multiply_factor,
+        settings->s3_upload_part_size_multiply_parts_count_threshold,
         settings->s3_max_single_part_upload_size);
 
     writeIntText(version, buffer);
@@ -1076,6 +1082,8 @@ DiskS3Settings::DiskS3Settings(
     const std::shared_ptr<Aws::S3::S3Client> & client_,
     size_t s3_max_single_read_retries_,
     size_t s3_min_upload_part_size_,
+    size_t s3_upload_part_size_multiply_factor_,
+    size_t s3_upload_part_size_multiply_parts_count_threshold_,
     size_t s3_max_single_part_upload_size_,
     size_t min_bytes_for_seek_,
     bool send_metadata_,
@@ -1085,6 +1093,8 @@ DiskS3Settings::DiskS3Settings(
     : client(client_)
     , s3_max_single_read_retries(s3_max_single_read_retries_)
     , s3_min_upload_part_size(s3_min_upload_part_size_)
+    , s3_upload_part_size_multiply_factor(s3_upload_part_size_multiply_factor_)
+    , s3_upload_part_size_multiply_parts_count_threshold(s3_upload_part_size_multiply_parts_count_threshold_)
     , s3_max_single_part_upload_size(s3_max_single_part_upload_size_)
     , min_bytes_for_seek(min_bytes_for_seek_)
     , send_metadata(send_metadata_)

src/Disks/S3/DiskS3.h

@@ -29,6 +29,8 @@ struct DiskS3Settings
         const std::shared_ptr<Aws::S3::S3Client> & client_,
         size_t s3_max_single_read_retries_,
         size_t s3_min_upload_part_size_,
+        size_t s3_upload_part_size_multiply_factor_,
+        size_t s3_upload_part_size_multiply_parts_count_threshold_,
         size_t s3_max_single_part_upload_size_,
         size_t min_bytes_for_seek_,
         bool send_metadata_,
@@ -39,6 +41,8 @@ struct DiskS3Settings
     std::shared_ptr<Aws::S3::S3Client> client;
     size_t s3_max_single_read_retries;
     size_t s3_min_upload_part_size;
+    size_t s3_upload_part_size_multiply_factor;
+    size_t s3_upload_part_size_multiply_parts_count_threshold;
     size_t s3_max_single_part_upload_size;
     size_t min_bytes_for_seek;
     bool send_metadata;

src/Disks/S3/registerDiskS3.cpp

@@ -155,6 +155,8 @@ std::unique_ptr<DiskS3Settings> getSettings(const Poco::Util::AbstractConfigurat
         getClient(config, config_prefix, context),
         config.getUInt64(config_prefix + ".s3_max_single_read_retries", context->getSettingsRef().s3_max_single_read_retries),
         config.getUInt64(config_prefix + ".s3_min_upload_part_size", context->getSettingsRef().s3_min_upload_part_size),
+        config.getUInt64(config_prefix + ".s3_upload_part_size_multiply_factor", context->getSettingsRef().s3_upload_part_size_multiply_factor),
+        config.getUInt64(config_prefix + ".s3_upload_part_size_multiply_parts_count_threshold", context->getSettingsRef().s3_upload_part_size_multiply_parts_count_threshold),
         config.getUInt64(config_prefix + ".s3_max_single_part_upload_size", context->getSettingsRef().s3_max_single_part_upload_size),
         config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024),
         config.getBool(config_prefix + ".send_metadata", false),

src/IO/ReadSettings.h

@@ -82,6 +82,7 @@ struct ReadSettings
     size_t http_max_tries = 1;
     size_t http_retry_initial_backoff_ms = 100;
     size_t http_retry_max_backoff_ms = 1600;
+    bool http_skip_not_found_url_for_globs = true;
 
     /// Set to true for MergeTree tables to make sure
     /// that last position (offset in compressed file) is always passed.

src/IO/ReadWriteBufferFromHTTP.h

@@ -129,6 +129,8 @@ namespace detail
         /// In case of redirects, save result uri to use it if we retry the request.
         std::optional<Poco::URI> saved_uri_redirect;
 
+        bool http_skip_not_found_url;
+
         ReadSettings settings;
         Poco::Logger * log;
 
@@ -146,7 +148,7 @@ namespace detail
             return read_range.begin + offset_from_begin_pos;
         }
 
-        std::istream * call(Poco::URI uri_, Poco::Net::HTTPResponse & response, const std::string & method_)
+        std::istream * callImpl(Poco::URI uri_, Poco::Net::HTTPResponse & response, const std::string & method_)
         {
             // With empty path poco will send "POST  HTTP/1.1" its bug.
             if (uri_.getPath().empty())
@@ -211,7 +213,7 @@ namespace detail
             {
                 try
                 {
-                    call(uri, response, Poco::Net::HTTPRequest::HTTP_HEAD);
+                    call(response, Poco::Net::HTTPRequest::HTTP_HEAD);
 
                     while (isRedirect(response.getStatus()))
                     {
@@ -220,7 +222,7 @@ namespace detail
 
                         session->updateSession(uri_redirect);
 
-                        istr = call(uri_redirect, response, method);
+                        istr = callImpl(uri_redirect, response, method);
                     }
 
                     break;
@@ -237,6 +239,17 @@ namespace detail
             return read_range.end;
         }
 
+        enum class InitializeError
+        {
+            /// If error is not retriable, `exception` variable must be set.
+            NON_RETRIABLE_ERROR,
+            /// Allows to skip not found urls for globs
+            SKIP_NOT_FOUND_URL,
+            NONE,
+        };
+
+        InitializeError initialization_error = InitializeError::NONE;
+
     public:
         using NextCallback = std::function<void(size_t)>;
         using OutStreamCallback = std::function<void(std::ostream &)>;
@@ -253,7 +266,8 @@ namespace detail
             Range read_range_ = {},
             const RemoteHostFilter & remote_host_filter_ = {},
             bool delay_initialization = false,
-            bool use_external_buffer_ = false)
+            bool use_external_buffer_ = false,
+            bool http_skip_not_found_url_ = false)
             : SeekableReadBufferWithSize(nullptr, 0)
             , uri {uri_}
             , method {!method_.empty() ? method_ : out_stream_callback_ ? Poco::Net::HTTPRequest::HTTP_POST : Poco::Net::HTTPRequest::HTTP_GET}
@@ -265,6 +279,7 @@ namespace detail
             , buffer_size {buffer_size_}
             , use_external_buffer {use_external_buffer_}
             , read_range(read_range_)
+            , http_skip_not_found_url(http_skip_not_found_url_)
             , settings {settings_}
             , log(&Poco::Logger::get("ReadWriteBufferFromHTTP"))
         {
@@ -277,17 +292,45 @@ namespace detail
                                 settings.http_max_tries, settings.http_retry_initial_backoff_ms, settings.http_retry_max_backoff_ms);
 
             if (!delay_initialization)
+            {
                 initialize();
+                if (exception)
+                    std::rethrow_exception(exception);
+            }
+        }
+
+        void call(Poco::Net::HTTPResponse & response, const String & method_)
+        {
+            try
+            {
+                istr = callImpl(saved_uri_redirect ? *saved_uri_redirect : uri, response, method_);
+            }
+            catch (...)
+            {
+                if (response.getStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND
+                    && http_skip_not_found_url)
+                {
+                    initialization_error = InitializeError::SKIP_NOT_FOUND_URL;
+                }
+                else
+                {
+                    throw;
+                }
+            }
         }
 
         /**
-         * Note: In case of error return false if error is not retriable, otherwise throw.
+         * Throws if error is retriable, otherwise sets initialization_error = NON_RETRIABLE_ERROR and
+         * saves exception into `exception` variable. In case url is not found and skip_not_found_url == true,
+         * sets initialization_error = SKIP_NOT_FOUND_URL, otherwise throws.
          */
-        bool initialize()
+        void initialize()
         {
             Poco::Net::HTTPResponse response;
 
-            istr = call(saved_uri_redirect ? *saved_uri_redirect : uri, response, method);
+            call(response, method);
+            if (initialization_error != InitializeError::NONE)
+                return;
 
             while (isRedirect(response.getStatus()))
             {
@@ -296,7 +339,7 @@ namespace detail
 
                 session->updateSession(uri_redirect);
 
-                istr = call(uri_redirect, response, method);
+                istr = callImpl(uri_redirect, response, method);
                 saved_uri_redirect = uri_redirect;
             }
 
@@ -310,7 +353,8 @@ namespace detail
                             Exception(ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE,
                                       "Cannot read with range: [{}, {}]", read_range.begin, read_range.end ? *read_range.end : '-'));
 
-                    return false;
+                    initialization_error = InitializeError::NON_RETRIABLE_ERROR;
+                    return;
                 }
                 else if (read_range.end)
                 {
@@ -345,12 +389,14 @@ namespace detail
                 sess->attachSessionData(e.message());
                 throw;
             }
-
-            return true;
         }
 
         bool nextImpl() override
         {
+            if (initialization_error == InitializeError::SKIP_NOT_FOUND_URL)
+                return false;
+            assert(initialization_error == InitializeError::NONE);
+
             if (next_callback)
                 next_callback(count());
 
@@ -392,14 +438,16 @@ namespace detail
                 {
                     if (!impl)
                     {
-                        /// If error is not retriable -- false is returned and exception is set.
+                        initialize();
-                        /// Otherwise the error is thrown and retries continue.
+                        if (initialization_error == InitializeError::NON_RETRIABLE_ERROR)
-                        bool initialized = initialize();
-                        if (!initialized)
                         {
                             assert(exception);
                             break;
                         }
+                        else if (initialization_error == InitializeError::SKIP_NOT_FOUND_URL)
+                        {
+                            return false;
+                        }
 
                         if (use_external_buffer)
                         {
@@ -570,11 +618,12 @@ public:
         Range read_range_ = {},
         const RemoteHostFilter & remote_host_filter_ = {},
         bool delay_initialization_ = true,
-        bool use_external_buffer_ = false)
+        bool use_external_buffer_ = false,
+        bool skip_not_found_url_ = false)
         : Parent(std::make_shared<UpdatableSession>(uri_, timeouts, max_redirects),
             uri_, credentials_, method_, out_stream_callback_, buffer_size_,
             settings_, http_header_entries_, read_range_, remote_host_filter_,
-            delay_initialization_, use_external_buffer_)
+            delay_initialization_, use_external_buffer_, skip_not_found_url_)
     {
     }
 };

src/IO/WriteBufferFromS3.cpp

@@ -54,6 +54,8 @@ WriteBufferFromS3::WriteBufferFromS3(
     const String & bucket_,
     const String & key_,
     size_t minimum_upload_part_size_,
+    size_t upload_part_size_multiply_factor_,
+    size_t upload_part_size_multiply_threshold_,
     size_t max_single_part_upload_size_,
     std::optional<std::map<String, String>> object_metadata_,
     size_t buffer_size_,
@@ -63,7 +65,9 @@ WriteBufferFromS3::WriteBufferFromS3(
     , key(key_)
     , object_metadata(std::move(object_metadata_))
     , client_ptr(std::move(client_ptr_))
-    , minimum_upload_part_size(minimum_upload_part_size_)
+    , upload_part_size(minimum_upload_part_size_)
+    , upload_part_size_multiply_factor(upload_part_size_multiply_factor_)
+    , upload_part_size_multiply_threshold(upload_part_size_multiply_threshold_)
     , max_single_part_upload_size(max_single_part_upload_size_)
     , schedule(std::move(schedule_))
 {
@@ -85,9 +89,10 @@ void WriteBufferFromS3::nextImpl()
     if (multipart_upload_id.empty() && last_part_size > max_single_part_upload_size)
         createMultipartUpload();
 
-    if (!multipart_upload_id.empty() && last_part_size > minimum_upload_part_size)
+    if (!multipart_upload_id.empty() && last_part_size > upload_part_size)
     {
         writePart();
+
         allocateBuffer();
     }
 
@@ -96,6 +101,9 @@ void WriteBufferFromS3::nextImpl()
 
 void WriteBufferFromS3::allocateBuffer()
 {
+    if (total_parts_uploaded != 0 && total_parts_uploaded % upload_part_size_multiply_threshold == 0)
+        upload_part_size *= upload_part_size_multiply_factor;
+
     temporary_buffer = Aws::MakeShared<Aws::StringStream>("temporary buffer");
     temporary_buffer->exceptions(std::ios::badbit);
     last_part_size = 0;
@@ -239,6 +247,8 @@ void WriteBufferFromS3::processUploadRequest(UploadPartTask & task)
     }
     else
         throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
+
+    total_parts_uploaded++;
 }
 
 void WriteBufferFromS3::completeMultipartUpload()

src/IO/WriteBufferFromS3.h

@@ -47,6 +47,8 @@ public:
         const String & bucket_,
         const String & key_,
         size_t minimum_upload_part_size_,
+        size_t upload_part_size_multiply_factor_,
+        size_t upload_part_size_multiply_threshold_,
         size_t max_single_part_upload_size_,
         std::optional<std::map<String, String>> object_metadata_ = std::nullopt,
         size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE,
@@ -85,11 +87,14 @@ private:
     String key;
     std::optional<std::map<String, String>> object_metadata;
     std::shared_ptr<Aws::S3::S3Client> client_ptr;
-    size_t minimum_upload_part_size;
+    size_t upload_part_size;
-    size_t max_single_part_upload_size;
+    const size_t upload_part_size_multiply_factor;
+    const size_t upload_part_size_multiply_threshold;
+    const size_t max_single_part_upload_size;
     /// Buffer to accumulate data.
     std::shared_ptr<Aws::StringStream> temporary_buffer;
-    size_t last_part_size;
+    size_t last_part_size = 0;
+    std::atomic<size_t> total_parts_uploaded = 0;
 
     /// Upload in S3 is made in parts.
     /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts.

src/Interpreters/Aggregator.cpp

@@ -855,12 +855,18 @@ void NO_INLINE Aggregator::executeWithoutKeyImpl(
 
 
 void NO_INLINE Aggregator::executeOnIntervalWithoutKeyImpl(
-    AggregatedDataWithoutKey & res,
+    AggregatedDataVariants & data_variants,
     size_t row_begin,
     size_t row_end,
     AggregateFunctionInstruction * aggregate_instructions,
-    Arena * arena)
+    Arena * arena) const
 {
+    /// `data_variants` will destroy the states of aggregate functions in the destructor
+    data_variants.aggregator = this;
+    data_variants.init(AggregatedDataVariants::Type::without_key);
+
+    AggregatedDataWithoutKey & res = data_variants.without_key;
+
     /// Adding values
     for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst)
     {
@@ -1623,15 +1629,32 @@ Block Aggregator::prepareBlockAndFill(
 }
 
 void Aggregator::addSingleKeyToAggregateColumns(
-    const AggregatedDataVariants & data_variants,
+    AggregatedDataVariants & data_variants,
     MutableColumns & aggregate_columns) const
 {
-    const auto & data = data_variants.without_key;
+    auto & data = data_variants.without_key;
-    for (size_t i = 0; i < params.aggregates_size; ++i)
+
+    size_t i = 0;
+    try
+    {
+        for (i = 0; i < params.aggregates_size; ++i)
         {
             auto & column_aggregate_func = assert_cast<ColumnAggregateFunction &>(*aggregate_columns[i]);
             column_aggregate_func.getData().push_back(data + offsets_of_aggregate_states[i]);
         }
+    }
+    catch (...)
+    {
+        /// Rollback
+        for (size_t rollback_i = 0; rollback_i < i; ++rollback_i)
+        {
+            auto & column_aggregate_func = assert_cast<ColumnAggregateFunction &>(*aggregate_columns[rollback_i]);
+            column_aggregate_func.getData().pop_back();
+        }
+        throw;
+    }
+
+    data = nullptr;
 }
 
 void Aggregator::addArenasToAggregateColumns(

src/Interpreters/Aggregator.h

@@ -1138,12 +1138,12 @@ private:
         AggregateFunctionInstruction * aggregate_instructions,
         Arena * arena) const;
 
-    static void executeOnIntervalWithoutKeyImpl(
+    void executeOnIntervalWithoutKeyImpl(
-        AggregatedDataWithoutKey & res,
+        AggregatedDataVariants & data_variants,
         size_t row_begin,
         size_t row_end,
         AggregateFunctionInstruction * aggregate_instructions,
-        Arena * arena);
+        Arena * arena) const;
 
     template <typename Method>
     void writeToTemporaryFileImpl(
@@ -1307,7 +1307,7 @@ private:
         NestedColumnsHolder & nested_columns_holder) const;
 
     void addSingleKeyToAggregateColumns(
-        const AggregatedDataVariants & data_variants,
+        AggregatedDataVariants & data_variants,
         MutableColumns & aggregate_columns) const;
 
     void addArenasToAggregateColumns(

src/Interpreters/ClusterDiscovery.cpp

@@ -229,7 +229,7 @@ ClusterPtr ClusterDiscovery::makeCluster(const ClusterInfo & cluster_info)
 
     bool secure = cluster_info.current_node.secure;
     auto cluster = std::make_shared<Cluster>(
-        context->getSettings(),
+        context->getSettingsRef(),
         shards,
         /* username= */ context->getUserName(),
         /* password= */ "",

src/Interpreters/Context.cpp

@@ -3190,6 +3190,7 @@ ReadSettings Context::getReadSettings() const
     res.http_max_tries = settings.http_max_tries;
     res.http_retry_initial_backoff_ms = settings.http_retry_initial_backoff_ms;
     res.http_retry_max_backoff_ms = settings.http_retry_max_backoff_ms;
+    res.http_skip_not_found_url_for_globs = settings.http_skip_not_found_url_for_globs;
 
     res.mmap_cache = getMMappedFileCache().get();
 

src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp

@@ -1,9 +1,9 @@
 #include <Interpreters/ExecuteScalarSubqueriesVisitor.h>
 
-#include <Columns/ColumnTuple.h>
 #include <Columns/ColumnNullable.h>
-#include <DataTypes/DataTypeTuple.h>
+#include <Columns/ColumnTuple.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeTuple.h>
 #include <IO/WriteHelpers.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/InterpreterSelectWithUnionQuery.h>
@@ -18,7 +18,14 @@
 #include <Parsers/ASTWithElement.h>
 #include <Parsers/queryToString.h>
 #include <Processors/Executors/PullingAsyncPipelineExecutor.h>
+#include <Common/ProfileEvents.h>
 
+namespace ProfileEvents
+{
+extern const Event ScalarSubqueriesGlobalCacheHit;
+extern const Event ScalarSubqueriesLocalCacheHit;
+extern const Event ScalarSubqueriesCacheMiss;
+}
 
 namespace DB
 {
@@ -72,40 +79,95 @@ static bool worthConvertingToLiteral(const Block & scalar)
     return !useless_literal_types.count(scalar_type_name);
 }
 
-void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr & ast, Data & data)
+static auto getQueryInterpreter(const ASTSubquery & subquery, ExecuteScalarSubqueriesMatcher::Data & data)
 {
-    auto hash = subquery.getTreeHash();
-    auto scalar_query_hash_str = toString(hash.first) + "_" + toString(hash.second);
-
-    Block scalar;
-    if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasScalar(scalar_query_hash_str))
-    {
-        scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str);
-    }
-    else if (data.scalars.count(scalar_query_hash_str))
-    {
-        scalar = data.scalars[scalar_query_hash_str];
-    }
-    else
-    {
     auto subquery_context = Context::createCopy(data.getContext());
     Settings subquery_settings = data.getContext()->getSettings();
     subquery_settings.max_result_rows = 1;
     subquery_settings.extremes = false;
     subquery_context->setSettings(subquery_settings);
+    if (!data.only_analyze && subquery_context->hasQueryContext())
+    {
+        /// Save current cached scalars in the context before analyzing the query
+        /// This is specially helpful when analyzing CTE scalars
+        auto context = subquery_context->getQueryContext();
+        for (const auto & it : data.scalars)
+            context->addScalar(it.first, it.second);
+    }
 
     ASTPtr subquery_select = subquery.children.at(0);
 
     auto options = SelectQueryOptions(QueryProcessingStage::Complete, data.subquery_depth + 1, true);
     options.analyze(data.only_analyze);
 
-        auto interpreter = InterpreterSelectWithUnionQuery(subquery_select, subquery_context, options);
+    return std::make_unique<InterpreterSelectWithUnionQuery>(subquery_select, subquery_context, options);
+}
+
+void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr & ast, Data & data)
+{
+    auto hash = subquery.getTreeHash();
+    auto scalar_query_hash_str = toString(hash.first) + "_" + toString(hash.second);
+
+    std::unique_ptr<InterpreterSelectWithUnionQuery> interpreter = nullptr;
+    bool hit = false;
+    bool is_local = false;
+
+    Block scalar;
+    if (data.local_scalars.count(scalar_query_hash_str))
+    {
+        hit = true;
+        scalar = data.local_scalars[scalar_query_hash_str];
+        is_local = true;
+        ProfileEvents::increment(ProfileEvents::ScalarSubqueriesLocalCacheHit);
+    }
+    else if (data.scalars.count(scalar_query_hash_str))
+    {
+        hit = true;
+        scalar = data.scalars[scalar_query_hash_str];
+        ProfileEvents::increment(ProfileEvents::ScalarSubqueriesGlobalCacheHit);
+    }
+    else
+    {
+        if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasScalar(scalar_query_hash_str))
+        {
+            if (!data.getContext()->getViewSource())
+            {
+                /// We aren't using storage views so we can safely use the context cache
+                scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str);
+                ProfileEvents::increment(ProfileEvents::ScalarSubqueriesGlobalCacheHit);
+                hit = true;
+            }
+            else
+            {
+                /// If we are under a context that uses views that means that the cache might contain values that reference
+                /// the original table and not the view, so in order to be able to check the global cache we need to first
+                /// make sure that the query doesn't use the view
+                /// Note in any case the scalar will end up cached in *data* so this won't be repeated inside this context
+                interpreter = getQueryInterpreter(subquery, data);
+                if (!interpreter->usesViewSource())
+                {
+                    scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str);
+                    ProfileEvents::increment(ProfileEvents::ScalarSubqueriesGlobalCacheHit);
+                    hit = true;
+                }
+            }
+        }
+    }
+
+    if (!hit)
+    {
+        if (!interpreter)
+            interpreter = getQueryInterpreter(subquery, data);
+
+        ProfileEvents::increment(ProfileEvents::ScalarSubqueriesCacheMiss);
+        is_local = interpreter->usesViewSource();
+
         Block block;
 
         if (data.only_analyze)
         {
             /// If query is only analyzed, then constants are not correct.
-            block = interpreter.getSampleBlock();
+            block = interpreter->getSampleBlock();
             for (auto & column : block)
             {
                 if (column.column->empty())
@@ -118,14 +180,14 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr
         }
         else
         {
-            auto io = interpreter.execute();
+            auto io = interpreter->execute();
 
             PullingAsyncPipelineExecutor executor(io.pipeline);
             while (block.rows() == 0 && executor.pull(block));
 
             if (block.rows() == 0)
             {
-                auto types = interpreter.getSampleBlock().getDataTypes();
+                auto types = interpreter->getSampleBlock().getDataTypes();
                 if (types.size() != 1)
                     types = {std::make_shared<DataTypeTuple>(types)};
 
@@ -218,6 +280,9 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr
         ast = std::move(func);
     }
 
+    if (is_local)
+        data.local_scalars[scalar_query_hash_str] = std::move(scalar);
+    else
         data.scalars[scalar_query_hash_str] = std::move(scalar);
 }
 

src/Interpreters/ExecuteScalarSubqueriesVisitor.h

@@ -19,11 +19,8 @@ struct ASTTableExpression;
     *
     * Features
     *
-    * A replacement occurs during query analysis, and not during the main runtime.
+    * A replacement occurs during query analysis, and not during the main runtime, so
-    * This means that the progress indicator will not work during the execution of these requests,
+    * the query result can be used for the index in the table.
-    *  and also such queries can not be aborted.
-    *
-    * But the query result can be used for the index in the table.
     *
     * Scalar subqueries are executed on the request-initializer server.
     * The request is sent to remote servers with already substituted constants.
@@ -37,6 +34,7 @@ public:
     {
         size_t subquery_depth;
         Scalars & scalars;
+        Scalars & local_scalars;
         bool only_analyze;
     };
 

src/Interpreters/ExpressionActions.cpp

@@ -159,8 +159,8 @@ static void setLazyExecutionInfo(
 
     const ActionsDAGReverseInfo::NodeInfo & node_info = reverse_info.nodes_info[reverse_info.reverse_index.at(node)];
 
-    /// If node is used in result, we can't enable lazy execution.
+    /// If node is used in result or it doesn't have parents, we can't enable lazy execution.
-    if (node_info.used_in_result)
+    if (node_info.used_in_result || node_info.parents.empty())
         lazy_execution_info.can_be_lazy_executed = false;
 
     /// To fill lazy execution info for current node we need to create it for all it's parents.

src/Interpreters/IInterpreterUnionOrSelectQuery.h

@@ -40,6 +40,15 @@ public:
 
     void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr &, ContextPtr) const override;
 
+    /// Returns whether the query uses the view source from the Context
+    /// The view source is a virtual storage that currently only materialized views use to replace the source table
+    /// with the incoming block only
+    /// This flag is useful to know for how long we can cache scalars generated by this query: If it doesn't use the virtual storage
+    /// then we can cache the scalars forever (for any query that doesn't use the virtual storage either), but if it does use the virtual
+    /// storage then we can only keep the scalar result around while we are working with that source block
+    /// You can find more details about this under ExecuteScalarSubqueriesMatcher::visit
+    bool usesViewSource() { return uses_view_source; }
+
 protected:
     ASTPtr query_ptr;
     ContextMutablePtr context;
@@ -48,6 +57,7 @@ protected:
     size_t max_streams = 1;
     bool settings_limit_offset_needed = false;
     bool settings_limit_offset_done = false;
+    bool uses_view_source = false;
 };
 }
 

src/Interpreters/InterpreterCreateQuery.cpp

@@ -2,6 +2,7 @@
 
 #include <filesystem>
 
+#include "Common/Exception.h"
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/escapeForFileName.h>
 #include <Common/typeid_cast.h>
@@ -12,6 +13,7 @@
 
 #include <Core/Defines.h>
 #include <Core/Settings.h>
+#include <Core/SettingsEnums.h>
 
 #include <IO/WriteBufferFromFile.h>
 #include <IO/WriteHelpers.h>
@@ -91,6 +93,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_DATABASE;
     extern const int PATH_ACCESS_DENIED;
     extern const int NOT_IMPLEMENTED;
+    extern const int ENGINE_REQUIRED;
 }
 
 namespace fs = std::filesystem;
@@ -157,6 +160,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
         throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", serializeAST(*create.storage));
     }
 
+    if (create.storage && !create.storage->engine)
+        throw Exception(ErrorCodes::INCORRECT_QUERY, "Database engine must be specified");
+
     if (create.storage->engine->name == "Atomic"
         || create.storage->engine->name == "Replicated"
         || create.storage->engine->name == "MaterializedPostgreSQL")
@@ -581,6 +587,17 @@ ConstraintsDescription InterpreterCreateQuery::getConstraintsDescription(const A
 
 InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create) const
 {
+    /// Set the table engine if it was not specified explicitly.
+    setEngine(create);
+
+    /// We have to check access rights again (in case engine was changed).
+    if (create.storage)
+    {
+        auto source_access_type = StorageFactory::instance().getSourceAccessType(create.storage->engine->name);
+        if (source_access_type != AccessType::NONE)
+            getContext()->checkAccess(source_access_type);
+    }
+
     TableProperties properties;
     TableLockHolder as_storage_lock;
 
@@ -645,7 +662,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
     }
     /// We can have queries like "CREATE TABLE <table> ENGINE=<engine>" if <engine>
     /// supports schema inference (will determine table structure in it's constructor).
-    else if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(create.storage->engine->name))
+    else if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(create.storage->engine->name)) // NOLINT
         throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY);
 
     /// Even if query has list of columns, canonicalize it (unfold Nested columns).
@@ -663,8 +680,6 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
     create.columns_list->setOrReplace(create.columns_list->projections, new_projections);
 
     validateTableStructure(create, properties);
-    /// Set the table engine if it was not specified explicitly.
-    setEngine(create);
 
     assert(as_database_saved.empty() && as_table_saved.empty());
     std::swap(create.as_database, as_database_saved);
@@ -718,30 +733,90 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
     }
 }
 
+String InterpreterCreateQuery::getTableEngineName(DefaultTableEngine default_table_engine)
+{
+    switch (default_table_engine)
+    {
+        case DefaultTableEngine::Log:
+            return "Log";
+
+        case DefaultTableEngine::StripeLog:
+            return "StripeLog";
+
+        case DefaultTableEngine::MergeTree:
+            return "MergeTree";
+
+        case DefaultTableEngine::ReplacingMergeTree:
+            return "ReplacingMergeTree";
+
+        case DefaultTableEngine::ReplicatedMergeTree:
+            return "ReplicatedMergeTree";
+
+        case DefaultTableEngine::ReplicatedReplacingMergeTree:
+            return "ReplicatedReplacingMergeTree";
+
+        case DefaultTableEngine::Memory:
+            return "Memory";
+
+        default:
+            throw Exception("default_table_engine is set to unknown value", ErrorCodes::LOGICAL_ERROR);
+    }
+}
+
+void InterpreterCreateQuery::setDefaultTableEngine(ASTStorage & storage, ContextPtr local_context)
+{
+    if (local_context->getSettingsRef().default_table_engine.value == DefaultTableEngine::None)
+        throw Exception(ErrorCodes::ENGINE_REQUIRED, "Table engine is not specified in CREATE query");
+
+    auto engine_ast = std::make_shared<ASTFunction>();
+    auto default_table_engine = local_context->getSettingsRef().default_table_engine.value;
+    engine_ast->name = getTableEngineName(default_table_engine);
+    engine_ast->no_empty_args = true;
+    storage.set(storage.engine, engine_ast);
+}
+
 void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
 {
     if (create.as_table_function)
         return;
 
-    if (create.storage || create.is_dictionary || create.isView())
+    if (create.is_dictionary || create.is_ordinary_view || create.is_live_view || create.is_window_view)
-    {
+        return;
-        if (create.temporary && create.storage && create.storage->engine && create.storage->engine->name != "Memory")
+
-            throw Exception(ErrorCodes::INCORRECT_QUERY,
+    if (create.is_materialized_view && create.to_table_id)
-                "Temporary tables can only be created with ENGINE = Memory, not {}", create.storage->engine->name);
-
         return;
-    }
 
     if (create.temporary)
     {
+        if (create.storage && create.storage->engine && create.storage->engine->name != "Memory")
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables can only be created with ENGINE = Memory, not {}",
+                create.storage->engine->name);
+
+        /// It's possible if some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not.
+        /// It makes sense when default_table_engine setting is used, but not for temporary tables.
+        /// For temporary tables we ignore this setting to allow CREATE TEMPORARY TABLE query without specifying ENGINE
+        /// even if setting is set to MergeTree or something like that (otherwise MergeTree will be substituted and query will fail).
+        if (create.storage && !create.storage->engine)
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Invalid storage definition for temporary table: must be either ENGINE = Memory or empty");
+
         auto engine_ast = std::make_shared<ASTFunction>();
         engine_ast->name = "Memory";
         engine_ast->no_empty_args = true;
         auto storage_ast = std::make_shared<ASTStorage>();
         storage_ast->set(storage_ast->engine, engine_ast);
         create.set(create.storage, storage_ast);
+        return;
     }
-    else if (!create.as_table.empty())
+
+    if (create.storage)
+    {
+        /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
+        if (!create.storage->engine)
+            setDefaultTableEngine(*create.storage, getContext());
+        return;
+    }
+
+    if (!create.as_table.empty())
     {
         /// NOTE Getting the structure from the table specified in the AS is done not atomically with the creation of the table.
 
@@ -754,24 +829,16 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name);
 
         if (as_create.is_ordinary_view)
-            throw Exception(
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a View", qualified_name);
-                "Cannot CREATE a table AS " + qualified_name + ", it is a View",
-                ErrorCodes::INCORRECT_QUERY);
 
         if (as_create.is_live_view)
-            throw Exception(
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Live View", qualified_name);
-                "Cannot CREATE a table AS " + qualified_name + ", it is a Live View",
-                ErrorCodes::INCORRECT_QUERY);
 
         if (as_create.is_window_view)
-            throw Exception(
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Window View", qualified_name);
-                "Cannot CREATE a table AS " + qualified_name + ", it is a Window View",
-                ErrorCodes::INCORRECT_QUERY);
 
         if (as_create.is_dictionary)
-            throw Exception(
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Dictionary", qualified_name);
-                "Cannot CREATE a table AS " + qualified_name + ", it is a Dictionary",
-                ErrorCodes::INCORRECT_QUERY);
 
         if (as_create.storage)
             create.set(create.storage, as_create.storage->ptr());
@@ -779,7 +846,12 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
             create.as_table_function = as_create.as_table_function->clone();
         else
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot set engine, it's a bug.");
+
+        return;
     }
+
+    create.set(create.storage, std::make_shared<ASTStorage>());
+    setDefaultTableEngine(*create.storage, getContext());
 }
 
 static void generateUUIDForTable(ASTCreateQuery & create)

src/Interpreters/InterpreterCreateQuery.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Core/NamesAndAliases.h>
+#include <Core/SettingsEnums.h>
 #include <Access/Common/AccessRightsElement.h>
 #include <Interpreters/IInterpreter.h>
 #include <Storages/ColumnsDescription.h>
@@ -15,6 +16,7 @@ namespace DB
 class ASTCreateQuery;
 class ASTExpressionList;
 class ASTConstraintDeclaration;
+class ASTStorage;
 class IDatabase;
 using DatabasePtr = std::shared_ptr<IDatabase>;
 
@@ -81,6 +83,8 @@ private:
     /// Calculate list of columns, constraints, indices, etc... of table. Rewrite query in canonical way.
     TableProperties getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create) const;
     void validateTableStructure(const ASTCreateQuery & create, const TableProperties & properties) const;
+    static String getTableEngineName(DefaultTableEngine default_table_engine);
+    static void setDefaultTableEngine(ASTStorage & storage, ContextPtr local_context);
     void setEngine(ASTCreateQuery & create) const;
     AccessRightsElements getRequiredAccess() const;
 

src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp

@@ -68,7 +68,10 @@ InterpreterSelectIntersectExceptQuery::InterpreterSelectIntersectExceptQuery(
     nested_interpreters.resize(num_children);
 
     for (size_t i = 0; i < num_children; ++i)
+    {
         nested_interpreters[i] = buildCurrentChildInterpreter(children.at(i));
+        uses_view_source |= nested_interpreters[i]->usesViewSource();
+    }
 
     Blocks headers(num_children);
     for (size_t query_num = 0; query_num < num_children; ++query_num)

src/Interpreters/InterpreterSelectQuery.cpp

@@ -64,8 +64,9 @@
 #include <Processors/Transforms/ExpressionTransform.h>
 #include <Processors/Transforms/FilterTransform.h>
 
-#include <Storages/MergeTree/MergeTreeWhereOptimizer.h>
 #include <Storages/IStorage.h>
+#include <Storages/MergeTree/MergeTreeWhereOptimizer.h>
+#include <Storages/StorageValues.h>
 #include <Storages/StorageView.h>
 
 #include <Functions/IFunction.h>
@@ -315,6 +316,8 @@ InterpreterSelectQuery::InterpreterSelectQuery(
     if (!has_input && !storage)
     {
         storage = joined_tables.getLeftTableStorage();
+        // Mark uses_view_source if the returned storage is the same as the one saved in viewSource
+        uses_view_source |= storage && storage == context->getViewSource();
         got_storage_from_query = true;
     }
 
@@ -336,6 +339,15 @@ InterpreterSelectQuery::InterpreterSelectQuery(
 
         joined_tables.reset(getSelectQuery());
         joined_tables.resolveTables();
+        if (auto view_source = context->getViewSource())
+        {
+            // If we are using a virtual block view to replace a table and that table is used
+            // inside the JOIN then we need to update uses_view_source accordingly so we avoid propagating scalars that we can't cache
+            const auto & storage_values = static_cast<const StorageValues &>(*view_source);
+            auto tmp_table_id = storage_values.getStorageID();
+            for (const auto & t : joined_tables.tablesWithColumns())
+                uses_view_source |= (t.table.database == tmp_table_id.database_name && t.table.table == tmp_table_id.table_name);
+        }
 
         if (storage && joined_tables.isLeftTableSubquery())
         {
@@ -351,7 +363,10 @@ InterpreterSelectQuery::InterpreterSelectQuery(
     {
         interpreter_subquery = joined_tables.makeLeftTableSubquery(options.subquery());
         if (interpreter_subquery)
+        {
             source_header = interpreter_subquery->getSampleBlock();
+            uses_view_source |= interpreter_subquery->usesViewSource();
+        }
     }
 
     joined_tables.rewriteDistributedInAndJoins(query_ptr);
@@ -389,9 +404,8 @@ InterpreterSelectQuery::InterpreterSelectQuery(
             query.setFinal();
 
         /// Save scalar sub queries's results in the query context
-        /// But discard them if the Storage has been modified
+        /// Note that we are only saving scalars and not local_scalars since the latter can't be safely shared across contexts
-        /// In an ideal situation we would only discard the scalars affected by the storage change
+        if (!options.only_analyze && context->hasQueryContext())
-        if (!options.only_analyze && context->hasQueryContext() && !context->getViewSource())
             for (const auto & it : syntax_analyzer_result->getScalars())
                 context->getQueryContext()->addScalar(it.first, it.second);
 
@@ -479,6 +493,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
             /// If there is an aggregation in the outer query, WITH TOTALS is ignored in the subquery.
             if (query_analyzer->hasAggregation())
                 interpreter_subquery->ignoreWithTotals();
+            uses_view_source |= interpreter_subquery->usesViewSource();
         }
 
         required_columns = syntax_analyzer_result->requiredSourceColumns();

src/Interpreters/InterpreterSelectWithUnionQuery.cpp

@@ -138,6 +138,9 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery(
 
         nested_interpreters.emplace_back(
             buildCurrentChildInterpreter(ast->list_of_selects->children.at(query_num), require_full_header ? Names() : current_required_result_column_names));
+        // We need to propagate the uses_view_source flag from children to the (self) parent since, if one of the children uses
+        // a view source that means that the parent uses it too and can be cached globally
+        uses_view_source |= nested_interpreters.back()->usesViewSource();
     }
 
     /// Determine structure of the result.

src/Interpreters/TreeRewriter.cpp

@@ -479,10 +479,11 @@ void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query,
 }
 
 /// Replacing scalar subqueries with constant values.
-void executeScalarSubqueries(ASTPtr & query, ContextPtr context, size_t subquery_depth, Scalars & scalars, bool only_analyze)
+void executeScalarSubqueries(
+    ASTPtr & query, ContextPtr context, size_t subquery_depth, Scalars & scalars, Scalars & local_scalars, bool only_analyze)
 {
     LogAST log;
-    ExecuteScalarSubqueriesVisitor::Data visitor_data{WithContext{context}, subquery_depth, scalars, only_analyze};
+    ExecuteScalarSubqueriesVisitor::Data visitor_data{WithContext{context}, subquery_depth, scalars, local_scalars, only_analyze};
     ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(query);
 }
 
@@ -1158,7 +1159,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect(
     removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates);
 
     /// Executing scalar subqueries - replacing them with constant values.
-    executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, select_options.only_analyze);
+    executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, result.local_scalars, select_options.only_analyze);
 
     if (settings.legacy_column_name_of_tuple_literal)
         markTupleLiteralsAsLegacy(query);
@@ -1248,7 +1249,7 @@ TreeRewriterResultPtr TreeRewriter::analyze(
     normalize(query, result.aliases, result.source_columns_set, false, settings, allow_self_aliases);
 
     /// Executing scalar subqueries. Column defaults could be a scalar subquery.
-    executeScalarSubqueries(query, getContext(), 0, result.scalars, !execute_scalar_subqueries);
+    executeScalarSubqueries(query, getContext(), 0, result.scalars, result.local_scalars, !execute_scalar_subqueries);
 
     if (settings.legacy_column_name_of_tuple_literal)
         markTupleLiteralsAsLegacy(query);

src/Interpreters/TreeRewriter.h

@@ -75,6 +75,7 @@ struct TreeRewriterResult
 
     /// Results of scalar sub queries
     Scalars scalars;
+    Scalars local_scalars;
 
     explicit TreeRewriterResult(
         const NamesAndTypesList & source_columns_,

src/Parsers/ASTSelectQuery.cpp

@@ -22,38 +22,20 @@ namespace ErrorCodes
 ASTPtr ASTSelectQuery::clone() const
 {
     auto res = std::make_shared<ASTSelectQuery>(*this);
+
+    /** NOTE Members must clone exactly in the same order in which they were inserted into `children` in ParserSelectQuery.
+     * This is important because the AST hash depends on the children order and this hash is used for multiple things,
+     * like the column identifiers in the case of subqueries in the IN statement or caching scalar queries (reused in CTEs so it's
+     * important for them to have the same hash).
+     * For distributed query processing, in case one of the servers is localhost and the other one is not, localhost query is executed
+     * within the process and is cloned, and the request is sent to the remote server in text form via TCP.
+     * And if the cloning order does not match the parsing order then different servers will get different identifiers.
+     *
+     * Since the positions map uses <key, position> we can copy it as is and ensure the new children array is created / pushed
+     * in the same order as the existing one */
     res->children.clear();
-    res->positions.clear();
+    for (const auto & child : children)
-
+        res->children.push_back(child->clone());
-#define CLONE(expr) res->setExpression(expr, getExpression(expr, true))
-
-    /** NOTE Members must clone exactly in the same order,
-        *  in which they were inserted into `children` in ParserSelectQuery.
-        * This is important because of the children's names the identifier (getTreeHash) is compiled,
-        *  which can be used for column identifiers in the case of subqueries in the IN statement.
-        * For distributed query processing, in case one of the servers is localhost and the other one is not,
-        *  localhost query is executed within the process and is cloned,
-        *  and the request is sent to the remote server in text form via TCP.
-        * And if the cloning order does not match the parsing order,
-        *  then different servers will get different identifiers.
-        */
-    CLONE(Expression::WITH);
-    CLONE(Expression::SELECT);
-    CLONE(Expression::TABLES);
-    CLONE(Expression::PREWHERE);
-    CLONE(Expression::WHERE);
-    CLONE(Expression::GROUP_BY);
-    CLONE(Expression::HAVING);
-    CLONE(Expression::WINDOW);
-    CLONE(Expression::ORDER_BY);
-    CLONE(Expression::LIMIT_BY_OFFSET);
-    CLONE(Expression::LIMIT_BY_LENGTH);
-    CLONE(Expression::LIMIT_BY);
-    CLONE(Expression::LIMIT_OFFSET);
-    CLONE(Expression::LIMIT_LENGTH);
-    CLONE(Expression::SETTINGS);
-
-#undef CLONE
 
     return res;
 }

src/Parsers/ParserCreateQuery.cpp

@@ -18,6 +18,7 @@
 #include <Parsers/ParserSelectWithUnionQuery.h>
 #include <Parsers/ParserSetQuery.h>
 #include <Common/typeid_cast.h>
+#include <Parsers/ASTColumnDeclaration.h>
 
 
 namespace DB
@@ -353,20 +354,26 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ASTPtr ttl_table;
     ASTPtr settings;
 
-    if (!s_engine.ignore(pos, expected))
+    bool storage_like = false;
-        return false;
 
+    if (s_engine.ignore(pos, expected))
+    {
         s_eq.ignore(pos, expected);
 
         if (!ident_with_optional_params_p.parse(pos, engine, expected))
             return false;
+        storage_like = true;
+    }
 
     while (true)
     {
         if (!partition_by && s_partition_by.ignore(pos, expected))
         {
             if (expression_p.parse(pos, partition_by, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -374,7 +381,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (!primary_key && s_primary_key.ignore(pos, expected))
         {
             if (expression_p.parse(pos, primary_key, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -382,7 +392,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (!order_by && s_order_by.ignore(pos, expected))
         {
             if (expression_p.parse(pos, order_by, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -390,7 +403,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (!sample_by && s_sample_by.ignore(pos, expected))
         {
             if (expression_p.parse(pos, sample_by, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -398,7 +414,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (!ttl_table && s_ttl.ignore(pos, expected))
         {
             if (parser_ttl_list.parse(pos, ttl_table, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -407,10 +426,14 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         {
             if (!settings_p.parse(pos, settings, expected))
                 return false;
+            storage_like = true;
         }
 
         break;
     }
+    // If any part of storage definition is found create storage node
+    if (!storage_like)
+        return false;
 
     auto storage = std::make_shared<ASTStorage>();
     storage->set(storage->engine, engine);
@@ -549,13 +572,11 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
 
         if (!storage_parse_result && !is_temporary)
         {
-            if (!s_as.ignore(pos, expected))
+            if (s_as.ignore(pos, expected) && !table_function_p.parse(pos, as_table_function, expected))
-                return false;
-            if (!table_function_p.parse(pos, as_table_function, expected))
-            {
                 return false;
         }
-        }
+
+        /// Will set default table engine if Storage clause was not parsed
     }
     /** Create queries without list of columns:
       *  - CREATE|ATTACH TABLE ... AS ...
@@ -590,10 +611,6 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
                 }
             }
         }
-        else if (!storage)
-        {
-            return false;
-        }
     }
     auto comment = parseComment(pos, expected);
 
@@ -625,12 +642,14 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     if (comment)
         query->set(query->comment, comment);
 
-    if (query->storage && query->columns_list && query->columns_list->primary_key)
+    if (query->columns_list && query->columns_list->primary_key)
-    {
-        if (query->storage->primary_key)
     {
+        /// If engine is not set will use default one
+        if (!query->storage)
+            query->set(query->storage, std::make_shared<ASTStorage>());
+        else if (query->storage->primary_key)
             throw Exception("Multiple primary keys are not allowed.", ErrorCodes::BAD_ARGUMENTS);
-        }
+
         query->storage->primary_key = query->columns_list->primary_key;
     }
 
@@ -1263,8 +1282,8 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     if (is_materialized_view && !to_table)
     {
         /// Internal ENGINE for MATERIALIZED VIEW must be specified.
-        if (!storage_p.parse(pos, storage, expected))
+        /// Actually check it in Interpreter as default_table_engine can be set
-            return false;
+        storage_p.parse(pos, storage, expected);
 
         if (s_populate.ignore(pos, expected))
             is_populate = true;

src/Parsers/ParserCreateQuery.h

@@ -276,7 +276,7 @@ protected:
 class ParserIndexDeclaration : public IParserBase
 {
 public:
-    ParserIndexDeclaration() {}
+    ParserIndexDeclaration() = default;
 
 protected:
     const char * getName() const override { return "index declaration"; }
@@ -336,7 +336,7 @@ protected:
 
 
 /**
-  * ENGINE = name [PARTITION BY expr] [ORDER BY expr] [PRIMARY KEY expr] [SAMPLE BY expr] [SETTINGS name = value, ...]
+  * [ENGINE = name] [PARTITION BY expr] [ORDER BY expr] [PRIMARY KEY expr] [SAMPLE BY expr] [SETTINGS name = value, ...]
   */
 class ParserStorage : public IParserBase
 {
@@ -391,7 +391,7 @@ class ParserTableOverrideDeclaration : public IParserBase
 {
 public:
     const bool is_standalone;
-    ParserTableOverrideDeclaration(bool is_standalone_ = true) : is_standalone(is_standalone_) { }
+    explicit ParserTableOverrideDeclaration(bool is_standalone_ = true) : is_standalone(is_standalone_) { }
 
 protected:
     const char * getName() const override { return "table override declaration"; }

src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp

@@ -356,6 +356,7 @@ void registerInputFormatJSONEachRow(FormatFactory & factory)
     });
 
     factory.registerFileExtension("ndjson", "JSONEachRow");
+    factory.registerFileExtension("jsonl", "JSONEachRow");
 
     factory.registerInputFormat("JSONStringsEachRow", [](
         ReadBuffer & buf,

src/Processors/Formats/Impl/TSKVRowInputFormat.cpp

@@ -20,7 +20,7 @@ namespace ErrorCodes
 
 
 TSKVRowInputFormat::TSKVRowInputFormat(ReadBuffer & in_, Block header_, Params params_, const FormatSettings & format_settings_)
-    : IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), name_map(header_.columns())
+    : IRowInputFormat(std::move(header_), in_, std::move(params_)), format_settings(format_settings_), name_map(getPort().getHeader().columns())
 {
     const auto & sample_block = getPort().getHeader();
     size_t num_columns = sample_block.columns();

src/Processors/QueryPlan/ReadFromMergeTree.cpp

@@ -921,8 +921,8 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
 
         auto reader_settings = getMergeTreeReaderSettings(context);
 
-        bool use_skip_indexes = context->getSettings().use_skip_indexes;
+        bool use_skip_indexes = settings.use_skip_indexes;
-        if (select.final() && !context->getSettings().use_skip_indexes_if_final)
+        if (select.final() && !settings.use_skip_indexes_if_final)
             use_skip_indexes = false;
 
         result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes(

src/Processors/Transforms/AggregatingInOrderTransform.cpp

@@ -121,7 +121,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk)
 
         /// Add data to aggr. state if interval is not empty. Empty when haven't found current key in new block.
         if (key_begin != key_end)
-            params->aggregator.executeOnIntervalWithoutKeyImpl(variants.without_key, key_begin, key_end, aggregate_function_instructions.data(), variants.aggregates_pool);
+            params->aggregator.executeOnIntervalWithoutKeyImpl(variants, key_begin, key_end, aggregate_function_instructions.data(), variants.aggregates_pool);
 
         current_memory_usage = getCurrentMemoryUsage() - initial_memory_usage;
 

src/Processors/Transforms/CreatingSetsTransform.h

@@ -44,8 +44,8 @@ public:
 private:
     SubqueryForSet subquery;
 
-    std::unique_ptr<PushingPipelineExecutor> executor;
     QueryPipeline table_out;
+    std::unique_ptr<PushingPipelineExecutor> executor;
     UInt64 read_rows = 0;
     Stopwatch watch;
 

src/Processors/Transforms/buildPushingToViewsChain.h

@@ -18,7 +18,7 @@ namespace DB
 
 struct ViewRuntimeData
 {
-    /// A query we should run over inserted block befire pushing into inner storage.
+    /// A query we should run over inserted block before pushing into inner storage.
     const ASTPtr query;
     /// This structure is expected by inner storage. Will convert query result to it.
     Block sample_block;

src/Processors/Transforms/getSourceFromASTInsertQuery.cpp

@@ -59,7 +59,7 @@ InputFormatPtr getInputFormatFromASTInsertQuery(
         : std::make_unique<EmptyReadBuffer>();
 
     /// Create a source from input buffer using format from query
-    auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettings().max_insert_block_size);
+    auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettingsRef().max_insert_block_size);
     source->addBuffer(std::move(input_buffer));
     return source;
 }

src/QueryPipeline/RemoteInserter.cpp

@@ -24,7 +24,9 @@ RemoteInserter::RemoteInserter(
     const String & query_,
     const Settings & settings_,
     const ClientInfo & client_info_)
-    : connection(connection_), query(query_)
+    : connection(connection_)
+    , query(query_)
+    , server_revision(connection.getServerRevision(timeouts))
 {
     ClientInfo modified_client_info = client_info_;
     modified_client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY;

src/QueryPipeline/RemoteInserter.h

@@ -35,12 +35,14 @@ public:
     ~RemoteInserter();
 
     const Block & getHeader() const { return header; }
+    UInt64 getServerRevision() const { return server_revision; }
 
 private:
     Connection & connection;
     String query;
     Block header;
     bool finished = false;
+    UInt64 server_revision;
 };
 
 }

src/Server/GRPCServer.cpp

@@ -51,6 +51,7 @@ using GRPCQueryInfo = clickhouse::grpc::QueryInfo;
 using GRPCResult = clickhouse::grpc::Result;
 using GRPCException = clickhouse::grpc::Exception;
 using GRPCProgress = clickhouse::grpc::Progress;
+using GRPCObsoleteTransportCompression = clickhouse::grpc::ObsoleteTransportCompression;
 
 namespace DB
 {
@@ -101,62 +102,6 @@ namespace
         });
     }
 
-    grpc_compression_algorithm parseCompressionAlgorithm(const String & str)
-    {
-        if (str == "none")
-            return GRPC_COMPRESS_NONE;
-        else if (str == "deflate")
-            return GRPC_COMPRESS_DEFLATE;
-        else if (str == "gzip")
-            return GRPC_COMPRESS_GZIP;
-        else if (str == "stream_gzip")
-            return GRPC_COMPRESS_STREAM_GZIP;
-        else
-            throw Exception("Unknown compression algorithm: '" + str + "'", ErrorCodes::INVALID_CONFIG_PARAMETER);
-    }
-
-    grpc_compression_level parseCompressionLevel(const String & str)
-    {
-        if (str == "none")
-            return GRPC_COMPRESS_LEVEL_NONE;
-        else if (str == "low")
-            return GRPC_COMPRESS_LEVEL_LOW;
-        else if (str == "medium")
-            return GRPC_COMPRESS_LEVEL_MED;
-        else if (str == "high")
-            return GRPC_COMPRESS_LEVEL_HIGH;
-        else
-            throw Exception("Unknown compression level: '" + str + "'", ErrorCodes::INVALID_CONFIG_PARAMETER);
-    }
-
-    grpc_compression_algorithm convertCompressionAlgorithm(const ::clickhouse::grpc::CompressionAlgorithm & algorithm)
-    {
-        if (algorithm == ::clickhouse::grpc::NO_COMPRESSION)
-            return GRPC_COMPRESS_NONE;
-        else if (algorithm == ::clickhouse::grpc::DEFLATE)
-            return GRPC_COMPRESS_DEFLATE;
-        else if (algorithm == ::clickhouse::grpc::GZIP)
-            return GRPC_COMPRESS_GZIP;
-        else if (algorithm == ::clickhouse::grpc::STREAM_GZIP)
-            return GRPC_COMPRESS_STREAM_GZIP;
-        else
-            throw Exception("Unknown compression algorithm: '" + ::clickhouse::grpc::CompressionAlgorithm_Name(algorithm) + "'", ErrorCodes::INVALID_GRPC_QUERY_INFO);
-    }
-
-    grpc_compression_level convertCompressionLevel(const ::clickhouse::grpc::CompressionLevel & level)
-    {
-        if (level == ::clickhouse::grpc::COMPRESSION_NONE)
-            return GRPC_COMPRESS_LEVEL_NONE;
-        else if (level == ::clickhouse::grpc::COMPRESSION_LOW)
-            return GRPC_COMPRESS_LEVEL_LOW;
-        else if (level == ::clickhouse::grpc::COMPRESSION_MEDIUM)
-            return GRPC_COMPRESS_LEVEL_MED;
-        else if (level == ::clickhouse::grpc::COMPRESSION_HIGH)
-            return GRPC_COMPRESS_LEVEL_HIGH;
-        else
-            throw Exception("Unknown compression level: '" + ::clickhouse::grpc::CompressionLevel_Name(level) + "'", ErrorCodes::INVALID_GRPC_QUERY_INFO);
-    }
-
     /// Gets file's contents as a string, throws an exception if failed.
     String readFile(const String & filepath)
     {
@@ -193,6 +138,102 @@ namespace
         return grpc::InsecureServerCredentials();
     }
 
+    /// Transport compression makes gRPC library to compress packed Result messages before sending them through network.
+    struct TransportCompression
+    {
+        grpc_compression_algorithm algorithm;
+        grpc_compression_level level;
+
+        /// Extracts the settings of transport compression from a query info if possible.
+        static std::optional<TransportCompression> fromQueryInfo(const GRPCQueryInfo & query_info)
+        {
+            TransportCompression res;
+            if (!query_info.transport_compression_type().empty())
+            {
+                res.setAlgorithm(query_info.transport_compression_type(), ErrorCodes::INVALID_GRPC_QUERY_INFO);
+                res.setLevel(query_info.transport_compression_level(), ErrorCodes::INVALID_GRPC_QUERY_INFO);
+                return res;
+            }
+
+            if (query_info.has_obsolete_result_compression())
+            {
+                switch (query_info.obsolete_result_compression().algorithm())
+                {
+                    case GRPCObsoleteTransportCompression::NO_COMPRESSION: res.algorithm = GRPC_COMPRESS_NONE; break;
+                    case GRPCObsoleteTransportCompression::DEFLATE: res.algorithm = GRPC_COMPRESS_DEFLATE; break;
+                    case GRPCObsoleteTransportCompression::GZIP: res.algorithm = GRPC_COMPRESS_GZIP; break;
+                    case GRPCObsoleteTransportCompression::STREAM_GZIP: res.algorithm = GRPC_COMPRESS_STREAM_GZIP; break;
+                    default: throw Exception(ErrorCodes::INVALID_GRPC_QUERY_INFO, "Unknown compression algorithm: {}", GRPCObsoleteTransportCompression::CompressionAlgorithm_Name(query_info.obsolete_result_compression().algorithm()));
+                }
+
+                switch (query_info.obsolete_result_compression().level())
+                {
+                    case GRPCObsoleteTransportCompression::COMPRESSION_NONE: res.level = GRPC_COMPRESS_LEVEL_NONE; break;
+                    case GRPCObsoleteTransportCompression::COMPRESSION_LOW: res.level = GRPC_COMPRESS_LEVEL_LOW; break;
+                    case GRPCObsoleteTransportCompression::COMPRESSION_MEDIUM: res.level = GRPC_COMPRESS_LEVEL_MED; break;
+                    case GRPCObsoleteTransportCompression::COMPRESSION_HIGH: res.level = GRPC_COMPRESS_LEVEL_HIGH; break;
+                    default: throw Exception(ErrorCodes::INVALID_GRPC_QUERY_INFO, "Unknown compression level: {}", GRPCObsoleteTransportCompression::CompressionLevel_Name(query_info.obsolete_result_compression().level()));
+                }
+                return res;
+            }
+
+            return std::nullopt;
+        }
+
+        /// Extracts the settings of transport compression from the server configuration.
+        static TransportCompression fromConfiguration(const Poco::Util::AbstractConfiguration & config)
+        {
+            TransportCompression res;
+            if (config.has("grpc.transport_compression_type"))
+            {
+                res.setAlgorithm(config.getString("grpc.transport_compression_type"), ErrorCodes::INVALID_CONFIG_PARAMETER);
+                res.setLevel(config.getInt("grpc.transport_compression_level", 0), ErrorCodes::INVALID_CONFIG_PARAMETER);
+            }
+            else
+            {
+                res.setAlgorithm(config.getString("grpc.compression", "none"), ErrorCodes::INVALID_CONFIG_PARAMETER);
+                res.setLevel(config.getString("grpc.compression_level", "none"), ErrorCodes::INVALID_CONFIG_PARAMETER);
+            }
+            return res;
+        }
+
+    private:
+        void setAlgorithm(const String & str, int error_code)
+        {
+            if (str == "none")
+                algorithm = GRPC_COMPRESS_NONE;
+            else if (str == "deflate")
+                algorithm = GRPC_COMPRESS_DEFLATE;
+            else if (str == "gzip")
+                algorithm = GRPC_COMPRESS_GZIP;
+            else if (str == "stream_gzip")
+                algorithm = GRPC_COMPRESS_STREAM_GZIP;
+            else
+                throw Exception(error_code, "Unknown compression algorithm: '{}'", str);
+        }
+
+        void setLevel(const String & str, int error_code)
+        {
+            if (str == "none")
+                level = GRPC_COMPRESS_LEVEL_NONE;
+            else if (str == "low")
+                level = GRPC_COMPRESS_LEVEL_LOW;
+            else if (str == "medium")
+                level = GRPC_COMPRESS_LEVEL_MED;
+            else if (str == "high")
+                level = GRPC_COMPRESS_LEVEL_HIGH;
+            else
+                throw Exception(error_code, "Unknown compression level: '{}'", str);
+        }
+
+        void setLevel(int level_, int error_code)
+        {
+            if (0 <= level_ && level_ < GRPC_COMPRESS_LEVEL_COUNT)
+                level = static_cast<grpc_compression_level>(level_);
+            else
+                throw Exception(error_code, "Compression level {} is out of range 0..{}", level_, GRPC_COMPRESS_LEVEL_COUNT - 1);
+        }
+    };
 
     /// Gets session's timeout from query info or from the server config.
     std::chrono::steady_clock::duration getSessionTimeout(const GRPCQueryInfo & query_info, const Poco::Util::AbstractConfiguration & config)
@@ -293,15 +334,10 @@ namespace
             return std::nullopt;
         }
 
-        void setResultCompression(grpc_compression_algorithm algorithm, grpc_compression_level level)
+        void setTransportCompression(const TransportCompression & transport_compression)
         {
-            grpc_context.set_compression_algorithm(algorithm);
+            grpc_context.set_compression_algorithm(transport_compression.algorithm);
-            grpc_context.set_compression_level(level);
+            grpc_context.set_compression_level(transport_compression.level);
-        }
-
-        void setResultCompression(const ::clickhouse::grpc::Compression & compression)
-        {
-            setResultCompression(convertCompressionAlgorithm(compression.algorithm()), convertCompressionLevel(compression.level()));
         }
 
     protected:
@@ -628,10 +664,11 @@ namespace
         ASTInsertQuery * insert_query = nullptr;
         String input_format;
         String input_data_delimiter;
+        CompressionMethod input_compression_method = CompressionMethod::None;
         PODArray<char> output;
         String output_format;
-        CompressionMethod compression_method = CompressionMethod::None;
+        CompressionMethod output_compression_method = CompressionMethod::None;
-        int compression_level = 0;
+        int output_compression_level = 0;
 
         uint64_t interactive_delay = 100000;
         bool send_exception_with_stacktrace = true;
@@ -815,9 +852,9 @@ namespace
         if (!query_info.database().empty())
             query_context->setCurrentDatabase(query_info.database());
 
-        /// Apply compression settings for this call.
+        /// Apply transport compression for this call.
-        if (query_info.has_result_compression())
+        if (auto transport_compression = TransportCompression::fromQueryInfo(query_info))
-            responder->setResultCompression(query_info.result_compression());
+            responder->setTransportCompression(*transport_compression);
 
         /// The interactive delay will be used to show progress.
         interactive_delay = settings.interactive_delay;
@@ -852,8 +889,16 @@ namespace
             output_format = query_context->getDefaultFormat();
 
         /// Choose compression.
-        compression_method = chooseCompressionMethod("", query_info.compression_type());
+        String input_compression_method_str = query_info.input_compression_type();
-        compression_level = query_info.compression_level();
+        if (input_compression_method_str.empty())
+            input_compression_method_str = query_info.obsolete_compression_type();
+        input_compression_method = chooseCompressionMethod("", input_compression_method_str);
+
+        String output_compression_method_str = query_info.output_compression_type();
+        if (output_compression_method_str.empty())
+            output_compression_method_str = query_info.obsolete_compression_type();
+        output_compression_method = chooseCompressionMethod("", output_compression_method_str);
+        output_compression_level = query_info.output_compression_level();
 
         /// Set callback to create and fill external tables
         query_context->setExternalTablesInitializer([this] (ContextPtr context)
@@ -984,7 +1029,7 @@ namespace
             return {nullptr, 0}; /// no more input data
         });
 
-        read_buffer = wrapReadBufferWithCompressionMethod(std::move(read_buffer), compression_method);
+        read_buffer = wrapReadBufferWithCompressionMethod(std::move(read_buffer), input_compression_method);
 
         assert(!pipeline);
         auto source = query_context->getInputFormat(
@@ -1112,13 +1157,13 @@ namespace
         if (io.pipeline.pulling())
             header = io.pipeline.getHeader();
 
-        if (compression_method != CompressionMethod::None)
+        if (output_compression_method != CompressionMethod::None)
             output.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data.
         write_buffer = std::make_unique<WriteBufferFromVector<PODArray<char>>>(output);
         nested_write_buffer = static_cast<WriteBufferFromVector<PODArray<char>> *>(write_buffer.get());
-        if (compression_method != CompressionMethod::None)
+        if (output_compression_method != CompressionMethod::None)
         {
-            write_buffer = wrapWriteBufferWithCompressionMethod(std::move(write_buffer), compression_method, compression_level);
+            write_buffer = wrapWriteBufferWithCompressionMethod(std::move(write_buffer), output_compression_method, output_compression_level);
             compressing_write_buffer = write_buffer.get();
         }
 
@@ -1414,10 +1459,10 @@ namespace
             return;
 
         PODArray<char> memory;
-        if (compression_method != CompressionMethod::None)
+        if (output_compression_method != CompressionMethod::None)
             memory.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data.
         std::unique_ptr<WriteBuffer> buf = std::make_unique<WriteBufferFromVector<PODArray<char>>>(memory);
-        buf = wrapWriteBufferWithCompressionMethod(std::move(buf), compression_method, compression_level);
+        buf = wrapWriteBufferWithCompressionMethod(std::move(buf), output_compression_method, output_compression_level);
         auto format = query_context->getOutputFormat(output_format, *buf, totals);
         format->write(materializeBlock(totals));
         format->finalize();
@@ -1432,10 +1477,10 @@ namespace
             return;
 
         PODArray<char> memory;
-        if (compression_method != CompressionMethod::None)
+        if (output_compression_method != CompressionMethod::None)
             memory.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data.
         std::unique_ptr<WriteBuffer> buf = std::make_unique<WriteBufferFromVector<PODArray<char>>>(memory);
-        buf = wrapWriteBufferWithCompressionMethod(std::move(buf), compression_method, compression_level);
+        buf = wrapWriteBufferWithCompressionMethod(std::move(buf), output_compression_method, output_compression_level);
         auto format = query_context->getOutputFormat(output_format, *buf, extremes);
         format->write(materializeBlock(extremes));
         format->finalize();
@@ -1772,8 +1817,9 @@ void GRPCServer::start()
     builder.RegisterService(&grpc_service);
     builder.SetMaxSendMessageSize(iserver.config().getInt("grpc.max_send_message_size", -1));
     builder.SetMaxReceiveMessageSize(iserver.config().getInt("grpc.max_receive_message_size", -1));
-    builder.SetDefaultCompressionAlgorithm(parseCompressionAlgorithm(iserver.config().getString("grpc.compression", "none")));
+    auto default_transport_compression = TransportCompression::fromConfiguration(iserver.config());
-    builder.SetDefaultCompressionLevel(parseCompressionLevel(iserver.config().getString("grpc.compression_level", "none")));
+    builder.SetDefaultCompressionAlgorithm(default_transport_compression.algorithm);
+    builder.SetDefaultCompressionLevel(default_transport_compression.level);
 
     queue = builder.AddCompletionQueue();
     grpc_server = builder.BuildAndStart();

src/Server/HTTP/HTTPServerConnection.cpp

@@ -36,7 +36,7 @@ void HTTPServerConnection::run()
 
                 if (request.isSecure())
                 {
-                    size_t hsts_max_age = context->getSettings().hsts_max_age.value;
+                    size_t hsts_max_age = context->getSettingsRef().hsts_max_age.value;
 
                     if (hsts_max_age > 0)
                         response.add("Strict-Transport-Security", "max-age=" + std::to_string(hsts_max_age));

src/Server/TCPHandler.cpp

@@ -359,6 +359,7 @@ void TCPHandler::runImpl()
                             return true;
 
                         sendProgress();
+                        sendProfileEvents();
                         sendLogs();
 
                         return false;

src/Server/grpc_protos/clickhouse_grpc.proto

@@ -45,21 +45,19 @@ message ExternalTable {
    map<string, string> settings = 5;
 }
 
-enum CompressionAlgorithm {
+message ObsoleteTransportCompression {
+   enum CompressionAlgorithm {
       NO_COMPRESSION = 0;
       DEFLATE = 1;
       GZIP = 2;
       STREAM_GZIP = 3;
-}
+   }
-
+   enum CompressionLevel {
-enum CompressionLevel {
       COMPRESSION_NONE = 0;
       COMPRESSION_LOW = 1;
       COMPRESSION_MEDIUM = 2;
       COMPRESSION_HIGH = 3;
-}
+   }
-
-message Compression {
    CompressionAlgorithm algorithm = 1;
    CompressionLevel level = 2;
 }
@@ -102,16 +100,16 @@ message QueryInfo {
    // `next_query_info` is allowed to be set only if a method with streaming input (i.e. ExecuteQueryWithStreamInput() or ExecuteQueryWithStreamIO()) is used.
    bool next_query_info = 16;
 
-   /// Controls how a ClickHouse server will compress query execution results before sending back to the client.
+   // Compression type for `input_data`.
-   /// If not set the compression settings from the configuration file will be used.
-   Compression result_compression = 17;
-
-   // Compression type for `input_data`, `output_data`, `totals` and `extremes`.
    // Supported compression types: none, gzip(gz), deflate, brotli(br), lzma(xz), zstd(zst), lz4, bz2.
-   // When used for `input_data` the client is responsible to compress data before putting it into `input_data`.
+   // The client is responsible to compress data before putting it into `input_data`.
-   // When used for `output_data` or `totals` or `extremes` the client receives compressed data and should decompress it by itself.
+   string input_compression_type = 20;
-   // In the latter case consider to specify also `compression_level`.
+
-   string compression_type = 18;
+   // Compression type for `output_data`, `totals` and `extremes`.
+   // Supported compression types: none, gzip(gz), deflate, brotli(br), lzma(xz), zstd(zst), lz4, bz2.
+   // The client receives compressed data and should decompress it by itself.
+   // Consider also setting `output_compression_level`.
+   string output_compression_type = 21;
 
    // Compression level.
    // WARNING: If it's not specified the compression level is set to zero by default which might be not the best choice for some compression types (see below).
@@ -123,7 +121,23 @@ message QueryInfo {
    // zstd: 1..22; 3 is recommended by default (compression level 0 also means 3)
    // lz4: 0..16; values < 0 mean fast acceleration
    // bz2: 1..9
-   int32 compression_level = 19;
+   int32 output_compression_level = 19;
+
+   // Transport compression is an alternative way to make the server to compress its response.
+   // This kind of compression implies that instead of compressing just `output` the server will compress whole packed messages of the `Result` type,
+   // and then gRPC implementation on client side will decompress those messages so client code won't be bothered with decompression.
+   // Here is a big difference between the transport compression and the compression enabled by setting `output_compression_type` because
+   // in case of the transport compression the client code receives already decompressed data in `output`.
+   // If the transport compression is not set here it can still be enabled by the server configuration.
+   // Supported compression types: none, deflate, gzip, stream_gzip
+   // Supported compression levels: 0..3
+   // WARNING: Don't set `transport_compression` and `output_compression` at the same time because it will make the server to compress its output twice!
+   string transport_compression_type = 22;
+   int32 transport_compression_level = 23;
+
+   /// Obsolete fields, should not be used in new code.
+   ObsoleteTransportCompression obsolete_result_compression = 17;
+   string obsolete_compression_type = 18;
 }
 
 enum LogsLevel {

src/Storages/Distributed/DirectoryMonitor.cpp

@@ -132,6 +132,7 @@ namespace
 
     struct DistributedHeader
     {
+        UInt64 revision = 0;
         Settings insert_settings;
         std::string insert_query;
         ClientInfo client_info;
@@ -166,9 +167,8 @@ namespace
             /// Read the parts of the header.
             ReadBufferFromString header_buf(header_data);
 
-            UInt64 initiator_revision;
+            readVarUInt(distributed_header.revision, header_buf);
-            readVarUInt(initiator_revision, header_buf);
+            if (DBMS_TCP_PROTOCOL_VERSION < distributed_header.revision)
-            if (DBMS_TCP_PROTOCOL_VERSION < initiator_revision)
             {
                 LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features.");
             }
@@ -177,7 +177,7 @@ namespace
             distributed_header.insert_settings.read(header_buf);
 
             if (header_buf.hasPendingData())
-                distributed_header.client_info.read(header_buf, initiator_revision);
+                distributed_header.client_info.read(header_buf, distributed_header.revision);
 
             if (header_buf.hasPendingData())
             {
@@ -188,10 +188,12 @@ namespace
 
             if (header_buf.hasPendingData())
             {
-                NativeReader header_block_in(header_buf, DBMS_TCP_PROTOCOL_VERSION);
+                NativeReader header_block_in(header_buf, distributed_header.revision);
                 distributed_header.block_header = header_block_in.read();
                 if (!distributed_header.block_header)
-                    throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read header from the {} batch", in.getFileName());
+                    throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA,
+                        "Cannot read header from the {} batch. Data was written with protocol version {}, current version: {}",
+                            in.getFileName(), distributed_header.revision, DBMS_TCP_PROTOCOL_VERSION);
             }
 
             /// Add handling new data here, for example:
@@ -264,10 +266,10 @@ namespace
         return nullptr;
     }
 
-    void writeAndConvert(RemoteInserter & remote, ReadBufferFromFile & in)
+    void writeAndConvert(RemoteInserter & remote, const DistributedHeader & distributed_header, ReadBufferFromFile & in)
     {
         CompressedReadBuffer decompressing_in(in);
-        NativeReader block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION);
+        NativeReader block_in(decompressing_in, distributed_header.revision);
 
         while (Block block = block_in.read())
         {
@@ -304,7 +306,7 @@ namespace
         {
             LOG_TRACE(log, "Processing batch {} with old format (no header)", in.getFileName());
 
-            writeAndConvert(remote, in);
+            writeAndConvert(remote, distributed_header, in);
             return;
         }
 
@@ -314,14 +316,20 @@ namespace
                 "Structure does not match (remote: {}, local: {}), implicit conversion will be done",
                 remote.getHeader().dumpStructure(), distributed_header.block_header.dumpStructure());
 
-            writeAndConvert(remote, in);
+            writeAndConvert(remote, distributed_header, in);
             return;
         }
 
         /// If connection does not use compression, we have to uncompress the data.
         if (!compression_expected)
         {
-            writeAndConvert(remote, in);
+            writeAndConvert(remote, distributed_header, in);
+            return;
+        }
+
+        if (distributed_header.revision != remote.getServerRevision())
+        {
+            writeAndConvert(remote, distributed_header, in);
             return;
         }
 
@@ -915,10 +923,10 @@ public:
         {
             in = std::make_unique<ReadBufferFromFile>(file_name);
             decompressing_in = std::make_unique<CompressedReadBuffer>(*in);
-            block_in = std::make_unique<NativeReader>(*decompressing_in, DBMS_TCP_PROTOCOL_VERSION);
             log = &Poco::Logger::get("DirectoryMonitorSource");
 
-            readDistributedHeader(*in, log);
+            auto distributed_header = readDistributedHeader(*in, log);
+            block_in = std::make_unique<NativeReader>(*decompressing_in, distributed_header.revision);
 
             first_block = block_in->read();
         }
@@ -1040,7 +1048,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
                 LOG_DEBUG(log, "Processing batch {} with old format (no header/rows)", in.getFileName());
 
                 CompressedReadBuffer decompressing_in(in);
-                NativeReader block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION);
+                NativeReader block_in(decompressing_in, distributed_header.revision);
 
                 while (Block block = block_in.read())
                 {

src/Storages/HDFS/StorageHDFS.cpp

@@ -372,8 +372,10 @@ String HDFSSource::getName() const
 
 Chunk HDFSSource::generate()
 {
-    if (!reader)
+    while (true)
-        return {};
+    {
+        if (!reader || isCancelled())
+            break;
 
         Chunk chunk;
         if (reader->pull(chunk))
@@ -407,9 +409,10 @@ Chunk HDFSSource::generate()
             read_buf.reset();
 
             if (!initialize())
-            return {};
+                break;
         }
-    return generate();
+    }
+    return {};
 }
 
 

src/Storages/HDFS/StorageHDFSCluster.cpp

@@ -69,7 +69,7 @@ Pipe StorageHDFSCluster::read(
     size_t /*max_block_size*/,
     unsigned /*num_streams*/)
 {
-    auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettings());
+    auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettingsRef());
 
     auto iterator = std::make_shared<HDFSSource::DisclosedGlobIterator>(context, uri);
     auto callback = std::make_shared<HDFSSource::IteratorWrapper>([iterator]() mutable -> String

src/Storages/MergeTree/KeyCondition.cpp

@@ -1292,8 +1292,8 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, ContextPtr context,
                 key_expr_type_not_null = key_expr_type;
 
             bool cast_not_needed = is_set_const /// Set args are already casted inside Set::createFromAST
-                || ((isNativeNumber(key_expr_type_not_null) || isDateTime(key_expr_type_not_null))
+                || ((isNativeInteger(key_expr_type_not_null) || isDateTime(key_expr_type_not_null))
-                    && (isNativeNumber(const_type) || isDateTime(const_type))); /// Numbers and DateTime are accurately compared without cast.
+                    && (isNativeInteger(const_type) || isDateTime(const_type))); /// Native integers and DateTime are accurately compared without cast.
 
             if (!cast_not_needed && !key_expr_type_not_null->equals(*const_type))
             {

src/Storages/PartitionedSink.cpp

@@ -1,5 +1,7 @@
 #include "PartitionedSink.h"
 
+#include <Common/ArenaUtils.h>
+
 #include <Functions/FunctionsConversion.h>
 
 #include <Interpreters/Context.h>
@@ -40,19 +42,18 @@ PartitionedSink::PartitionedSink(
 }
 
 
-SinkPtr PartitionedSink::getSinkForPartition(const String & partition_id)
+SinkPtr PartitionedSink::getSinkForPartitionKey(StringRef partition_key)
 {
-    auto it = sinks.find(partition_id);
+    auto it = partition_id_to_sink.find(partition_key);
-    if (it == sinks.end())
+    if (it == partition_id_to_sink.end())
     {
-        auto sink = createSinkForPartition(partition_id);
+        auto sink = createSinkForPartition(partition_key.toString());
-        std::tie(it, std::ignore) = sinks.emplace(partition_id, sink);
+        std::tie(it, std::ignore) = partition_id_to_sink.emplace(partition_key, sink);
     }
 
     return it->second;
 }
 
-
 void PartitionedSink::consume(Chunk chunk)
 {
     const auto & columns = chunk.getColumns();
@@ -61,45 +62,59 @@ void PartitionedSink::consume(Chunk chunk)
     block_with_partition_by_expr.setColumns(columns);
     partition_by_expr->execute(block_with_partition_by_expr);
 
-    const auto * column = block_with_partition_by_expr.getByName(partition_by_column_name).column.get();
+    const auto * partition_by_result_column = block_with_partition_by_expr.getByName(partition_by_column_name).column.get();
 
-    std::unordered_map<String, size_t> sub_chunks_indices;
+    size_t chunk_rows = chunk.getNumRows();
-    IColumn::Selector selector;
+    chunk_row_index_to_partition_index.resize(chunk_rows);
-    for (size_t row = 0; row < chunk.getNumRows(); ++row)
+
+    partition_id_to_chunk_index.clear();
+
+    for (size_t row = 0; row < chunk_rows; ++row)
     {
-        auto value = column->getDataAt(row);
+        auto partition_key = partition_by_result_column->getDataAt(row);
-        auto [it, inserted] = sub_chunks_indices.emplace(value, sub_chunks_indices.size());
+        auto [it, inserted] = partition_id_to_chunk_index.insert(makePairNoInit(partition_key, partition_id_to_chunk_index.size()));
-        selector.push_back(it->second);
+        if (inserted)
+            it->value.first = copyStringInArena(partition_keys_arena, partition_key);
+
+        chunk_row_index_to_partition_index[row] = it->getMapped();
     }
 
-    Chunks sub_chunks;
+    size_t columns_size = columns.size();
-    sub_chunks.reserve(sub_chunks_indices.size());
+    size_t partitions_size = partition_id_to_chunk_index.size();
-    for (size_t column_index = 0; column_index < columns.size(); ++column_index)
+
+    Chunks partition_index_to_chunk;
+    partition_index_to_chunk.reserve(partitions_size);
+
+    for (size_t column_index = 0; column_index < columns_size; ++column_index)
     {
-        MutableColumns column_sub_chunks = columns[column_index]->scatter(sub_chunks_indices.size(), selector);
+        MutableColumns partition_index_to_column_split = columns[column_index]->scatter(partitions_size, chunk_row_index_to_partition_index);
-        if (column_index == 0) /// Set sizes for sub-chunks.
+
+        /// Add chunks into partition_index_to_chunk with sizes of result columns
+        if (column_index == 0)
         {
-            for (const auto & column_sub_chunk : column_sub_chunks)
+            for (const auto & partition_column : partition_index_to_column_split)
             {
-                sub_chunks.emplace_back(Columns(), column_sub_chunk->size());
+                partition_index_to_chunk.emplace_back(Columns(), partition_column->size());
-            }
-        }
-        for (size_t sub_chunk_index = 0; sub_chunk_index < column_sub_chunks.size(); ++sub_chunk_index)
-        {
-            sub_chunks[sub_chunk_index].addColumn(std::move(column_sub_chunks[sub_chunk_index]));
             }
         }
 
-    for (const auto & [partition_id, sub_chunk_index] : sub_chunks_indices)
+        for (size_t partition_index = 0; partition_index  < partitions_size; ++partition_index)
         {
-        getSinkForPartition(partition_id)->consume(std::move(sub_chunks[sub_chunk_index]));
+            partition_index_to_chunk[partition_index].addColumn(std::move(partition_index_to_column_split[partition_index]));
+        }
+    }
+
+    for (const auto & [partition_key, partition_index] : partition_id_to_chunk_index)
+    {
+        auto sink = getSinkForPartitionKey(partition_key);
+        sink->consume(std::move(partition_index_to_chunk[partition_index]));
     }
 }
 
 
 void PartitionedSink::onFinish()
 {
-    for (auto & [partition_id, sink] : sinks)
+    for (auto & [_, sink] : partition_id_to_sink)
     {
         sink->onFinish();
     }

src/Storages/PartitionedSink.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <Common/HashTable/HashMap.h>
+#include <Common/Arena.h>
+#include <absl/container/flat_hash_map.h>
 #include <Processors/Sinks/SinkToStorage.h>
 #include <Interpreters/ExpressionAnalyzer.h>
 #include <Interpreters/Context_fwd.h>
@@ -34,9 +37,13 @@ private:
     ExpressionActionsPtr partition_by_expr;
     String partition_by_column_name;
 
-    std::unordered_map<String, SinkPtr> sinks;
+    absl::flat_hash_map<StringRef, SinkPtr> partition_id_to_sink;
+    HashMapWithSavedHash<StringRef, size_t> partition_id_to_chunk_index;
+    IColumn::Selector chunk_row_index_to_partition_index;
+    Arena partition_keys_arena;
+
+    SinkPtr getSinkForPartitionKey(StringRef partition_key);
 
-    SinkPtr getSinkForPartition(const String & partition_id);
 };
 
 }

src/Storages/StorageFactory.cpp

@@ -66,6 +66,7 @@ StoragePtr StorageFactory::get(
     bool has_force_restore_data_flag) const
 {
     String name, comment;
+
     ASTStorage * storage_def = query.storage;
 
     bool has_engine_args = false;
@@ -107,7 +108,10 @@ StoragePtr StorageFactory::get(
         }
         else
         {
-            if (!storage_def)
+            if (!query.storage)
+                throw Exception("Incorrect CREATE query: storage required", ErrorCodes::INCORRECT_QUERY);
+
+            if (!storage_def->engine)
                 throw Exception("Incorrect CREATE query: ENGINE required", ErrorCodes::ENGINE_REQUIRED);
 
             const ASTFunction & engine_def = *storage_def->engine;

src/Storages/StorageFile.cpp

@@ -879,9 +879,10 @@ SinkToStoragePtr StorageFile::write(
             path = paths.back();
             fs::create_directories(fs::path(path).parent_path());
 
+            std::error_code error_code;
             if (!context->getSettingsRef().engine_file_truncate_on_insert && !is_path_with_globs
                 && !FormatFactory::instance().checkIfFormatSupportAppend(format_name, context, format_settings) && fs::exists(paths.back())
-                && fs::file_size(paths.back()) != 0)
+                && fs::file_size(paths.back(), error_code) != 0 && !error_code)
             {
                 if (context->getSettingsRef().engine_file_allow_create_multiple_files)
                 {

src/Storages/StorageS3.cpp

@@ -302,8 +302,10 @@ String StorageS3Source::getName() const
 
 Chunk StorageS3Source::generate()
 {
-    if (!reader)
+    while (true)
-        return {};
+    {
+        if (!reader || isCancelled())
+            break;
 
         Chunk chunk;
         if (reader->pull(chunk))
@@ -332,10 +334,10 @@ Chunk StorageS3Source::generate()
             read_buf.reset();
 
             if (!initialize())
-            return {};
+                break;
         }
-
+    }
-    return generate();
+    return {};
 }
 
 static bool checkIfObjectExists(const std::shared_ptr<Aws::S3::S3Client> & client, const String & bucket, const String & key)
@@ -385,13 +387,18 @@ public:
         const String & bucket,
         const String & key,
         size_t min_upload_part_size,
+        size_t upload_part_size_multiply_factor,
+        size_t upload_part_size_multiply_parts_count_threshold,
         size_t max_single_part_upload_size)
         : SinkToStorage(sample_block_)
         , sample_block(sample_block_)
         , format_settings(format_settings_)
     {
         write_buf = wrapWriteBufferWithCompressionMethod(
-            std::make_unique<WriteBufferFromS3>(client, bucket, key, min_upload_part_size, max_single_part_upload_size), compression_method, 3);
+            std::make_unique<WriteBufferFromS3>(
+                client, bucket, key, min_upload_part_size,
+                upload_part_size_multiply_factor, upload_part_size_multiply_parts_count_threshold,
+                max_single_part_upload_size), compression_method, 3);
         writer = FormatFactory::instance().getOutputFormatParallelIfPossible(format, *write_buf, sample_block, context, {}, format_settings);
     }
 
@@ -440,6 +447,8 @@ public:
         const String & bucket_,
         const String & key_,
         size_t min_upload_part_size_,
+        size_t upload_part_size_multiply_factor_,
+        size_t upload_part_size_multiply_parts_count_threshold_,
         size_t max_single_part_upload_size_)
         : PartitionedSink(partition_by, context_, sample_block_)
         , format(format_)
@@ -450,6 +459,8 @@ public:
         , bucket(bucket_)
         , key(key_)
         , min_upload_part_size(min_upload_part_size_)
+        , upload_part_size_multiply_factor(upload_part_size_multiply_factor_)
+        , upload_part_size_multiply_parts_count_threshold(upload_part_size_multiply_parts_count_threshold_)
         , max_single_part_upload_size(max_single_part_upload_size_)
         , format_settings(format_settings_)
     {
@@ -473,6 +484,8 @@ public:
             partition_bucket,
             partition_key,
             min_upload_part_size,
+            upload_part_size_multiply_factor,
+            upload_part_size_multiply_parts_count_threshold,
             max_single_part_upload_size
         );
     }
@@ -487,6 +500,8 @@ private:
     const String bucket;
     const String key;
     size_t min_upload_part_size;
+    size_t upload_part_size_multiply_factor;
+    size_t upload_part_size_multiply_parts_count_threshold;
     size_t max_single_part_upload_size;
     std::optional<FormatSettings> format_settings;
 
@@ -527,6 +542,8 @@ StorageS3::StorageS3(
     const String & format_name_,
     UInt64 max_single_read_retries_,
     UInt64 min_upload_part_size_,
+    UInt64 upload_part_size_multiply_factor_,
+    UInt64 upload_part_size_multiply_parts_count_threshold_,
     UInt64 max_single_part_upload_size_,
     UInt64 max_connections_,
     const ColumnsDescription & columns_,
@@ -543,6 +560,8 @@ StorageS3::StorageS3(
     , format_name(format_name_)
     , max_single_read_retries(max_single_read_retries_)
     , min_upload_part_size(min_upload_part_size_)
+    , upload_part_size_multiply_factor(upload_part_size_multiply_factor_)
+    , upload_part_size_multiply_parts_count_threshold(upload_part_size_multiply_parts_count_threshold_)
     , max_single_part_upload_size(max_single_part_upload_size_)
     , compression_method(compression_method_)
     , name(uri_.storage_name)
@@ -669,6 +688,8 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr
             client_auth.uri.bucket,
             keys.back(),
             min_upload_part_size,
+            upload_part_size_multiply_factor,
+            upload_part_size_multiply_parts_count_threshold,
             max_single_part_upload_size);
     }
     else
@@ -712,6 +733,8 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr
             client_auth.uri.bucket,
             keys.back(),
             min_upload_part_size,
+            upload_part_size_multiply_factor,
+            upload_part_size_multiply_parts_count_threshold,
             max_single_part_upload_size);
     }
 }
@@ -923,7 +946,10 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory)
         S3::URI s3_uri(Poco::URI(configuration.url));
         auto max_single_read_retries = args.getLocalContext()->getSettingsRef().s3_max_single_read_retries;
         auto min_upload_part_size = args.getLocalContext()->getSettingsRef().s3_min_upload_part_size;
+        auto upload_part_size_multiply_factor = args.getLocalContext()->getSettingsRef().s3_upload_part_size_multiply_factor;
+        auto upload_part_size_multiply_parts_count_threshold = args.getLocalContext()->getSettingsRef().s3_upload_part_size_multiply_parts_count_threshold;
         auto max_single_part_upload_size = args.getLocalContext()->getSettingsRef().s3_max_single_part_upload_size;
+
         auto max_connections = args.getLocalContext()->getSettingsRef().s3_max_connections;
 
         ASTPtr partition_by;
@@ -938,6 +964,8 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory)
             configuration.format,
             max_single_read_retries,
             min_upload_part_size,
+            upload_part_size_multiply_factor,
+            upload_part_size_multiply_parts_count_threshold,
             max_single_part_upload_size,
             max_connections,
             args.columns,

src/Storages/StorageS3.h

@@ -126,6 +126,8 @@ public:
         const String & format_name_,
         UInt64 max_single_read_retries_,
         UInt64 min_upload_part_size_,
+        UInt64 upload_part_size_multiply_factor_,
+        UInt64 upload_part_size_multiply_parts_count_threshold_,
         UInt64 max_single_part_upload_size_,
         UInt64 max_connections_,
         const ColumnsDescription & columns_,
@@ -193,6 +195,8 @@ private:
     String format_name;
     UInt64 max_single_read_retries;
     size_t min_upload_part_size;
+    size_t upload_part_size_multiply_factor;
+    size_t upload_part_size_multiply_parts_count_threshold;
     size_t max_single_part_upload_size;
     String compression_method;
     String name;

src/Storages/StorageS3Cluster.cpp

@@ -82,7 +82,7 @@ Pipe StorageS3Cluster::read(
 {
     StorageS3::updateClientAndAuthSettings(context, client_auth);
 
-    auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettings());
+    auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettingsRef());
     StorageS3::updateClientAndAuthSettings(context, client_auth);
 
     auto iterator = std::make_shared<StorageS3Source::DisclosedGlobIterator>(*client_auth.client, client_auth.uri);