Merge branch 'master' into keeperVersion

.clang-format

@@ -12,6 +12,7 @@ BraceWrapping:
     AfterUnion: true
     BeforeCatch: true
     BeforeElse: true
+    BeforeLambdaBody: true
     IndentBraces: false
 BreakConstructorInitializersBeforeComma: false
 Cpp11BracedListStyle: true

.github/workflows/nightly.yml

@@ -0,0 +1,73 @@
+name: NightlyBuilds
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+"on":
+  schedule:
+    - cron: '13 3 * * *'
+
+jobs:
+  DockerHubPushAarch64:
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix aarch64 --all
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
+  DockerHubPushAmd64:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix amd64 --all
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
+  DockerHubPush:
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download changed aarch64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}
+      - name: Download changed amd64 images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/changed_images.json

.potato.yml

@@ -1,27 +0,0 @@
-# This is the configuration file with settings for Potato.
-# Potato is an internal Yandex technology that allows us to sync internal [Yandex.Tracker](https://yandex.com/tracker/) and GitHub.
-
-# For all PRs where documentation is needed, just add a 'pr-feature' label and we will include it into documentation sprints.
-
-# The project name.
-name: clickhouse
-# Object handlers defines which handlers we use.
-handlers:
-  # The handler for creating an Yandex.Tracker issue.
-  - name: issue-create
-    params:
-      triggers:
-        # The trigger for creating the Yandex.Tracker issue. When the specified event occurs, it transfers PR data to Yandex.Tracker.
-        github:pullRequest:labeled:
-          data:
-          # The Yandex.Tracker queue to create the issue in. Each issue in Tracker belongs to one of the project queues.
-          queue: CLICKHOUSEDOCS
-          # The issue title.
-          summary: '[Potato] Pull Request #{{pullRequest.number}}'
-          # The issue description.
-          description: >
-                {{pullRequest.description}}
-
-                Ссылка на Pull Request: {{pullRequest.webUrl}}
-          # The condition for creating the Yandex.Tracker issue.
-          condition: eventPayload.labels.filter(label => ['pr-feature'].includes(label.name)).length

base/base/LineReader.cpp

@@ -109,10 +109,10 @@ void LineReader::Suggest::addWords(Words && new_words)
         std::lock_guard lock(mutex);
         addNewWords(words, new_words, std::less<std::string>{});
         addNewWords(words_no_case, new_words_no_case, NoCaseCompare{});
-    }
 
-    assert(std::is_sorted(words.begin(), words.end()));
+        assert(std::is_sorted(words.begin(), words.end()));
-    assert(std::is_sorted(words_no_case.begin(), words_no_case.end(), NoCaseCompare{}));
+        assert(std::is_sorted(words_no_case.begin(), words_no_case.end(), NoCaseCompare{}));
+    }
 }
 
 LineReader::LineReader(const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_)

base/base/sort.h

@@ -2,6 +2,63 @@
 
 #include <pdqsort.h>
 
+#ifndef NDEBUG
+
+#include <pcg_random.hpp>
+#include <base/getThreadId.h>
+
+/** Same as libcxx std::__debug_less. Just without dependency on private part of standard library.
+  * Check that Comparator induce strict weak ordering.
+  */
+template <typename Comparator>
+class DebugLessComparator
+{
+public:
+    constexpr DebugLessComparator(Comparator & cmp_)
+        : cmp(cmp_)
+    {}
+
+    template <typename LhsType, typename RhsType>
+    constexpr bool operator()(const LhsType & lhs, const RhsType & rhs)
+    {
+        bool lhs_less_than_rhs = cmp(lhs, rhs);
+        if (lhs_less_than_rhs)
+            assert(!cmp(rhs, lhs));
+
+        return lhs_less_than_rhs;
+    }
+
+    template <typename LhsType, typename RhsType>
+    constexpr bool operator()(LhsType & lhs, RhsType & rhs)
+    {
+        bool lhs_less_than_rhs = cmp(lhs, rhs);
+        if (lhs_less_than_rhs)
+            assert(!cmp(rhs, lhs));
+
+        return lhs_less_than_rhs;
+    }
+
+private:
+    Comparator & cmp;
+};
+
+template <typename Comparator>
+using ComparatorWrapper = DebugLessComparator<Comparator>;
+
+template <typename RandomIt>
+void shuffle(RandomIt first, RandomIt last)
+{
+    static thread_local pcg64 rng(getThreadId());
+    std::shuffle(first, last, rng);
+}
+
+#else
+
+template <typename Comparator>
+using ComparatorWrapper = Comparator;
+
+#endif
+
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wold-style-cast"
 
@@ -10,19 +67,48 @@
 template <typename RandomIt>
 void nth_element(RandomIt first, RandomIt nth, RandomIt last)
 {
-    ::miniselect::floyd_rivest_select(first, nth, last);
+    using value_type = typename std::iterator_traits<RandomIt>::value_type;
-}
+    using comparator = std::less<value_type>;
 
-template <typename RandomIt>
+    comparator compare;
-void partial_sort(RandomIt first, RandomIt middle, RandomIt last)
+    ComparatorWrapper<comparator> compare_wrapper = compare;
-{
+
-    ::miniselect::floyd_rivest_partial_sort(first, middle, last);
+#ifndef NDEBUG
+    ::shuffle(first, last);
+#endif
+
+    ::miniselect::floyd_rivest_select(first, nth, last, compare_wrapper);
+
+#ifndef NDEBUG
+    ::shuffle(first, nth);
+
+    if (nth != last)
+        ::shuffle(nth + 1, last);
+#endif
 }
 
 template <typename RandomIt, typename Compare>
 void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare)
 {
-    ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare);
+#ifndef NDEBUG
+    ::shuffle(first, last);
+#endif
+
+    ComparatorWrapper<Compare> compare_wrapper = compare;
+    ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare_wrapper);
+
+#ifndef NDEBUG
+    ::shuffle(middle, last);
+#endif
+}
+
+template <typename RandomIt>
+void partial_sort(RandomIt first, RandomIt middle, RandomIt last)
+{
+    using value_type = typename std::iterator_traits<RandomIt>::value_type;
+    using comparator = std::less<value_type>;
+
+    ::partial_sort(first, middle, last, comparator());
 }
 
 #pragma GCC diagnostic pop
@@ -30,7 +116,12 @@ void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compar
 template <typename RandomIt, typename Compare>
 void sort(RandomIt first, RandomIt last, Compare compare)
 {
-    ::pdqsort(first, last, compare);
+#ifndef NDEBUG
+    ::shuffle(first, last);
+#endif
+
+    ComparatorWrapper<Compare> compare_wrapper = compare;
+    ::pdqsort(first, last, compare_wrapper);
 }
 
 template <typename RandomIt>
@@ -38,5 +129,5 @@ void sort(RandomIt first, RandomIt last)
 {
     using value_type = typename std::iterator_traits<RandomIt>::value_type;
     using comparator = std::less<value_type>;
-    ::pdqsort(first, last, comparator());
+    ::sort(first, last, comparator());
 }

contrib/rocksdb-cmake/CMakeLists.txt

@@ -127,11 +127,6 @@ endif()
 
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_definitions(-DOS_MACOSX)
-  if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
-    add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
-    # no debug info for IOS, that will make our library big
-    add_definitions(-DNDEBUG)
-  endif()
 elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
   add_definitions(-DOS_LINUX)
 elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")

docker-compose.yml

@@ -1,15 +0,0 @@
-version: "2"
-
-services:
-  builder:
-    image: clickhouse/clickhouse-builder
-    build: docker/builder
-  client:
-    image: clickhouse/clickhouse-client
-    build: docker/client
-    command: ['--host', 'server']
-  server:
-    image: clickhouse/clickhouse-server
-    build: docker/server
-    ports:
-      - 8123:8123

docker/images.json

@@ -32,6 +32,7 @@
         "dependent": []
     },
     "docker/test/pvs": {
+        "only_amd64": true,
         "name": "clickhouse/pvs-test",
         "dependent": []
     },
@@ -72,6 +73,7 @@
         "dependent": []
     },
     "docker/test/integration/runner": {
+        "only_amd64": true,
         "name": "clickhouse/integration-tests-runner",
         "dependent": []
     },
@@ -124,6 +126,7 @@
         "dependent": []
     },
     "docker/test/integration/kerberos_kdc": {
+        "only_amd64": true,
         "name": "clickhouse/kerberos-kdc",
         "dependent": []
     },
@@ -137,6 +140,7 @@
          ]
     },
     "docker/test/integration/kerberized_hadoop": {
+        "only_amd64": true,
         "name": "clickhouse/kerberized-hadoop",
         "dependent": []
     },

docker/test/fuzzer/run-fuzzer.sh

@@ -185,15 +185,14 @@ handle SIGUSR2 nostop noprint pass
 handle SIG$RTMIN nostop noprint pass
 info signals
 continue
+gcore
 backtrace full
-info locals
+thread apply all backtrace full
 info registers
 disassemble /s
 up
-info locals
 disassemble /s
 up
-info locals
 disassemble /s
 p \"done\"
 detach
@@ -314,6 +313,11 @@ quit
             || echo "Fuzzer failed ($fuzzer_exit_code). See the logs." ; } \
             | tail -1 > description.txt
     fi
+
+    if test -f core.*; then
+        pigz core.*
+        mv core.*.gz core.gz
+    fi
 }
 
 case "$stage" in
@@ -345,6 +349,10 @@ case "$stage" in
     time fuzz
     ;&
 "report")
+CORE_LINK=''
+if [ -f core.gz ]; then
+    CORE_LINK='<a href="core.gz">core.gz</a>'
+fi
 cat > report.html <<EOF ||:
 <!DOCTYPE html>
 <html lang="en">
@@ -386,6 +394,7 @@ th { cursor: pointer; }
 <a href="fuzzer.log">fuzzer.log</a>
 <a href="server.log">server.log</a>
 <a href="main.log">main.log</a>
+${CORE_LINK}
 </p>
 <table>
 <tr><th>Test name</th><th>Test status</th><th>Description</th></tr>

docker/test/integration/kerberized_hadoop/Dockerfile

@@ -15,9 +15,10 @@ RUN curl -o krb5-libs-1.10.3-65.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault.
     rm -fr *.rpm
 
 RUN cd /tmp && \
-	curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \
+    curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \
-	tar xzf commons-daemon-1.0.15-src.tar.gz && \
+    tar xzf commons-daemon-1.0.15-src.tar.gz && \
-	cd commons-daemon-1.0.15-src/src/native/unix && \
+    cd commons-daemon-1.0.15-src/src/native/unix && \
-	./configure && \
+    ./configure && \
-	make && \
+    make && \
-	cp ./jsvc /usr/local/hadoop/sbin
+    cp ./jsvc /usr/local/hadoop-2.7.0/sbin && \
+    [ -e /usr/local/hadoop ] || ln -s ./hadoop-2.7.0 /usr/local/hadoop

docker/test/integration/runner/Dockerfile

@@ -58,9 +58,7 @@ RUN apt-get update \
 
 RUN dockerd --version; docker --version
 
-ARG TARGETARCH
+RUN python3 -m pip install \
-# FIXME: psycopg2-binary is not available for aarch64, we skip it for now
-RUN test x$TARGETARCH = xarm64 || ( python3 -m pip install \
     PyMySQL \
     aerospike==4.0.0 \
     avro==1.10.2 \
@@ -90,7 +88,7 @@ RUN test x$TARGETARCH = xarm64 || ( python3 -m pip install \
     urllib3 \
     requests-kerberos \
     pyhdfs \
-    azure-storage-blob )
+    azure-storage-blob
 
 COPY modprobe.sh /usr/local/bin/modprobe
 COPY dockerd-entrypoint.sh /usr/local/bin/

docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml

@@ -4,7 +4,7 @@ services:
   kerberizedhdfs1:
     cap_add:
       - DAC_READ_SEARCH
-    image: clickhouse/kerberized-hadoop
+    image: clickhouse/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest}
     hostname: kerberizedhdfs1
     restart: always
     volumes:

docker/test/integration/runner/dockerd-entrypoint.sh

@@ -45,6 +45,7 @@ export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest}
 export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest}
 export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest}
 export DOCKER_KERBEROS_KDC_TAG=${DOCKER_KERBEROS_KDC_TAG:=latest}
+export DOCKER_KERBERIZED_HADOOP_TAG=${DOCKER_KERBERIZED_HADOOP_TAG:=latest}
 
 cd /ClickHouse/tests/integration
 exec "$@"

docker/test/performance-comparison/Dockerfile

@@ -1,5 +1,5 @@
 # docker build -t clickhouse/performance-comparison .
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"

docker/test/pvs/Dockerfile

@@ -4,11 +4,7 @@
 ARG FROM_TAG=latest
 FROM clickhouse/binary-builder:$FROM_TAG
 
-# PVS studio doesn't support aarch64/arm64, so there is a check for it everywhere
+RUN apt-get update --yes \
-# We'll produce an empty image for arm64
-ARG TARGETARCH
-
-RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \
     && apt-get install \
         bash \
         wget \
@@ -21,7 +17,7 @@ RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \
         libprotoc-dev \
         libgrpc++-dev \
         libc-ares-dev \
-        --yes --no-install-recommends )
+        --yes --no-install-recommends
 
 #RUN wget -nv -O - http://files.viva64.com/etc/pubkey.txt | sudo apt-key add -
 #RUN sudo wget -nv -O /etc/apt/sources.list.d/viva64.list http://files.viva64.com/etc/viva64.list
@@ -33,7 +29,7 @@ RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \
 
 ENV PKG_VERSION="pvs-studio-latest"
 
-RUN test x$TARGETARCH = xarm64 || ( set -x \
+RUN set -x \
     && export PUBKEY_HASHSUM="ad369a2e9d8b8c30f5a9f2eb131121739b79c78e03fef0f016ea51871a5f78cd4e6257b270dca0ac3be3d1f19d885516" \
     && wget -nv https://files.viva64.com/etc/pubkey.txt -O /tmp/pubkey.txt \
     && echo "${PUBKEY_HASHSUM} /tmp/pubkey.txt" | sha384sum -c \
@@ -41,7 +37,7 @@ RUN test x$TARGETARCH = xarm64 || ( set -x \
     && wget -nv "https://files.viva64.com/${PKG_VERSION}.deb" \
     && { debsig-verify ${PKG_VERSION}.deb \
     || echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \
-    && dpkg -i "${PKG_VERSION}.deb" )
+    && dpkg -i "${PKG_VERSION}.deb"
 
 ENV CCACHE_DIR=/test_output/ccache
 

docker/test/stress/run.sh

@@ -148,14 +148,12 @@ info signals
 continue
 gcore
 backtrace full
-info locals
+thread apply all backtrace full
 info registers
 disassemble /s
 up
-info locals
 disassemble /s
 up
-info locals
 disassemble /s
 p \"done\"
 detach
@@ -269,5 +267,5 @@ clickhouse-local --structure "test String, res String" -q "SELECT 'failure', tes
 # Default filename is 'core.PROCESS_ID'
 for core in core.*; do
     pigz $core
-    mv $core.gz /output/
+    mv $core.gz /test_output/
 done

docker/test/testflows/runner/Dockerfile

@@ -43,24 +43,27 @@ RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0
 ENV DOCKER_CHANNEL stable
 ENV DOCKER_VERSION 20.10.6
 
-RUN set -eux; \
+# Architecture of the image when BuildKit/buildx is used
-  \
+ARG TARGETARCH
-# this "case" statement is generated via "update.sh"
+
-  \
+# Install docker
-  if ! wget -nv -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \
+RUN arch=${TARGETARCH:-amd64} \
-    echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \
+  && case $arch in \
-    exit 1; \
+      amd64) rarch=x86_64 ;; \
-  fi; \
+      arm64) rarch=aarch64 ;; \
-  \
+    esac \
-  tar --extract \
+  && set -eux \
+  && if ! wget -nv -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${rarch}/docker-${DOCKER_VERSION}.tgz"; then \
+    echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${rarch}'" \
+    && exit 1; \
+  fi \
+  && tar --extract \
     --file docker.tgz \
     --strip-components 1 \
     --directory /usr/local/bin/ \
-  ; \
+  && rm docker.tgz \
-  rm docker.tgz; \
+  && dockerd --version \
-  \
+  && docker --version
-  dockerd --version; \
-  docker --version
 
 COPY modprobe.sh /usr/local/bin/modprobe
 COPY dockerd-entrypoint.sh /usr/local/bin/

docs/en/engines/table-engines/mergetree-family/mergetree.md

@@ -886,3 +886,12 @@ S3 disk can be configured as `main` or `cold` storage:
 ```
 
 In case of `cold` option a data can be moved to S3 if local disk free size will be smaller than `move_factor * disk_size` or by TTL move rule.
+
+## Virtual Columns {#virtual-columns}
+
+-   `_part` — Name of a part.
+-   `_part_index` — Sequential index of the part in the query result.
+-   `_partition_id` — Name of a partition.
+-   `_part_uuid` — Unique part identifier (if enabled MergeTree setting `assign_part_uuids`).
+-   `_partition_value` — Values (a tuple) of a `partition by` expression.
+-   `_sample_factor` — Sample factor (from the query).

docs/en/engines/table-engines/special/distributed.md

@@ -209,6 +209,8 @@ When querying a `Distributed` table, `SELECT` queries are sent to all shards and
 
 When the `max_parallel_replicas` option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas).
 
+To learn more about how distibuted `in` and `global in` queries are processed, refer to [this](../../../sql-reference/operators/in.md#select-distributed-subqueries) documentation. 
+
 ## Virtual Columns {#virtual-columns}
 
 -   `_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md).

docs/en/engines/table-engines/special/url.md

@@ -7,18 +7,29 @@ toc_title: URL
 
 Queries data to/from a remote HTTP/HTTPS server. This engine is similar to the [File](../../../engines/table-engines/special/file.md) engine.
 
-Syntax: `URL(URL, Format)`
+Syntax: `URL(URL [,Format] [,CompressionMethod])`
+
+- The `URL` parameter must conform to the structure of a Uniform Resource Locator. The specified URL must point to a server that uses HTTP or HTTPS. This does not require any additional headers for getting a response from the server.
+
+- The `Format` must be one that ClickHouse can use in `SELECT` queries and, if necessary, in `INSERTs`. For the full list of supported formats, see [Formats](../../../interfaces/formats.md#formats).
+
+- `CompressionMethod` indicates that whether the HTTP body should be compressed. If the compression is enabled, the HTTP packets sent by the URL engine contain 'Content-Encoding' header to indicate which compression method is used. 
+
+To enable compression, please first make sure the remote HTTP endpoint indicated by the `URL` parameter supports corresponding compression algorithm.
+   
+The supported `CompressionMethod` should be one of following:
+- gzip or gz
+- deflate
+- brotli or br
+- lzma or xz
+- zstd or zst
+- lz4
+- bz2
+- snappy
+- none
 
 ## Usage {#using-the-engine-in-the-clickhouse-server}
 
-The `format` must be one that ClickHouse can use in
-`SELECT` queries and, if necessary, in `INSERTs`. For the full list of supported formats, see
-[Formats](../../../interfaces/formats.md#formats).
-
-The `URL` must conform to the structure of a Uniform Resource Locator. The specified URL must point to a server
-that uses HTTP or HTTPS. This does not require any
-additional headers for getting a response from the server.
-
 `INSERT` and `SELECT` queries are transformed to `POST` and `GET` requests,
 respectively. For processing `POST` requests, the remote server must support
 [Chunked transfer encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding).

docs/en/introduction/adopters.md

@@ -67,6 +67,7 @@ toc_title: Adopters
 | <a href="https://geniee.co.jp" class="favicon">Geniee</a> | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) |
 | <a href="https://www.genotek.ru/" class="favicon">Genotek</a> | Bioinformatics | Main product | — | — | [Video, August 2020](https://youtu.be/v3KyZbz9lEE) |
 | <a href="https://gigapipe.com/" class="favicon">Gigapipe</a> | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) |
+| <a href="https://gigasheet.co/" class="favicon">Gigasheet</a> | Analytics | Main product | — | — | Direct Reference, February 2022|
 | <a href="https://glaber.io/" class="favicon">Glaber</a> | Monitoring | Main product | — | — | [Website](https://glaber.io/) |
 | <a href="https://graphcdn.io/" class="favicon">GraphCDN</a> | CDN | Traffic Analytics | — | — | [Blog Post in English, August 2021](https://altinity.com/blog/delivering-insight-on-graphql-apis-with-clickhouse-at-graphcdn/) |
 | <a href="https://www.grouparoo.com" class="favicon">Grouparoo</a> | Data Warehouse Integrations | Main product | — | — | [Official Website, November 2021](https://www.grouparoo.com/integrations) |

docs/en/operations/settings/settings.md

@@ -1803,6 +1803,20 @@ If an INSERTed block is skipped due to deduplication in the source table, there
 At the same time, this behaviour “breaks” `INSERT` idempotency. If an `INSERT` into the main table was successful and `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` allows for changing this behaviour. On retry, a materialized view will receive the repeat insert and will perform a deduplication check by itself,
 ignoring check result for the source table, and will insert rows lost because of the first failure.
 
+## insert_deduplication_token {#insert_deduplication_token}
+
+The setting allows a user to provide own deduplication semantic in MergeTree/ReplicatedMergeTree  
+For example, by providing a unique value for the setting in each INSERT statement,
+user can avoid the same inserted data being deduplicated
+
+Possilbe values:
+
+-  Any string
+
+Default value: empty string (disabled)
+
+`insert_deduplication_token` is used for deduplication _only_ when not empty
+
 ## max_network_bytes {#settings-max-network-bytes}
 
 Limits the data volume (in bytes) that is received or transmitted over the network when executing a query. This setting applies to every individual query.
@@ -2304,7 +2318,7 @@ Possible values:
 -   1 — Enabled.
 -   0 — Disabled.
 
-Default value: `0`.
+Default value: `1`.
 
 ## output_format_parallel_formatting {#output-format-parallel-formatting}
 
@@ -2315,7 +2329,7 @@ Possible values:
 -   1 — Enabled.
 -   0 — Disabled.
 
-Default value: `0`.
+Default value: `1`.
 
 ## min_chunk_bytes_for_parallel_parsing {#min-chunk-bytes-for-parallel-parsing}
 

docs/en/sql-reference/operators/in.md

@@ -216,6 +216,17 @@ This is more optimal than using the normal IN. However, keep the following point
 
 It also makes sense to specify a local table in the `GLOBAL IN` clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers.
 
+### Distributed Subqueries and max_rows_in_set
+
+You can use [`max_rows_in_set`](../../operations/settings/query-complexity.md#max-rows-in-set) and [`max_bytes_in_set`](../../operations/settings/query-complexity.md#max-rows-in-set) to control how much data is tranferred during distributed queries. 
+
+This is specially important if the  `global in` query returns a large amount of data. Consider the following sql - 
+```sql
+select * from table1 where col1 global in (select col1 from table2 where <some_predicate>)
+```
+ 
+If `some_predicate` is not selective enough, it will return large amount of data and cause performance issues. In such cases, it is wise to limit the data transfer over the network. Also, note that [`set_overflow_mode`](../../operations/settings/query-complexity.md#set_overflow_mode) is set to `throw` (by default) meaning that an exception is raised when these thresholds are met.
+
 ### Distributed Subqueries and max_parallel_replicas {#max_parallel_replica-subqueries}
 
 When max_parallel_replicas is greater than 1, distributed queries are further transformed. For example, the following:

docs/en/sql-reference/statements/alter/column.md

@@ -197,12 +197,13 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
 
 ## MATERIALIZE COLUMN {#materialize-column}
 
-Materializes the column in the parts where the column is missing. This is useful in case of creating a new column with complicated `DEFAULT` or `MATERIALIZED` expression. Calculation of the column directly on `SELECT` query can cause bigger request execution time, so it is reasonable to use `MATERIALIZE COLUMN` for such columns. To perform same manipulation for existing column, use `FINAL` modifier.
+Materializes or updates a column with an expression for a default value (`DEFAULT` or `MATERIALIZED`).
+It is used if it is necessary to add or update a column with a complicated expression, because evaluating such an expression directly on `SELECT` executing turns out to be expensive. 
 
 Syntax:
 
 ```sql
-ALTER TABLE table MATERIALIZE COLUMN col [FINAL];
+ALTER TABLE table MATERIALIZE COLUMN col;
 ```
 
 **Example**
@@ -211,20 +212,34 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL];
 DROP TABLE IF EXISTS tmp;
 SET mutations_sync = 2;
 CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple();
-INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10;
+INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5;
 ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x);
 
 ALTER TABLE tmp MATERIALIZE COLUMN s;
 
+SELECT groupArray(x), groupArray(s) FROM (select x,s from tmp order by x);
+
+┌─groupArray(x)─┬─groupArray(s)─────────┐
+│ [0,1,2,3,4]   │ ['0','1','2','3','4'] │
+└───────────────┴───────────────────────┘
+
+ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(round(100/x));
+
+INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5,5;
+
 SELECT groupArray(x), groupArray(s) FROM tmp;
-```
 
-**Result:**
+┌─groupArray(x)─────────┬─groupArray(s)──────────────────────────────────┐
+│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','20','17','14','12','11'] │
+└───────────────────────┴────────────────────────────────────────────────┘
 
-```sql
+ALTER TABLE tmp MATERIALIZE COLUMN s;
-┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐
+
-│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │
+SELECT groupArray(x), groupArray(s) FROM tmp;
-└───────────────────────┴───────────────────────────────────────────┘
+
+┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────────────────┐
+│ [0,1,2,3,4,5,6,7,8,9] │ ['inf','100','50','33','25','20','17','14','12','11'] │
+└───────────────────────┴───────────────────────────────────────────────────────┘
 ```
 
 **See Also**

docs/ru/engines/table-engines/mergetree-family/mergetree.md

@@ -872,3 +872,13 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd'
 ```
 
 Если диск сконфигурирован как `cold`, данные будут переноситься в S3 при срабатывании правил TTL или когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`.
+
+## Виртуальные столбцы {#virtual-columns}
+
+-   `_part` — Имя куска.
+-   `_part_index` — Номер куска по порядку в результате запроса.
+-   `_partition_id` — Имя партиции.
+-   `_part_uuid` — Уникальный идентификатор куска (если включена MergeTree настройка `assign_part_uuids`).
+-   `_partition_value` — Значения (кортеж) выражения `partition by`.
+-   `_sample_factor` — Коэффициент сэмплирования (из запроса).
+

docs/ru/operations/settings/settings.md

@@ -2119,7 +2119,7 @@ ClickHouse генерирует исключение:
 -   1 — включен режим параллельного разбора.
 -   0 — отключен режим параллельного разбора.
 
-Значение по умолчанию: `0`.
+Значение по умолчанию: `1`.
 
 ## output_format_parallel_formatting {#output-format-parallel-formatting}
 
@@ -2130,7 +2130,7 @@ ClickHouse генерирует исключение:
 -   1 — включен режим параллельного форматирования.
 -   0 — отключен режим параллельного форматирования.
 
-Значение по умолчанию: `0`.
+Значение по умолчанию: `1`.
 
 ## min_chunk_bytes_for_parallel_parsing {#min-chunk-bytes-for-parallel-parsing}
 

docs/ru/sql-reference/statements/alter/column.md

@@ -197,12 +197,13 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
 
 ## MATERIALIZE COLUMN {#materialize-column}
 
-Материализует столбец таблицы в кусках, в которых отсутствуют значения. Используется, если необходимо создать новый столбец со сложным материализованным выражением или выражением для заполнения по умолчанию (`DEFAULT`), потому как вычисление такого столбца прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным. Чтобы совершить ту же операцию для существующего столбца, используйте модификатор `FINAL`.
+Материализует или обновляет столбец таблицы с выражением для значения по умолчанию (`DEFAULT` или `MATERIALIZED`).
+Используется, если необходимо добавить или обновить столбец со сложным выражением, потому как вычисление такого выражения прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным.
 
 Синтаксис:
 
 ```sql
-ALTER TABLE table MATERIALIZE COLUMN col [FINAL];
+ALTER TABLE table MATERIALIZE COLUMN col;
 ```
 
 **Пример**
@@ -211,21 +212,39 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL];
 DROP TABLE IF EXISTS tmp;
 SET mutations_sync = 2;
 CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple();
-INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10;
+INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5;
 ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x);
 
 ALTER TABLE tmp MATERIALIZE COLUMN s;
 
+SELECT groupArray(x), groupArray(s) FROM (select x,s from tmp order by x);
+
+┌─groupArray(x)─┬─groupArray(s)─────────┐
+│ [0,1,2,3,4]   │ ['0','1','2','3','4'] │
+└───────────────┴───────────────────────┘
+
+ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(round(100/x));
+
+INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5,5;
+
 SELECT groupArray(x), groupArray(s) FROM tmp;
+
+┌─groupArray(x)─────────┬─groupArray(s)──────────────────────────────────┐
+│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','20','17','14','12','11'] │
+└───────────────────────┴────────────────────────────────────────────────┘
+
+ALTER TABLE tmp MATERIALIZE COLUMN s;
+
+SELECT groupArray(x), groupArray(s) FROM tmp;
+
+┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────────────────┐
+│ [0,1,2,3,4,5,6,7,8,9] │ ['inf','100','50','33','25','20','17','14','12','11'] │
+└───────────────────────┴───────────────────────────────────────────────────────┘
 ```
 
-**Результат:**
+**Смотрите также**
 
-```sql
+- [MATERIALIZED](../../statements/create/table.md#materialized).
-┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐
-│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │
-└───────────────────────┴───────────────────────────────────────────┘
-```
 
 ## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter}
 

docs/zh/engines/database-engines/index.md

@@ -14,7 +14,7 @@ toc_title: Introduction
 
 -   [MySQL](../../engines/database-engines/mysql.md)
 
--   [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md)
+-   [MaterializeMySQL](../../engines/database-engines/materialized-mysql.md)
 
 -   [Lazy](../../engines/database-engines/lazy.md)
 

docs/zh/engines/database-engines/materialized-mysql.md

@@ -1 +0,0 @@
-../../../en/engines/database-engines/materialized-mysql.md

docs/zh/engines/database-engines/materialized-mysql.md

@@ -0,0 +1,274 @@
+---
+toc_priority: 29
+toc_title: MaterializedMySQL
+---
+
+# [experimental] MaterializedMySQL {#materialized-mysql}
+
+!!! warning "警告"
+    这是一个实验性的特性，不应该在生产中使用.
+
+
+创建ClickHouse数据库，包含MySQL中所有的表，以及这些表中的所有数据。
+
+ClickHouse服务器作为MySQL副本工作。它读取binlog并执行DDL和DML查询。
+
+## 创建数据库 {#creating-a-database}
+
+``` sql
+CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
+ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
+[TABLE OVERRIDE table1 (...), TABLE OVERRIDE table2 (...)]
+```
+
+**引擎参数**
+
+-   `host:port` — MySQL 服务地址.
+-   `database` — MySQL 数据库名称.
+-   `user` — MySQL 用户名.
+-   `password` — MySQL 用户密码.
+
+**引擎配置**
+
+
+- `max_rows_in_buffer` — 允许在内存中缓存数据的最大行数(对于单个表和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值:`65 505`。
+- `max_bytes_in_buffer` - 允许在内存中缓存数据的最大字节数(对于单个表和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `1 048 576 `。
+- `max_rows_in_buffers` - 允许在内存中缓存数据的最大行数(用于数据库和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `65 505`。
+- `max_bytes_in_buffers` - 允许在内存中缓存数据的最大字节数(用于数据库和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `1 048 576`。
+- `max_flush_data_time ` - 允许数据在内存中缓存的最大毫秒数(对于数据库和无法查询的缓存数据)。当超过这个时间，数据将被物化。默认值: `1000`。
+- `max_wait_time_when_mysql_unavailable` - MySQL不可用时的重试间隔(毫秒)。负值禁用重试。默认值:`1000`。
+— `allows_query_when_mysql_lost `—允许在MySQL丢失时查询物化表。默认值:`0`(`false`)。
+
+```sql
+CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***')
+     SETTINGS
+        allows_query_when_mysql_lost=true,
+        max_wait_time_when_mysql_unavailable=10000;
+```
+
+**MySQL服务器端配置**
+
+为了`MaterializedMySQL`的正确工作，有一些必须设置的`MySQL`端配置设置:
+
+- `default_authentication_plugin = mysql_native_password `，因为 `MaterializedMySQL` 只能授权使用该方法。
+- `gtid_mode = on`，因为基于GTID的日志记录是提供正确的 `MaterializedMySQL`复制的强制要求。
+
+!!! attention "注意"
+    当打开`gtid_mode`时，您还应该指定`enforce_gtid_consistency = on`。
+
+## 虚拟列 {#virtual-columns}
+
+当使用`MaterializeMySQL`数据库引擎时，[ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md)表与虚拟的`_sign`和`_version`列一起使用。
+
+- `_version` — 事务版本. 类型 [UInt64](../../sql-reference/data-types/int-uint.md).
+- `_sign` — 删除标记. 类型 [Int8](../../sql-reference/data-types/int-uint.md). 可能的值:
+    - `1` — 行没有删除,
+    - `-1` — 行已被删除.
+
+## 支持的数据类型 {#data_types-support}
+
+| MySQL                   | ClickHouse                                                   |
+|-------------------------|--------------------------------------------------------------|
+| TINY                    | [Int8](../../sql-reference/data-types/int-uint.md)           |
+| SHORT                   | [Int16](../../sql-reference/data-types/int-uint.md)          |
+| INT24                   | [Int32](../../sql-reference/data-types/int-uint.md)          |
+| LONG                    | [UInt32](../../sql-reference/data-types/int-uint.md)         |
+| LONGLONG                | [UInt64](../../sql-reference/data-types/int-uint.md)         |
+| FLOAT                   | [Float32](../../sql-reference/data-types/float.md)           |
+| DOUBLE                  | [Float64](../../sql-reference/data-types/float.md)           |
+| DECIMAL, NEWDECIMAL     | [Decimal](../../sql-reference/data-types/decimal.md)         |
+| DATE, NEWDATE           | [Date](../../sql-reference/data-types/date.md)               |
+| DATETIME, TIMESTAMP     | [DateTime](../../sql-reference/data-types/datetime.md)       |
+| DATETIME2, TIMESTAMP2   | [DateTime64](../../sql-reference/data-types/datetime64.md)   |
+| YEAR                    | [UInt16](../../sql-reference/data-types/int-uint.md)         |
+| TIME                    | [Int64](../../sql-reference/data-types/int-uint.md)          |
+| ENUM                    | [Enum](../../sql-reference/data-types/enum.md)               |
+| STRING                  | [String](../../sql-reference/data-types/string.md)           |
+| VARCHAR, VAR_STRING     | [String](../../sql-reference/data-types/string.md)           |
+| BLOB                    | [String](../../sql-reference/data-types/string.md)           |
+| GEOMETRY                | [String](../../sql-reference/data-types/string.md)           |
+| BINARY                  | [FixedString](../../sql-reference/data-types/fixedstring.md) |
+| BIT                     | [UInt64](../../sql-reference/data-types/int-uint.md)         |
+| SET                     | [UInt64](../../sql-reference/data-types/int-uint.md)         |
+
+[Nullable](../../sql-reference/data-types/nullable.md) 已经被支持.
+
+MySQL中的Time 类型，会被ClickHouse转换成微秒来存储
+
+不支持其他类型。如果MySQL表包含此类类型的列，ClickHouse抛出异常"Unhandled data type"并停止复制。
+
+## 规范和推荐用法 {#specifics-and-recommendations}
+
+### 兼容性限制 {#compatibility-restrictions}
+
+除了数据类型的限制之外，还有一些限制与`MySQL`数据库相比有所不同，这应该在复制之前解决:
+
+- `MySQL` 中的每个表都应该包含 `PRIMARY KEY`。
+- 对于表的复制，那些包含 `ENUM` 字段值超出范围的行(在 `ENUM` 签名中指定)将不起作用。
+
+### DDL Queries {#ddl-queries}
+
+MySQL DDL 语句会被转换成对应的ClickHouse DDL 语句，比如： ([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create/index.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md)). 如果ClickHouse 无法解析某些语句DDL 操作，则会跳过。
+
+
+### 数据复制 {#data-replication}
+
+MaterializedMySQL不支持直接的 `INSERT`， `DELETE` 和 `UPDATE` 查询。然而，它们在数据复制方面得到了支持:
+
+- MySQL `INSERT`查询被转换为`_sign=1`的INSERT查询。
+- MySQL `DELETE`查询被转换为`INSERT`，并且`_sign=-1`。
+- 如果主键被修改了，MySQL的 `UPDATE` 查询将被转换为 `INSERT` 带 `_sign=1`  和INSERT 带有_sign=-1;如果主键没有被修改，则转换为`INSERT`和`_sign=1`。
+
+###  MaterializedMySQL 数据表查询 {#select}
+
+`SELECT` 查询从 `MaterializedMySQL`表有一些细节:
+
+ - 如果在SELECT查询中没有指定`_version`，则 [FINAL](../../sql-reference/statements/select/from.md#select-from- FINAL)修饰符被使用，所以只有带有 `MAX(_version)`的行会返回每个主键值。
+
+ - 如果在SELECT查询中没有指定 `_sign`，则默认使用 `WHERE _sign=1 `。所以被删除的行不是
+包含在结果集中。
+
+ - 结果包括列注释，以防MySQL数据库表中存在这些列注释。
+
+### 索引转换 {#index-conversion}
+
+在ClickHouse表中，MySQL的 `PRIMARY KEY` 和 `INDEX` 子句被转换为 `ORDER BY` 元组。
+
+ClickHouse只有一个物理排序，由 `order by` 条件决定。要创建一个新的物理排序，请使用[materialized views](../../sql-reference/statements/create/view.md#materialized)。
+
+**注意**
+
+- `_sign=-1` 的行不会被物理地从表中删除。
+- 级联 `UPDATE/DELETE` 查询不支持 `MaterializedMySQL` 引擎，因为他们在 MySQL binlog中不可见的
+— 复制很容易被破坏。
+— 禁止对数据库和表进行手工操作。
+- `MaterializedMySQL` 受[optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert)设置的影响。当MySQL服务器中的一个表发生变化时，数据会合并到 `MaterializedMySQL` 数据库中相应的表中。
+
+### 表重写 {#table-overrides}
+
+表覆盖可用于自定义ClickHouse DDL查询，从而允许您对应用程序进行模式优化。这对于控制分区特别有用，分区对MaterializedMySQL的整体性能非常重要。
+
+这些是你可以对MaterializedMySQL表重写的模式转换操作:
+
+ * 修改列类型。必须与原始类型兼容，否则复制将失败。例如，可以将`UInt32`列修改为`UInt64`，不能将 `String` 列修改为 `Array(String)`。
+ * 修改 [column TTL](../table-engines/mergetree-family/mergetree/#mergetree-column-ttl).
+ * 修改 [column compression codec](../../sql-reference/statements/create/table/#codecs).
+ * 增加 [ALIAS columns](../../sql-reference/statements/create/table/#alias).
+ * 增加 [skipping indexes](../table-engines/mergetree-family/mergetree/#table_engine-mergetree-data_skipping-indexes)
+ * 增加 [projections](../table-engines/mergetree-family/mergetree/#projections). 
+ 请注意，当使用 `SELECT ... FINAL ` (MaterializedMySQL默认是这样做的) 时，预测优化是被禁用的，所以这里是受限的， `INDEX ... TYPE hypothesis `[在v21.12的博客文章中描述]](https://clickhouse.com/blog/en/2021/clickhouse-v21.12-released/)可能在这种情况下更有用。
+ * 修改 [PARTITION BY](../table-engines/mergetree-family/custom-partitioning-key/)
+ * 修改 [ORDER BY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
+ * 修改 [PRIMARY KEY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
+ * 增加 [SAMPLE BY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
+ * 增加 [table TTL](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
+
+```sql
+CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
+[SETTINGS ...]
+[TABLE OVERRIDE table_name (
+    [COLUMNS (
+        [col_name [datatype] [ALIAS expr] [CODEC(...)] [TTL expr], ...]
+        [INDEX index_name expr TYPE indextype[(...)] GRANULARITY val, ...]
+        [PROJECTION projection_name (SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]), ...]
+    )]
+    [ORDER BY expr]
+    [PRIMARY KEY expr]
+    [PARTITION BY expr]
+    [SAMPLE BY expr]
+    [TTL expr]
+), ...]
+```
+
+示例:
+
+```sql
+CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
+TABLE OVERRIDE table1 (
+    COLUMNS (
+        userid UUID,
+        category LowCardinality(String),
+        timestamp DateTime CODEC(Delta, Default)
+    )
+    PARTITION BY toYear(timestamp)
+),
+TABLE OVERRIDE table2 (
+    COLUMNS (
+        client_ip String TTL created + INTERVAL 72 HOUR
+    )
+    SAMPLE BY ip_hash
+)
+```
+
+
+`COLUMNS`列表是稀疏的;根据指定修改现有列，添加额外的ALIAS列。不可能添加普通列或实体化列。具有不同类型的已修改列必须可从原始类型赋值。在执行`CREATE DATABASE` 查询时，目前还没有验证这个或类似的问题，因此需要格外小心。
+
+您可以为还不存在的表指定重写。
+
+!!! warning "警告"
+    如果使用时不小心，很容易用表重写中断复制。例如:
+
+    * 如果一个ALIAS列被添加了一个表覆盖，并且一个具有相同名称的列后来被添加到源MySQL表，在ClickHouse中转换后的ALTER table查询将失败并停止复制。
+    * 目前可以添加引用可空列的覆盖，而非空列是必需的，例如 `ORDER BY` 或 `PARTITION BY`。这将导致CREATE TABLE查询失败，也会导致复制停止。
+
+## 使用示例 {#examples-of-use}
+
+ MySQL 查询语句:
+
+``` sql
+mysql> CREATE DATABASE db;
+mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT);
+mysql> INSERT INTO db.test VALUES (1, 11), (2, 22);
+mysql> DELETE FROM db.test WHERE a=1;
+mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16);
+mysql> UPDATE db.test SET c='Wow!', b=222;
+mysql> SELECT * FROM test;
+```
+
+```text
+┌─a─┬───b─┬─c────┐
+│ 2 │ 222 │ Wow! │
+└───┴─────┴──────┘
+```
+
+ClickHouse中的数据库，与MySQL服务器交换数据:
+
+创建的数据库和表:
+
+``` sql
+CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***');
+SHOW TABLES FROM mysql;
+```
+
+``` text
+┌─name─┐
+│ test │
+└──────┘
+```
+
+数据插入之后：
+
+``` sql
+SELECT * FROM mysql.test;
+```
+
+``` text
+┌─a─┬──b─┐
+│ 1 │ 11 │
+│ 2 │ 22 │
+└───┴────┘
+```
+
+删除数据后，添加列并更新:
+
+``` sql
+SELECT * FROM mysql.test;
+```
+
+``` text
+┌─a─┬───b─┬─c────┐
+│ 2 │ 222 │ Wow! │
+└───┴─────┴──────┘
+```
+
+[来源文章](https://clickhouse.com/docs/en/engines/database-engines/materialized-mysql/) <!--hide-->

docs/zh/engines/database-engines/postgresql.md

@@ -24,6 +24,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cac
 -   `database` — 远程数据库名次
 -   `user` — PostgreSQL用户名称
 -   `password` — PostgreSQL用户密码
+-   `schema` - PostgreSQL 模式
 -   `use_table_cache` —  定义数据库表结构是否已缓存或不进行。可选的。默认值： `0`.
 
 ## 支持的数据类型 {#data_types-support}

docs/zh/engines/database-engines/replicated.md

@@ -31,6 +31,7 @@ CREATE DATABASE testdb ENGINE = Replicated('zoo_path', 'shard_name', 'replica_na
 
 当创建数据库的新副本时，该副本会自己创建表。如果副本已经不可用很长一段时间，并且已经滞后于复制日志-它用ZooKeeper中的当前元数据检查它的本地元数据，将带有数据的额外表移动到一个单独的非复制数据库(以免意外地删除任何多余的东西)，创建缺失的表，如果表名已经被重命名，则更新表名。数据在`ReplicatedMergeTree`级别被复制，也就是说，如果表没有被复制，数据将不会被复制(数据库只负责元数据)。
 
+允许[`ALTER TABLE ATTACH|FETCH|DROP|DROP DETACHED|DETACH PARTITION|PART`](../../sql-reference/statements/alter/partition.md)查询，但不允许复制。数据库引擎将只向当前副本添加/获取/删除分区/部件。但是，如果表本身使用了Replicated表引擎，那么数据将在使用`ATTACH`后被复制。
 ## 使用示例 {#usage-example}
 
 创建三台主机的集群:

docs/zh/whats-new/roadmap.md

@@ -5,6 +5,6 @@ toc_title: Roadmap
 
 # Roadmap {#roadmap}
 
-`2021年Roadmap`已公布供公开讨论查看[这里](https://github.com/ClickHouse/ClickHouse/issues/17623).
+`2022年Roadmap`已公布供公开讨论查看 [这里](https://github.com/ClickHouse/ClickHouse/issues/32513).
 
 {## [源文章](https://clickhouse.com/docs/en/roadmap/) ##}

programs/local/LocalServer.cpp

@@ -313,9 +313,15 @@ void LocalServer::cleanup()
 }
 
 
+static bool checkIfStdinIsRegularFile()
+{
+    struct stat file_stat;
+    return fstat(STDIN_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode);
+}
+
 std::string LocalServer::getInitialCreateTableQuery()
 {
-    if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format"))
+    if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!checkIfStdinIsRegularFile() || !config().has("query")))
         return {};
 
     auto table_name = backQuoteIfNeed(config().getString("table-name", "table"));
@@ -337,8 +343,9 @@ std::string LocalServer::getInitialCreateTableQuery()
         format_from_file_name = FormatFactory::instance().getFormatFromFileName(file_name, false);
     }
 
-    auto data_format
+    auto data_format = backQuoteIfNeed(
-        = backQuoteIfNeed(config().getString("table-data-format", format_from_file_name.empty() ? "TSV" : format_from_file_name));
+        config().getString("table-data-format", config().getString("format", format_from_file_name.empty() ? "TSV" : format_from_file_name)));
+
 
     if (table_structure == "auto")
         table_structure = "";
@@ -518,22 +525,17 @@ void LocalServer::processConfig()
 
         if (config().has("multiquery"))
             is_multiquery = true;
-
-        load_suggestions = true;
     }
     else
     {
-        if (delayed_interactive)
-        {
-            load_suggestions = true;
-        }
-
         need_render_progress = config().getBool("progress", false);
         echo_queries = config().hasOption("echo") || config().hasOption("verbose");
         ignore_error = config().getBool("ignore-error", false);
         is_multiquery = true;
     }
+
     print_stack_trace = config().getBool("stacktrace", false);
+    load_suggestions = (is_interactive || delayed_interactive) && !config().getBool("disable_suggestion", false);
 
     auto logging = (config().has("logger.console")
                     || config().has("logger.level")

programs/server/Server.cpp

@@ -22,6 +22,7 @@
 #include <base/getMemoryAmount.h>
 #include <base/errnoToString.h>
 #include <base/coverage.h>
+#include <Common/MemoryTracker.h>
 #include <Common/ClickHouseRevision.h>
 #include <Common/DNSResolver.h>
 #include <Common/CurrentMetrics.h>
@@ -925,6 +926,14 @@ if (ThreadFuzzer::instance().isEffective())
             total_memory_tracker.setDescription("(total)");
             total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
 
+            auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker();
+            if (config->has("global_memory_usage_overcommit_max_wait_microseconds"))
+            {
+                UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 0);
+                global_overcommit_tracker->setMaxWaitTime(max_overcommit_wait_time);
+            }
+            total_memory_tracker.setOvercommitTracker(global_overcommit_tracker);
+
             // FIXME logging-related things need synchronization -- see the 'Logger * log' saved
             // in a lot of places. For now, disable updating log configuration without server restart.
             //setTextLog(global_context->getTextLog());

programs/server/config.xml

@@ -217,13 +217,12 @@
         <!-- The following file is used only if ssl_require_client_auth=1 -->
         <ssl_ca_cert_file>/path/to/ssl_ca_cert_file</ssl_ca_cert_file>
 
-        <!-- Default compression algorithm (applied if client doesn't specify another algorithm, see result_compression in QueryInfo).
+        <!-- Default transport compression type (can be overridden by client, see the transport_compression_type field in QueryInfo).
              Supported algorithms: none, deflate, gzip, stream_gzip -->
-        <compression>deflate</compression>
+        <transport_compression_type>none</transport_compression_type>
 
-        <!-- Default compression level (applied if client doesn't specify another level, see result_compression in QueryInfo).
+        <!-- Default transport compression level. Supported levels: 0..3 -->
-             Supported levels: none, low, medium, high -->
+        <transport_compression_level>0</transport_compression_level>
-        <compression_level>medium</compression_level>
 
         <!-- Send/receive message size limits in bytes. -1 means unlimited -->
         <max_send_message_size>-1</max_send_message_size>

programs/static-files-disk-uploader/static-files-disk-uploader.cpp

@@ -31,7 +31,7 @@ namespace ErrorCodes
  * If test-mode option is added, files will be put by given url via PUT request.
  */
 
-void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, WriteBuffer & metadata_buf)
+void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, bool link, WriteBuffer & metadata_buf)
 {
     String remote_path;
     RE2::FullMatch(file_path.string(), EXTRACT_PATH_PATTERN, &remote_path);
@@ -52,22 +52,29 @@ void processFile(const fs::path & file_path, const fs::path & dst_path, bool tes
 
     auto dst_file_path = fs::path(dst_path) / remote_path;
 
-    auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
+    if (link)
-    std::shared_ptr<WriteBuffer> dst_buf;
+    {
-
+        fs::create_symlink(file_path, dst_file_path);
-    /// test mode for integration tests.
+    }
-    if (test_mode)
-        dst_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT);
     else
-        dst_buf = std::make_shared<WriteBufferFromFile>(dst_file_path);
+    {
+        auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
+        std::shared_ptr<WriteBuffer> dst_buf;
 
-    copyData(*src_buf, *dst_buf);
+        /// test mode for integration tests.
-    dst_buf->next();
+        if (test_mode)
-    dst_buf->finalize();
+            dst_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT);
+        else
+            dst_buf = std::make_shared<WriteBufferFromFile>(dst_file_path);
+
+        copyData(*src_buf, *dst_buf);
+        dst_buf->next();
+        dst_buf->finalize();
+    }
 };
 
 
-void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode)
+void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode, bool link)
 {
     std::cerr << "Data path: " << data_path << ", destination path: " << dst_path << std::endl;
 
@@ -94,7 +101,7 @@ void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_
     {
         if (dir_it->is_directory())
         {
-            processFile(dir_it->path(), dst_path, test_mode, *root_meta);
+            processFile(dir_it->path(), dst_path, test_mode, link, *root_meta);
 
             String directory_prefix;
             RE2::FullMatch(dir_it->path().string(), EXTRACT_PATH_PATTERN, &directory_prefix);
@@ -115,14 +122,14 @@ void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_
 
             fs::directory_iterator files_end;
             for (fs::directory_iterator file_it(dir_it->path()); file_it != files_end; ++file_it)
-                processFile(file_it->path(), dst_path, test_mode, *directory_meta);
+                processFile(file_it->path(), dst_path, test_mode, link, *directory_meta);
 
             directory_meta->next();
             directory_meta->finalize();
         }
         else
         {
-            processFile(dir_it->path(), dst_path, test_mode, *root_meta);
+            processFile(dir_it->path(), dst_path, test_mode, link, *root_meta);
         }
     }
     root_meta->next();
@@ -141,6 +148,7 @@ try
         ("help,h", "produce help message")
         ("metadata-path", po::value<std::string>(), "Metadata path (select data_paths from system.tables where name='table_name'")
         ("test-mode", "Use test mode, which will put data on given url via PUT")
+        ("link", "Create symlinks instead of copying")
         ("url", po::value<std::string>(), "Web server url for test mode")
         ("output-dir", po::value<std::string>(), "Directory to put files in non-test mode");
 
@@ -186,7 +194,7 @@ try
             root_path = fs::current_path();
     }
 
-    processTableFiles(fs_path, root_path, test_mode);
+    processTableFiles(fs_path, root_path, test_mode, options.count("link"));
 
     return 0;
 }

src/Access/Common/AccessType.h

@@ -86,7 +86,7 @@ enum class AccessType
     M(CREATE_DICTIONARY, "", DICTIONARY, CREATE) /* allows to execute {CREATE|ATTACH} DICTIONARY */\
     M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables;
                                                      implicitly enabled by the grant CREATE_TABLE on any table */ \
-    M(CREATE_FUNCTION, "", DATABASE, CREATE) /* allows to execute CREATE FUNCTION */ \
+    M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \
     M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \
     \
     M(DROP_DATABASE, "", DATABASE, DROP) /* allows to execute {DROP|DETACH} DATABASE */\
@@ -94,7 +94,7 @@ enum class AccessType
     M(DROP_VIEW, "", VIEW, DROP) /* allows to execute {DROP|DETACH} TABLE for views;
                                     implicitly enabled by the grant DROP_TABLE */\
     M(DROP_DICTIONARY, "", DICTIONARY, DROP) /* allows to execute {DROP|DETACH} DICTIONARY */\
-    M(DROP_FUNCTION, "", DATABASE, DROP) /* allows to execute DROP FUNCTION */\
+    M(DROP_FUNCTION, "", GLOBAL, DROP) /* allows to execute DROP FUNCTION */\
     M(DROP, "", GROUP, ALL) /* allows to execute {DROP|DETACH} */\
     \
     M(TRUNCATE, "TRUNCATE TABLE", TABLE, ALL) \
@@ -113,9 +113,9 @@ enum class AccessType
     M(ALTER_ROLE, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(DROP_ROLE, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(ROLE_ADMIN, "", GLOBAL, ACCESS_MANAGEMENT) /* allows to grant and revoke the roles which are not granted to the current user with admin option */\
-    M(CREATE_ROW_POLICY, "CREATE POLICY", GLOBAL, ACCESS_MANAGEMENT) \
+    M(CREATE_ROW_POLICY, "CREATE POLICY", TABLE, ACCESS_MANAGEMENT) \
-    M(ALTER_ROW_POLICY, "ALTER POLICY", GLOBAL, ACCESS_MANAGEMENT) \
+    M(ALTER_ROW_POLICY, "ALTER POLICY", TABLE, ACCESS_MANAGEMENT) \
-    M(DROP_ROW_POLICY, "DROP POLICY", GLOBAL, ACCESS_MANAGEMENT) \
+    M(DROP_ROW_POLICY, "DROP POLICY", TABLE, ACCESS_MANAGEMENT) \
     M(CREATE_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(ALTER_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \
     M(DROP_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \
@@ -124,7 +124,7 @@ enum class AccessType
     M(DROP_SETTINGS_PROFILE, "DROP PROFILE", GLOBAL, ACCESS_MANAGEMENT) \
     M(SHOW_USERS, "SHOW CREATE USER", GLOBAL, SHOW_ACCESS) \
     M(SHOW_ROLES, "SHOW CREATE ROLE", GLOBAL, SHOW_ACCESS) \
-    M(SHOW_ROW_POLICIES, "SHOW POLICIES, SHOW CREATE ROW POLICY, SHOW CREATE POLICY", GLOBAL, SHOW_ACCESS) \
+    M(SHOW_ROW_POLICIES, "SHOW POLICIES, SHOW CREATE ROW POLICY, SHOW CREATE POLICY", TABLE, SHOW_ACCESS) \
     M(SHOW_QUOTAS, "SHOW CREATE QUOTA", GLOBAL, SHOW_ACCESS) \
     M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \
     M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \

src/Access/ContextAccess.cpp

@@ -425,6 +425,7 @@ bool ContextAccess::checkAccessImplHelper(const AccessFlags & flags, const Args
             | AccessType::TRUNCATE;
 
         const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY;
+        const AccessFlags function_ddl = AccessType::CREATE_FUNCTION | AccessType::DROP_FUNCTION;
         const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
         const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
         const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS;
@@ -432,7 +433,7 @@ bool ContextAccess::checkAccessImplHelper(const AccessFlags & flags, const Args
         const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY;
         const AccessFlags not_readonly_1_flags = AccessType::CREATE_TEMPORARY_TABLE;
 
-        const AccessFlags ddl_flags = table_ddl | dictionary_ddl;
+        const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl;
         const AccessFlags introspection_flags = AccessType::INTROSPECTION;
     };
     static const PrecalculatedFlags precalc;

src/Access/tests/gtest_access_rights_ops.cpp

@@ -45,7 +45,15 @@ TEST(AccessRights, Union)
     lhs.grant(AccessType::INSERT);
     rhs.grant(AccessType::ALL, "db1");
     lhs.makeUnion(rhs);
-    ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, CREATE FUNCTION, DROP, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
+    ASSERT_EQ(lhs.toString(),
+              "GRANT INSERT ON *.*, "
+              "GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, "
+              "CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, "
+              "TRUNCATE, OPTIMIZE, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, "
+              "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
+              "SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
+              "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
+              "SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
 }
 
 

src/AggregateFunctions/ReservoirSampler.h

@@ -239,6 +239,7 @@ private:
 
     UInt64 genRandom(size_t lim)
     {
+        assert(lim > 0);
         /// With a large number of values, we will generate random numbers several times slower.
         if (lim <= static_cast<UInt64>(rng.max()))
             return static_cast<UInt32>(rng()) % static_cast<UInt32>(lim);

src/Client/ClientBase.cpp

@@ -1317,7 +1317,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
         if (insert && insert->select)
             insert->tryFindInputFunction(input_function);
 
-        bool is_async_insert = global_context->getSettings().async_insert && insert && insert->hasInlinedData();
+        bool is_async_insert = global_context->getSettingsRef().async_insert && insert && insert->hasInlinedData();
 
         /// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately.
         if (insert && (!insert->select || input_function) && !insert->watch && !is_async_insert)

src/Columns/ColumnArray.cpp

@@ -50,12 +50,12 @@ ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr &&
     if (!offsets_concrete)
         throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::LOGICAL_ERROR);
 
-    if (!offsets_concrete->empty() && nested_column)
+    if (!offsets_concrete->empty() && data)
     {
         Offset last_offset = offsets_concrete->getData().back();
 
         /// This will also prevent possible overflow in offset.
-        if (nested_column->size() != last_offset)
+        if (data->size() != last_offset)
             throw Exception("offsets_column has data inconsistent with nested_column", ErrorCodes::LOGICAL_ERROR);
     }
 

src/Common/ArenaUtils.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <string.h>
+#include <string>
+#include <base/StringRef.h>
+
+/** Copy string value into Arena.
+  * Arena should support method:
+  * char * alloc(size_t size).
+  */
+template <typename Arena>
+inline StringRef copyStringInArena(Arena & arena, StringRef value)
+{
+    size_t key_size = value.size;
+    char * place_for_key = arena.alloc(key_size);
+    memcpy(reinterpret_cast<void *>(place_for_key), reinterpret_cast<const void *>(value.data), key_size);
+    StringRef result{place_for_key, key_size};
+
+    return result;
+}

src/Common/ColumnsHashing.h

@@ -387,47 +387,52 @@ struct HashMethodSingleLowCardinalityColumn : public SingleColumnMethod
     }
 
     template <typename Data>
-    ALWAYS_INLINE FindResult findFromRow(Data & data, size_t row_, Arena & pool)
+    ALWAYS_INLINE FindResult findKey(Data & data, size_t row_, Arena & pool)
     {
         size_t row = getIndexAt(row_);
 
         if (is_nullable && row == 0)
         {
             if constexpr (has_mapped)
-                return FindResult(data.hasNullKeyData() ? &data.getNullKeyData() : nullptr, data.hasNullKeyData());
+                return FindResult(data.hasNullKeyData() ? &data.getNullKeyData() : nullptr, data.hasNullKeyData(), 0);
             else
-                return FindResult(data.hasNullKeyData());
+                return FindResult(data.hasNullKeyData(), 0);
         }
 
         if (visit_cache[row] != VisitValue::Empty)
         {
             if constexpr (has_mapped)
-                return FindResult(&mapped_cache[row], visit_cache[row] == VisitValue::Found);
+                return FindResult(&mapped_cache[row], visit_cache[row] == VisitValue::Found, 0);
             else
-                return FindResult(visit_cache[row] == VisitValue::Found);
+                return FindResult(visit_cache[row] == VisitValue::Found, 0);
         }
 
         auto key_holder = getKeyHolder(row_, pool);
 
-        typename Data::iterator it;
+        typename Data::LookupResult it;
         if (saved_hash)
-            it = data.find(*key_holder, saved_hash[row]);
+            it = data.find(keyHolderGetKey(key_holder), saved_hash[row]);
         else
-            it = data.find(*key_holder);
+            it = data.find(keyHolderGetKey(key_holder));
 
-        bool found = it != data.end();
+        bool found = it;
         visit_cache[row] = found ? VisitValue::Found : VisitValue::NotFound;
 
         if constexpr (has_mapped)
         {
             if (found)
-                mapped_cache[row] = it->second;
+                mapped_cache[row] = it->getMapped();
         }
 
+        size_t offset = 0;
+
+        if constexpr (FindResult::has_offset)
+            offset = found ? data.offsetInternal(it) : 0;
+
         if constexpr (has_mapped)
-            return FindResult(&mapped_cache[row], found);
+            return FindResult(&mapped_cache[row], found, offset);
         else
-            return FindResult(found);
+            return FindResult(found, offset);
     }
 
     template <typename Data>

src/Common/MemoryTracker.cpp

@@ -1,6 +1,7 @@
 #include "MemoryTracker.h"
 
 #include <IO/WriteHelpers.h>
+#include <Common/VariableContext.h>
 #include <Interpreters/TraceCollector.h>
 #include <Common/Exception.h>
 #include <Common/LockMemoryExceptionInThread.h>
@@ -8,6 +9,7 @@
 #include <Common/formatReadable.h>
 #include <Common/ProfileEvents.h>
 #include <Common/thread_local_rng.h>
+#include <Common/OvercommitTracker.h>
 #include <base/logger_useful.h>
 
 #include <atomic>
@@ -95,7 +97,7 @@ void MemoryTracker::logMemoryUsage(Int64 current) const
 }
 
 
-void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
+void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryTracker * query_tracker)
 {
     if (size < 0)
         throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Negative size ({}) is passed to MemoryTracker. It is a bug.", size);
@@ -104,7 +106,8 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
     {
         /// Since the MemoryTrackerBlockerInThread should respect the level, we should go to the next parent.
         if (auto * loaded_next = parent.load(std::memory_order_relaxed))
-            loaded_next->allocImpl(size, throw_if_memory_exceeded);
+            loaded_next->allocImpl(size, throw_if_memory_exceeded,
+                level == VariableContext::Process ? this : query_tracker);
         return;
     }
 
@@ -186,18 +189,30 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
 
     if (unlikely(current_hard_limit && will_be > current_hard_limit) && memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded)
     {
-        /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
+        bool need_to_throw = true;
-        MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
+        bool try_to_free_memory = overcommit_tracker != nullptr && query_tracker != nullptr;
-        ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
+        if (try_to_free_memory)
-        const auto * description = description_ptr.load(std::memory_order_relaxed);
+            need_to_throw = overcommit_tracker->needToStopQuery(query_tracker);
-        throw DB::Exception(
+
-            DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,
+        if (need_to_throw)
-            "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}",
+        {
-            description ? " " : "",
+            /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
-            description ? description : "",
+            MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
-            formatReadableSizeWithBinarySuffix(will_be),
+            ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
-            size,
+            const auto * description = description_ptr.load(std::memory_order_relaxed);
-            formatReadableSizeWithBinarySuffix(current_hard_limit));
+            throw DB::Exception(
+                DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,
+                "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}",
+                description ? " " : "",
+                description ? description : "",
+                formatReadableSizeWithBinarySuffix(will_be),
+                size,
+                formatReadableSizeWithBinarySuffix(current_hard_limit));
+        }
+        else
+        {
+            will_be = amount.load(std::memory_order_relaxed);
+        }
     }
 
     bool peak_updated;
@@ -221,7 +236,8 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
     }
 
     if (auto * loaded_next = parent.load(std::memory_order_relaxed))
-        loaded_next->allocImpl(size, throw_if_memory_exceeded);
+        loaded_next->allocImpl(size, throw_if_memory_exceeded,
+            level == VariableContext::Process ? this : query_tracker);
 }
 
 void MemoryTracker::alloc(Int64 size)
@@ -302,10 +318,23 @@ void MemoryTracker::free(Int64 size)
 }
 
 
+OvercommitRatio MemoryTracker::getOvercommitRatio()
+{
+    return { amount.load(std::memory_order_relaxed), soft_limit.load(std::memory_order_relaxed) };
+}
+
+
+OvercommitRatio MemoryTracker::getOvercommitRatio(Int64 limit)
+{
+    return { amount.load(std::memory_order_relaxed), limit };
+}
+
+
 void MemoryTracker::resetCounters()
 {
     amount.store(0, std::memory_order_relaxed);
     peak.store(0, std::memory_order_relaxed);
+    soft_limit.store(0, std::memory_order_relaxed);
     hard_limit.store(0, std::memory_order_relaxed);
     profiler_limit.store(0, std::memory_order_relaxed);
 }
@@ -330,6 +359,12 @@ void MemoryTracker::set(Int64 to)
 }
 
 
+void MemoryTracker::setSoftLimit(Int64 value)
+{
+    soft_limit.store(value, std::memory_order_relaxed);
+}
+
+
 void MemoryTracker::setHardLimit(Int64 value)
 {
     hard_limit.store(value, std::memory_order_relaxed);

src/Common/MemoryTracker.h

@@ -28,6 +28,9 @@ extern thread_local bool memory_tracker_always_throw_logical_error_on_allocation
 #define ALLOW_ALLOCATIONS_IN_SCOPE static_assert(true)
 #endif
 
+struct OvercommitRatio;
+struct OvercommitTracker;
+
 /** Tracks memory consumption.
   * It throws an exception if amount of consumed memory become greater than certain limit.
   * The same memory tracker could be simultaneously used in different threads.
@@ -40,6 +43,7 @@ class MemoryTracker
 private:
     std::atomic<Int64> amount {0};
     std::atomic<Int64> peak {0};
+    std::atomic<Int64> soft_limit {0};
     std::atomic<Int64> hard_limit {0};
     std::atomic<Int64> profiler_limit {0};
 
@@ -61,6 +65,8 @@ private:
     /// This description will be used as prefix into log messages (if isn't nullptr)
     std::atomic<const char *> description_ptr = nullptr;
 
+    OvercommitTracker * overcommit_tracker = nullptr;
+
     bool updatePeak(Int64 will_be, bool log_memory_usage);
     void logMemoryUsage(Int64 current) const;
 
@@ -83,7 +89,7 @@ public:
 
     void allocNoThrow(Int64 size);
 
-    void allocImpl(Int64 size, bool throw_if_memory_exceeded);
+    void allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryTracker * query_tracker = nullptr);
 
     void realloc(Int64 old_size, Int64 new_size)
     {
@@ -108,8 +114,14 @@ public:
         return peak.load(std::memory_order_relaxed);
     }
 
+    void setSoftLimit(Int64 value);
     void setHardLimit(Int64 value);
 
+    Int64 getSoftLimit() const
+    {
+        return soft_limit.load(std::memory_order_relaxed);
+    }
+
     /** Set limit if it was not set.
       * Otherwise, set limit to new value, if new value is greater than previous limit.
       */
@@ -159,6 +171,14 @@ public:
         description_ptr.store(description, std::memory_order_relaxed);
     }
 
+    OvercommitRatio getOvercommitRatio();
+    OvercommitRatio getOvercommitRatio(Int64 limit);
+
+    void setOvercommitTracker(OvercommitTracker * tracker) noexcept
+    {
+        overcommit_tracker = tracker;
+    }
+
     /// Reset the accumulated data
     void resetCounters();
 

src/Common/OvercommitTracker.cpp

@@ -0,0 +1,119 @@
+#include "OvercommitTracker.h"
+
+#include <chrono>
+#include <mutex>
+#include <Interpreters/ProcessList.h>
+
+using namespace std::chrono_literals;
+
+OvercommitTracker::OvercommitTracker()
+    : max_wait_time(0us)
+    , picked_tracker(nullptr)
+    , cancelation_state(QueryCancelationState::NONE)
+{}
+
+void OvercommitTracker::setMaxWaitTime(UInt64 wait_time)
+{
+    std::lock_guard guard(overcommit_m);
+    max_wait_time = wait_time * 1us;
+}
+
+bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker)
+{
+    std::unique_lock<std::mutex> lk(overcommit_m);
+
+    pickQueryToExclude();
+    assert(cancelation_state == QueryCancelationState::RUNNING);
+
+    // If no query was chosen we need to stop current query.
+    // This may happen if no soft limit is set.
+    if (picked_tracker == nullptr)
+    {
+        cancelation_state = QueryCancelationState::NONE;
+        return true;
+    }
+    if (picked_tracker == tracker)
+        return true;
+    return !cv.wait_for(lk, max_wait_time, [this]()
+    {
+        return cancelation_state == QueryCancelationState::NONE;
+    });
+}
+
+void OvercommitTracker::unsubscribe(MemoryTracker * tracker)
+{
+    std::unique_lock<std::mutex> lk(overcommit_m);
+    if (picked_tracker == tracker)
+    {
+        LOG_DEBUG(getLogger(), "Picked query stopped");
+
+        picked_tracker = nullptr;
+        cancelation_state = QueryCancelationState::NONE;
+        cv.notify_all();
+    }
+}
+
+UserOvercommitTracker::UserOvercommitTracker(DB::ProcessListForUser * user_process_list_)
+    : user_process_list(user_process_list_)
+{}
+
+void UserOvercommitTracker::pickQueryToExcludeImpl()
+{
+    MemoryTracker * query_tracker = nullptr;
+    OvercommitRatio current_ratio{0, 0};
+    // At this moment query list must be read only.
+    // BlockQueryIfMemoryLimit is used in ProcessList to guarantee this.
+    auto & queries = user_process_list->queries;
+    LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", queries.size());
+    for (auto const & query : queries)
+    {
+        if (query.second->isKilled())
+            continue;
+
+        auto * memory_tracker = query.second->getMemoryTracker();
+        if (!memory_tracker)
+            continue;
+
+        auto ratio = memory_tracker->getOvercommitRatio();
+        LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit);
+        if (ratio.soft_limit != 0 && current_ratio < ratio)
+        {
+            query_tracker = memory_tracker;
+            current_ratio   = ratio;
+        }
+    }
+    LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}",
+        current_ratio.committed, current_ratio.soft_limit);
+    picked_tracker = query_tracker;
+}
+
+void GlobalOvercommitTracker::pickQueryToExcludeImpl()
+{
+    MemoryTracker * query_tracker = nullptr;
+    OvercommitRatio current_ratio{0, 0};
+    process_list->processEachQueryStatus([&](DB::QueryStatus const & query)
+    {
+        if (query.isKilled())
+            return;
+
+        Int64 user_soft_limit = 0;
+        if (auto const * user_process_list = query.getUserProcessList())
+            user_soft_limit = user_process_list->user_memory_tracker.getSoftLimit();
+        if (user_soft_limit == 0)
+            return;
+
+        auto * memory_tracker = query.getMemoryTracker();
+        if (!memory_tracker)
+            return;
+        auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit);
+        LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit);
+        if (current_ratio < ratio)
+        {
+            query_tracker = memory_tracker;
+            current_ratio   = ratio;
+        }
+    });
+    LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}",
+        current_ratio.committed, current_ratio.soft_limit);
+    picked_tracker = query_tracker;
+}

src/Common/OvercommitTracker.h

@@ -0,0 +1,155 @@
+#pragma once
+
+#include <base/logger_useful.h>
+#include <base/types.h>
+#include <boost/core/noncopyable.hpp>
+#include <Poco/Logger.h>
+#include <cassert>
+#include <chrono>
+#include <condition_variable>
+#include <mutex>
+#include <unordered_map>
+
+// This struct is used for the comparison of query memory usage.
+struct OvercommitRatio
+{
+    OvercommitRatio(Int64 committed_, Int64 soft_limit_)
+        : committed(committed_)
+        , soft_limit(soft_limit_)
+    {}
+
+    friend bool operator<(OvercommitRatio const & lhs, OvercommitRatio const & rhs) noexcept
+    {
+        // (a / b < c / d) <=> (a * d < c * b)
+        return (lhs.committed * rhs.soft_limit) < (rhs.committed * lhs.soft_limit)
+            || (lhs.soft_limit == 0 && rhs.soft_limit > 0)
+            || (lhs.committed == 0 && rhs.committed == 0 && lhs.soft_limit > rhs.soft_limit);
+    }
+
+    // actual query memory usage
+    Int64 committed;
+    // guaranteed amount of memory query can use
+    Int64 soft_limit;
+};
+
+class MemoryTracker;
+
+// Usually it's hard to set some reasonable hard memory limit
+// (especially, the default value). This class introduces new
+// mechanisim for the limiting of memory usage.
+// Soft limit represents guaranteed amount of memory query/user
+// may use. It's allowed to exceed this limit. But if hard limit
+// is reached, query with the biggest overcommit ratio
+// is killed to free memory.
+struct OvercommitTracker : boost::noncopyable
+{
+    OvercommitTracker();
+
+    void setMaxWaitTime(UInt64 wait_time);
+
+    bool needToStopQuery(MemoryTracker * tracker);
+
+    void unsubscribe(MemoryTracker * tracker);
+
+    virtual ~OvercommitTracker() = default;
+
+protected:
+    virtual void pickQueryToExcludeImpl() = 0;
+
+    mutable std::mutex overcommit_m;
+    mutable std::condition_variable cv;
+
+    std::chrono::microseconds max_wait_time;
+
+    enum class QueryCancelationState
+    {
+        NONE,
+        RUNNING,
+    };
+
+    // Specifies memory tracker of the chosen to stop query.
+    // If soft limit is not set, all the queries which reach hard limit must stop.
+    // This case is represented as picked tracker pointer is set to nullptr and
+    // overcommit tracker is in RUNNING state.
+    MemoryTracker * picked_tracker;
+    QueryCancelationState cancelation_state;
+
+    virtual Poco::Logger * getLogger() = 0;
+
+private:
+
+    void pickQueryToExclude()
+    {
+        if (cancelation_state != QueryCancelationState::RUNNING)
+        {
+            pickQueryToExcludeImpl();
+            cancelation_state = QueryCancelationState::RUNNING;
+        }
+    }
+
+    friend struct BlockQueryIfMemoryLimit;
+};
+
+namespace DB
+{
+    class ProcessList;
+    struct ProcessListForUser;
+}
+
+struct UserOvercommitTracker : OvercommitTracker
+{
+    explicit UserOvercommitTracker(DB::ProcessListForUser * user_process_list_);
+
+    ~UserOvercommitTracker() override = default;
+
+protected:
+    void pickQueryToExcludeImpl() override final;
+
+    Poco::Logger * getLogger() override final { return logger; }
+private:
+    DB::ProcessListForUser * user_process_list;
+    Poco::Logger * logger = &Poco::Logger::get("UserOvercommitTracker");
+};
+
+struct GlobalOvercommitTracker : OvercommitTracker
+{
+    explicit GlobalOvercommitTracker(DB::ProcessList * process_list_)
+        : process_list(process_list_)
+    {}
+
+    ~GlobalOvercommitTracker() override = default;
+
+protected:
+    void pickQueryToExcludeImpl() override final;
+
+    Poco::Logger * getLogger() override final { return logger; }
+private:
+    DB::ProcessList * process_list;
+    Poco::Logger * logger = &Poco::Logger::get("GlobalOvercommitTracker");
+};
+
+// UserOvercommitTracker requires to check the whole list of user's queries
+// to pick one to stop. BlockQueryIfMemoryLimit struct allows to wait until
+// query selection is finished. It's used in ProcessList to make user query
+// list immutable when UserOvercommitTracker reads it.
+struct BlockQueryIfMemoryLimit
+{
+    BlockQueryIfMemoryLimit(OvercommitTracker const & overcommit_tracker)
+        : mutex(overcommit_tracker.overcommit_m)
+        , lk(mutex)
+    {
+        if (overcommit_tracker.cancelation_state == OvercommitTracker::QueryCancelationState::RUNNING)
+        {
+            overcommit_tracker.cv.wait_for(lk, overcommit_tracker.max_wait_time, [&overcommit_tracker]()
+            {
+                return overcommit_tracker.cancelation_state == OvercommitTracker::QueryCancelationState::NONE;
+            });
+        }
+    }
+
+    ~BlockQueryIfMemoryLimit() = default;
+
+private:
+    std::mutex & mutex;
+    std::unique_lock<std::mutex> lk;
+};

src/Coordination/KeeperServer.cpp

@@ -286,10 +286,7 @@ RaftAppendResult KeeperServer::putRequestBatch(const KeeperStorage::RequestsForS
     for (const auto & [session_id, request] : requests_for_sessions)
         entries.push_back(getZooKeeperLogEntry(session_id, request));
 
-    {
+    return raft_instance->append_entries(entries);
-        std::lock_guard lock(append_entries_mutex);
-        return raft_instance->append_entries(entries);
-    }
 }
 
 bool KeeperServer::isLeader() const

src/Coordination/KeeperServer.h

@@ -28,8 +28,6 @@ private:
     nuraft::ptr<nuraft::asio_service> asio_service;
     nuraft::ptr<nuraft::rpc_listener> asio_listener;
 
-    std::mutex append_entries_mutex;
-
     std::mutex initialized_mutex;
     std::atomic<bool> initialized_flag = false;
     std::condition_variable initialized_cv;

src/Coordination/SnapshotableHashTable.h

@@ -2,6 +2,7 @@
 #include <base/StringRef.h>
 #include <Common/HashTable/HashMap.h>
 #include <Common/ArenaWithFreeLists.h>
+#include <Common/ArenaUtils.h>
 #include <unordered_map>
 #include <list>
 #include <atomic>
@@ -39,6 +40,8 @@ private:
     size_t current_version{0};
     size_t snapshot_up_to_version{0};
     ArenaWithFreeLists arena;
+    /// Collect invalid iterators to avoid traversing the whole list
+    std::vector<Mapped> snapshot_invalid_iters;
 
     uint64_t approximate_data_size{0};
 
@@ -116,17 +119,6 @@ private:
         }
     }
 
-    StringRef copyStringInArena(const std::string & value_to_copy)
-    {
-        size_t value_to_copy_size = value_to_copy.size();
-        char * place_for_key = arena.alloc(value_to_copy_size);
-        memcpy(reinterpret_cast<void *>(place_for_key), reinterpret_cast<const void *>(value_to_copy.data()), value_to_copy_size);
-        StringRef updated_value{place_for_key, value_to_copy_size};
-
-        return updated_value;
-    }
-
-
 public:
 
     using iterator = typename List::iterator;
@@ -140,7 +132,8 @@ public:
 
         if (!it)
         {
-            ListElem elem{copyStringInArena(key), value, current_version};
+
+            ListElem elem{copyStringInArena(arena, key), value, current_version};
             auto itr = list.insert(list.end(), std::move(elem));
             bool inserted;
             map.emplace(itr->key, it, inserted, hash_value);
@@ -162,7 +155,7 @@ public:
 
         if (it == map.end())
         {
-            ListElem elem{copyStringInArena(key), value, current_version};
+            ListElem elem{copyStringInArena(arena, key), value, current_version};
             auto itr = list.insert(list.end(), std::move(elem));
             bool inserted;
             map.emplace(itr->key, it, inserted, hash_value);
@@ -178,6 +171,7 @@ public:
                 list_itr->active_in_map = false;
                 auto new_list_itr = list.insert(list.end(), std::move(elem));
                 it->getMapped() = new_list_itr;
+                snapshot_invalid_iters.push_back(list_itr);
             }
             else
             {
@@ -198,6 +192,7 @@ public:
         if (snapshot_mode)
         {
             list_itr->active_in_map = false;
+            snapshot_invalid_iters.push_back(list_itr);
             list_itr->free_key = true;
             map.erase(it->getKey());
         }
@@ -237,6 +232,7 @@ public:
             {
                 auto elem_copy = *(list_itr);
                 list_itr->active_in_map = false;
+                snapshot_invalid_iters.push_back(list_itr);
                 updater(elem_copy.value);
                 elem_copy.version = current_version;
                 auto itr = list.insert(list.end(), std::move(elem_copy));
@@ -277,23 +273,15 @@ public:
 
     void clearOutdatedNodes()
     {
-        auto start = list.begin();
+        for (auto & itr: snapshot_invalid_iters)
-        auto end = list.end();
-        for (auto itr = start; itr != end;)
         {
-            if (!itr->active_in_map)
+            assert(!itr->active_in_map);
-            {
+            updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size, itr->value.sizeInBytes(), 0);
-                updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size, itr->value.sizeInBytes(), 0);
+            if (itr->free_key)
-                if (itr->free_key)
+                arena.free(const_cast<char *>(itr->key.data), itr->key.size);
-                    arena.free(const_cast<char *>(itr->key.data), itr->key.size);
+            list.erase(itr);
-                itr = list.erase(itr);
-            }
-            else
-            {
-                assert(!itr->free_key);
-                itr++;
-            }
         }
+        snapshot_invalid_iters.clear();
     }
 
     void clear()

src/Core/Settings.h

@@ -358,11 +358,15 @@ class IColumn;
     M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
     \
     M(UInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \
+    M(UInt64, max_guaranteed_memory_usage, 0, "Maximum guaranteed memory usage for processing of single query. It represents soft limit. Zero means unlimited.", 0) \
     M(UInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \
+    M(UInt64, max_guaranteed_memory_usage_for_user, 0, "Maximum guaranteed memory usage for processing all concurrently running queries for the user. It represents soft limit. Zero means unlimited.", 0) \
     M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \
     M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \
     M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
     \
+    M(UInt64, memory_usage_overcommit_max_wait_microseconds, 0, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown", 0) \
+    \
     M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \
     M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \
     M(UInt64, max_network_bandwidth_for_user, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running user queries. Zero means unlimited.", 0)\
@@ -429,6 +433,7 @@ class IColumn;
     M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \
     \
     M(DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic, "Default database engine.", 0) \
+    M(DefaultTableEngine, default_table_engine, DefaultTableEngine::None, "Default table engine used when ENGINE is not set in CREATE statement.",0) \
     M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \
     M(Bool, database_atomic_wait_for_drop_and_detach_synchronously, false, "When executing DROP or DETACH TABLE in Atomic database, wait for table data to be finally dropped or detached.", 0) \
     M(Bool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \
@@ -482,7 +487,6 @@ class IColumn;
     M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \
     M(Bool, optimize_skip_merged_partitions, false, "Skip partitions with one part with level > 0 in optimize final", 0) \
     M(Bool, optimize_on_insert, true, "Do the same transformation for inserted block of data as if merge was done on this block.", 0) \
-    M(Bool, allow_experimental_projection_optimization, false, "Enable projection optimization when processing SELECT queries", 0) \
     M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \
     M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \
     M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \
@@ -547,7 +551,7 @@ class IColumn;
     M(Int64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \
     M(Int64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \
     \
-    M(UInt64, http_max_tries, 1, "Max attempts to read via http.", 0) \
+    M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \
     M(UInt64, http_retry_initial_backoff_ms, 100, "Min milliseconds for backoff, when retrying read via http", 0) \
     M(UInt64, http_retry_max_backoff_ms, 10000, "Max milliseconds for backoff, when retrying read via http", 0) \
     \
@@ -582,6 +586,7 @@ class IColumn;
     MAKE_OBSOLETE(M, UInt64, merge_tree_clear_old_parts_interval_seconds, 1) \
     MAKE_OBSOLETE(M, UInt64, partial_merge_join_optimizations, 0) \
     MAKE_OBSOLETE(M, MaxThreads, max_alter_threads, 0) \
+    MAKE_OBSOLETE(M, Bool, allow_experimental_projection_optimization, true) \
     /** The section above is for obsolete settings. Do not add anything there. */
 
 

src/Core/SettingsEnums.cpp

@@ -93,6 +93,16 @@ IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultDatabaseEngine, ErrorCodes::BAD_ARGUME
     {{"Ordinary", DefaultDatabaseEngine::Ordinary},
      {"Atomic",   DefaultDatabaseEngine::Atomic}})
 
+IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultTableEngine, ErrorCodes::BAD_ARGUMENTS,
+    {{"None", DefaultTableEngine::None},
+     {"Log", DefaultTableEngine::Log},
+     {"StripeLog", DefaultTableEngine::StripeLog},
+     {"MergeTree", DefaultTableEngine::MergeTree},
+     {"ReplacingMergeTree", DefaultTableEngine::ReplacingMergeTree},
+     {"ReplicatedMergeTree", DefaultTableEngine::ReplicatedMergeTree},
+     {"ReplicatedReplacingMergeTree", DefaultTableEngine::ReplicatedReplacingMergeTree},
+     {"Memory", DefaultTableEngine::Memory}})
+
 IMPLEMENT_SETTING_MULTI_ENUM(MySQLDataTypesSupport, ErrorCodes::UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL,
     {{"decimal",    MySQLDataTypesSupport::DECIMAL},
      {"datetime64", MySQLDataTypesSupport::DATETIME64}})

src/Core/SettingsEnums.h

@@ -120,6 +120,19 @@ enum class DefaultDatabaseEngine
 
 DECLARE_SETTING_ENUM(DefaultDatabaseEngine)
 
+enum class DefaultTableEngine
+{
+    None = 0, /// Disable. Need to use ENGINE =
+    Log,
+    StripeLog,
+    MergeTree,
+    ReplacingMergeTree,
+    ReplicatedMergeTree,
+    ReplicatedReplacingMergeTree,
+    Memory,
+};
+
+DECLARE_SETTING_ENUM(DefaultTableEngine)
 
 enum class MySQLDataTypesSupport
 {

src/Databases/DatabaseOnDisk.cpp

@@ -77,6 +77,10 @@ std::pair<String, StoragePtr> createTableFromAST(
         /// - the code is simpler, since the query is already brought to a suitable form.
         if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns)
         {
+            if (!ast_create_query.storage || !ast_create_query.storage->engine)
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid storage definition in metadata file: "
+                                                           "it's a bug or result of manual intervention in metadata files");
+
             if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(ast_create_query.storage->engine->name))
                 throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
             /// Leave columns empty.

src/Databases/MySQL/MaterializedMySQLSyncThread.cpp

@@ -316,7 +316,7 @@ getTableOutput(const String & database_name, const String & table_name, ContextM
     return std::move(res.pipeline);
 }
 
-static inline String reWriteMysqlQueryColumn(mysqlxx::Pool::Entry & connection, const String & database_name, const String & table_name, const Settings & global_settings)
+static inline String rewriteMysqlQueryColumn(mysqlxx::Pool::Entry & connection, const String & database_name, const String & table_name, const Settings & global_settings)
 {
     Block tables_columns_sample_block
             {
@@ -376,7 +376,7 @@ static inline void dumpDataForTables(
 
             auto pipeline = getTableOutput(database_name, table_name, query_context);
             StreamSettings mysql_input_stream_settings(context->getSettingsRef());
-            String mysql_select_all_query = "SELECT " + reWriteMysqlQueryColumn(connection, mysql_database_name, table_name, context->getSettings()) + " FROM "
+            String mysql_select_all_query = "SELECT " + rewriteMysqlQueryColumn(connection, mysql_database_name, table_name, context->getSettingsRef()) + " FROM "
                     + backQuoteIfNeed(mysql_database_name) + "." + backQuoteIfNeed(table_name);
             LOG_INFO(&Poco::Logger::get("MaterializedMySQLSyncThread(" + database_name + ")"), "mysql_select_all_query is {}", mysql_select_all_query);
             auto input = std::make_unique<MySQLSource>(connection, mysql_select_all_query, pipeline.getHeader(), mysql_input_stream_settings);

src/Dictionaries/CacheDictionaryStorage.h

@@ -8,10 +8,10 @@
 #include <Common/randomSeed.h>
 #include <Common/Arena.h>
 #include <Common/ArenaWithFreeLists.h>
+#include <Common/ArenaUtils.h>
 #include <Common/HashTable/LRUHashMap.h>
 #include <Dictionaries/DictionaryStructure.h>
 #include <Dictionaries/ICacheDictionaryStorage.h>
-#include <Dictionaries/DictionaryHelpers.h>
 
 
 namespace DB

src/Dictionaries/DictionaryHelpers.h

@@ -623,17 +623,6 @@ void mergeBlockWithPipe(
     }
 }
 
-template <typename Arena>
-static StringRef copyStringInArena(Arena & arena, StringRef value)
-{
-    size_t key_size = value.size;
-    char * place_for_key = arena.alloc(key_size);
-    memcpy(reinterpret_cast<void *>(place_for_key), reinterpret_cast<const void *>(value.data), key_size);
-    StringRef result{place_for_key, key_size};
-
-    return result;
-}
-
 /**
  * Returns ColumnVector data as PaddedPodArray.
 

src/Dictionaries/ExecutablePoolDictionarySource.cpp

@@ -197,7 +197,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory)
 
         size_t max_command_execution_time = config.getUInt64(settings_config_prefix + ".max_command_execution_time", 10);
 
-        size_t max_execution_time_seconds = static_cast<size_t>(context->getSettings().max_execution_time.totalSeconds());
+        size_t max_execution_time_seconds = static_cast<size_t>(context->getSettingsRef().max_execution_time.totalSeconds());
         if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds)
             max_command_execution_time = max_execution_time_seconds;
 

src/Dictionaries/FlatDictionary.cpp

@@ -3,6 +3,7 @@
 #include <Core/Defines.h>
 #include <Common/HashTable/HashMap.h>
 #include <Common/HashTable/HashSet.h>
+#include <Common/ArenaUtils.h>
 
 #include <DataTypes/DataTypesDecimal.h>
 #include <IO/WriteHelpers.h>
@@ -13,7 +14,7 @@
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <Processors/Executors/PullingPipelineExecutor.h>
 
-#include <Dictionaries//DictionarySource.h>
+#include <Dictionaries/DictionarySource.h>
 #include <Dictionaries/DictionaryFactory.h>
 #include <Dictionaries/HierarchyDictionariesUtils.h>
 

src/Dictionaries/HashedArrayDictionary.cpp

@@ -1,5 +1,6 @@
 #include "HashedArrayDictionary.h"
 
+#include <Common/ArenaUtils.h>
 #include <Core/Defines.h>
 #include <DataTypes/DataTypesDecimal.h>
 #include <Columns/ColumnsNumber.h>

src/Dictionaries/HashedDictionary.cpp

@@ -1,5 +1,6 @@
 #include "HashedDictionary.h"
 
+#include <Common/ArenaUtils.h>
 #include <Core/Defines.h>
 #include <DataTypes/DataTypesDecimal.h>
 #include <Columns/ColumnsNumber.h>

src/Dictionaries/RangeHashedDictionary.cpp

@@ -1,5 +1,7 @@
 #include <Dictionaries/RangeHashedDictionary.h>
 
+#include <Common/ArenaUtils.h>
+
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypesDecimal.h>
 #include <DataTypes/DataTypeEnum.h>

src/Dictionaries/SSDCacheDictionaryStorage.h

@@ -16,6 +16,7 @@
 #include <Common/randomSeed.h>
 #include <Common/Arena.h>
 #include <Common/ArenaWithFreeLists.h>
+#include <Common/ArenaUtils.h>
 #include <Common/MemorySanitizer.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/HashTable/HashMap.h>

src/Disks/IStoragePolicy.cpp

@@ -0,0 +1,34 @@
+#include <Disks/IStoragePolicy.h>
+#include <Common/quoteString.h>
+#include <Common/Exception.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int UNKNOWN_VOLUME;
+    extern const int UNKNOWN_DISK;
+}
+
+DiskPtr IStoragePolicy::getDiskByName(const String & disk_name) const
+{
+    auto disk = tryGetDiskByName(disk_name);
+    if (!disk)
+        throw Exception(ErrorCodes::UNKNOWN_DISK,
+            "No such disk {} in storage policy {}", backQuote(disk_name), backQuote(getName()));
+
+    return disk;
+}
+
+VolumePtr IStoragePolicy::getVolumeByName(const String & volume_name) const
+{
+    auto volume = tryGetVolumeByName(volume_name);
+    if (!volume)
+        throw Exception(ErrorCodes::UNKNOWN_VOLUME,
+            "No such volume {} in storage policy {}", backQuote(volume_name), backQuote(getName()));
+
+    return volume;
+}
+
+}

src/Disks/IStoragePolicy.h

@@ -39,7 +39,8 @@ public:
     /// Used when it's not important, for example for
     /// mutations files
     virtual DiskPtr getAnyDisk() const = 0;
-    virtual DiskPtr getDiskByName(const String & disk_name) const = 0;
+    virtual DiskPtr tryGetDiskByName(const String & disk_name) const = 0;
+    DiskPtr getDiskByName(const String & disk_name) const;
     /// Get free space from most free disk
     virtual UInt64 getMaxUnreservedFreeSpace() const = 0;
     /// Reserves space on any volume with index > min_volume_index or returns nullptr
@@ -53,7 +54,8 @@ public:
     virtual ReservationPtr makeEmptyReservationOnLargestDisk() const = 0;
     /// Get volume by index.
     virtual VolumePtr getVolume(size_t index) const = 0;
-    virtual VolumePtr getVolumeByName(const String & volume_name) const = 0;
+    virtual VolumePtr tryGetVolumeByName(const String & volume_name) const = 0;
+    VolumePtr getVolumeByName(const String & volume_name) const;
     /// Checks if storage policy can be replaced by another one.
     virtual void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const = 0;
     /// Find volume index, which contains disk

src/Disks/StoragePolicy.cpp

@@ -179,7 +179,7 @@ DiskPtr StoragePolicy::getAnyDisk() const
 }
 
 
-DiskPtr StoragePolicy::getDiskByName(const String & disk_name) const
+DiskPtr StoragePolicy::tryGetDiskByName(const String & disk_name) const
 {
     for (auto && volume : volumes)
         for (auto && disk : volume->getDisks())
@@ -265,11 +265,11 @@ VolumePtr StoragePolicy::getVolume(size_t index) const
 }
 
 
-VolumePtr StoragePolicy::getVolumeByName(const String & volume_name) const
+VolumePtr StoragePolicy::tryGetVolumeByName(const String & volume_name) const
 {
     auto it = volume_index_by_volume_name.find(volume_name);
     if (it == volume_index_by_volume_name.end())
-        throw Exception("No such volume " + backQuote(volume_name) + " in storage policy " + backQuote(name), ErrorCodes::UNKNOWN_VOLUME);
+        return nullptr;
     return getVolume(it->second);
 }
 

src/Disks/StoragePolicy.h

@@ -52,7 +52,7 @@ public:
     /// mutations files
     DiskPtr getAnyDisk() const override;
 
-    DiskPtr getDiskByName(const String & disk_name) const override;
+    DiskPtr tryGetDiskByName(const String & disk_name) const override;
 
     /// Get free space from most free disk
     UInt64 getMaxUnreservedFreeSpace() const override;
@@ -84,7 +84,7 @@ public:
     /// Get volume by index.
     VolumePtr getVolume(size_t index) const override;
 
-    VolumePtr getVolumeByName(const String & volume_name) const override;
+    VolumePtr tryGetVolumeByName(const String & volume_name) const override;
 
     /// Checks if storage policy can be replaced by another one.
     void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const override;

src/Functions/geoToH3.cpp

@@ -11,6 +11,7 @@
 #include <Common/typeid_cast.h>
 #include <base/range.h>
 
+#include <constants.h>
 #include <h3api.h>
 
 
@@ -20,6 +21,8 @@ namespace ErrorCodes
 {
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int INCORRECT_DATA;
+    extern const int ILLEGAL_COLUMN;
+    extern const int ARGUMENT_OUT_OF_BOUND;
 }
 
 namespace
@@ -68,9 +71,35 @@ public:
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
-        const auto * col_lon = arguments[0].column.get();
+        const auto * col_lon = checkAndGetColumn<ColumnFloat64>(arguments[0].column.get());
-        const auto * col_lat = arguments[1].column.get();
+        if (!col_lon)
-        const auto * col_res = arguments[2].column.get();
+            throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Illegal type {} of argument {} of function {}. Must be Float64.",
+                    arguments[0].type->getName(),
+                    1,
+                    getName());
+        const auto & data_lon = col_lon->getData();
+
+        const auto * col_lat = checkAndGetColumn<ColumnFloat64>(arguments[1].column.get());
+        if (!col_lat)
+            throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Illegal type {} of argument {} of function {}. Must be Float64.",
+                    arguments[1].type->getName(),
+                    2,
+                    getName());
+        const auto & data_lat = col_lat->getData();
+
+        const auto * col_res = checkAndGetColumn<ColumnUInt8>(arguments[2].column.get());
+        if (!col_res)
+            throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Illegal type {} of argument {} of function {}. Must be UInt8.",
+                    arguments[2].type->getName(),
+                    3,
+                    getName());
+        const auto & data_res = col_res->getData();
 
         auto dst = ColumnVector<UInt64>::create();
         auto & dst_data = dst->getData();
@@ -78,9 +107,17 @@ public:
 
         for (size_t row = 0; row < input_rows_count; ++row)
         {
-            const double lon = col_lon->getFloat64(row);
+            const double lon = data_lon[row];
-            const double lat = col_lat->getFloat64(row);
+            const double lat = data_lat[row];
-            const UInt8 res = col_res->getUInt(row);
+            const UInt8 res = data_res[row];
+
+            if (res > MAX_H3_RES)
+                throw Exception(
+                        ErrorCodes::ARGUMENT_OUT_OF_BOUND,
+                        "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ",
+                        toString(res),
+                        getName(),
+                        MAX_H3_RES);
 
             LatLng coord;
             coord.lng = degsToRads(lon);

src/IO/ReadWriteBufferFromHTTP.h

@@ -19,6 +19,7 @@
 #include <Common/DNSResolver.h>
 #include <Common/RemoteHostFilter.h>
 #include <Common/config.h>
+#include <Common/config_version.h>
 #include <base/logger_useful.h>
 #include <Poco/URIStreamFactory.h>
 
@@ -291,6 +292,18 @@ namespace detail
                                 "0 < http_retry_initial_backoff_ms < settings.http_retry_max_backoff_ms (now 0 < {} < {})",
                                 settings.http_max_tries, settings.http_retry_initial_backoff_ms, settings.http_retry_max_backoff_ms);
 
+            // Configure User-Agent if it not already set.
+            const std::string user_agent = "User-Agent";
+            auto iter = std::find_if(http_header_entries.begin(), http_header_entries.end(), [&user_agent](const HTTPHeaderEntry & entry)
+            {
+                return std::get<0>(entry) == user_agent;
+            });
+
+            if (iter == http_header_entries.end())
+            {
+                http_header_entries.emplace_back(std::make_pair("User-Agent", fmt::format("ClickHouse/{}", VERSION_STRING)));
+            }
+
             if (!delay_initialization)
             {
                 initialize();

src/IO/WriteBufferFromHTTP.cpp

@@ -10,6 +10,7 @@ WriteBufferFromHTTP::WriteBufferFromHTTP(
     const Poco::URI & uri,
     const std::string & method,
     const std::string & content_type,
+    const std::string & content_encoding,
     const ConnectionTimeouts & timeouts,
     size_t buffer_size_)
     : WriteBufferFromOStream(buffer_size_)
@@ -24,6 +25,9 @@ WriteBufferFromHTTP::WriteBufferFromHTTP(
         request.set("Content-Type", content_type);
     }
 
+    if (!content_encoding.empty())
+        request.set("Content-Encoding", content_encoding);
+
     LOG_TRACE((&Poco::Logger::get("WriteBufferToHTTP")), "Sending request to {}", uri.toString());
 
     ostr = &session->sendRequest(request);
@@ -31,6 +35,10 @@ WriteBufferFromHTTP::WriteBufferFromHTTP(
 
 void WriteBufferFromHTTP::finalizeImpl()
 {
+    // for compressed body, the data is stored in buffered first
+    // here, make sure the content in the buffer has been flushed
+    this->nextImpl();
+
     receiveResponse(*session, request, response, false);
     /// TODO: Response body is ignored.
 }

src/IO/WriteBufferFromHTTP.h

@@ -21,6 +21,7 @@ public:
     explicit WriteBufferFromHTTP(const Poco::URI & uri,
                                  const std::string & method = Poco::Net::HTTPRequest::HTTP_POST, // POST or PUT only
                                  const std::string & content_type = "",
+                                 const std::string & content_encoding = "",
                                  const ConnectionTimeouts & timeouts = {},
                                  size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
 

src/IO/tests/gtest_archive_reader_and_writer.cpp

@@ -328,14 +328,16 @@ TEST_P(ArchiveReaderAndWriterTest, ArchiveNotExist)
 }
 
 
+#if USE_MINIZIP
+
 namespace
 {
     const char * supported_archive_file_exts[] =
     {
-#if USE_MINIZIP
         ".zip",
-#endif
     };
 }
 
 INSTANTIATE_TEST_SUITE_P(All, ArchiveReaderAndWriterTest, ::testing::ValuesIn(supported_archive_file_exts));
+
+#endif

src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp

@@ -5,6 +5,7 @@
 #include <Parsers/formatAST.h>
 #include <Access/AccessControl.h>
 #include <Access/Common/AccessFlags.h>
+#include <Access/Common/AccessRightsElement.h>
 #include <Access/RowPolicy.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/executeDDLQueryOnCluster.h>
@@ -45,22 +46,24 @@ namespace
 BlockIO InterpreterCreateRowPolicyQuery::execute()
 {
     auto & query = query_ptr->as<ASTCreateRowPolicyQuery &>();
-    auto & access_control = getContext()->getAccessControl();
+    auto required_access = getRequiredAccess();
-    getContext()->checkAccess(query.alter ? AccessType::ALTER_ROW_POLICY : AccessType::CREATE_ROW_POLICY);
 
     if (!query.cluster.empty())
     {
         query.replaceCurrentUserTag(getContext()->getUserName());
-        return executeDDLQueryOnCluster(query_ptr, getContext());
+        return executeDDLQueryOnCluster(query_ptr, getContext(), required_access);
     }
 
     assert(query.names->cluster.empty());
+    auto & access_control = getContext()->getAccessControl();
+    getContext()->checkAccess(required_access);
+
+    query.replaceEmptyDatabase(getContext()->getCurrentDatabase());
+
     std::optional<RolesOrUsersSet> roles_from_query;
     if (query.roles)
         roles_from_query = RolesOrUsersSet{*query.roles, access_control, getContext()->getUserID()};
 
-    query.replaceEmptyDatabase(getContext()->getCurrentDatabase());
-
     if (query.alter)
     {
         auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
@@ -105,4 +108,15 @@ void InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(RowPolicy & polic
     updateRowPolicyFromQueryImpl(policy, query, {}, {});
 }
 
+
+AccessRightsElements InterpreterCreateRowPolicyQuery::getRequiredAccess() const
+{
+    const auto & query = query_ptr->as<const ASTCreateRowPolicyQuery &>();
+    AccessRightsElements res;
+    auto access_type = (query.alter ? AccessType::ALTER_ROW_POLICY : AccessType::CREATE_ROW_POLICY);
+    for (const auto & row_policy_name : query.names->full_names)
+        res.emplace_back(access_type, row_policy_name.database, row_policy_name.table_name);
+    return res;
+}
+
 }

src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h

@@ -6,8 +6,8 @@
 
 namespace DB
 {
-
 class ASTCreateRowPolicyQuery;
+class AccessRightsElements;
 struct RowPolicy;
 
 class InterpreterCreateRowPolicyQuery : public IInterpreter, WithMutableContext
@@ -20,6 +20,8 @@ public:
     static void updateRowPolicyFromQuery(RowPolicy & policy, const ASTCreateRowPolicyQuery & query);
 
 private:
+    AccessRightsElements getRequiredAccess() const;
+
     ASTPtr query_ptr;
 };
 

src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp

@@ -49,12 +49,37 @@ AccessRightsElements InterpreterDropAccessEntityQuery::getRequiredAccess() const
     AccessRightsElements res;
     switch (query.type)
     {
-        case AccessEntityType::USER: res.emplace_back(AccessType::DROP_USER); return res;
+        case AccessEntityType::USER:
-        case AccessEntityType::ROLE: res.emplace_back(AccessType::DROP_ROLE); return res;
+        {
-        case AccessEntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::DROP_SETTINGS_PROFILE); return res;
+            res.emplace_back(AccessType::DROP_USER);
-        case AccessEntityType::ROW_POLICY: res.emplace_back(AccessType::DROP_ROW_POLICY); return res;
+            return res;
-        case AccessEntityType::QUOTA: res.emplace_back(AccessType::DROP_QUOTA); return res;
+        }
-        case AccessEntityType::MAX: break;
+        case AccessEntityType::ROLE:
+        {
+            res.emplace_back(AccessType::DROP_ROLE);
+            return res;
+        }
+        case AccessEntityType::SETTINGS_PROFILE:
+        {
+            res.emplace_back(AccessType::DROP_SETTINGS_PROFILE);
+            return res;
+        }
+        case AccessEntityType::ROW_POLICY:
+        {
+            if (query.row_policy_names)
+            {
+                for (const auto & row_policy_name : query.row_policy_names->full_names)
+                    res.emplace_back(AccessType::DROP_ROW_POLICY, row_policy_name.database, row_policy_name.table_name);
+            }
+            return res;
+        }
+        case AccessEntityType::QUOTA:
+        {
+            res.emplace_back(AccessType::DROP_QUOTA);
+            return res;
+        }
+        case AccessEntityType::MAX:
+            break;
     }
     throw Exception(
         toString(query.type) + ": type is not supported by DROP query", ErrorCodes::NOT_IMPLEMENTED);

src/Interpreters/Access/InterpreterDropAccessEntityQuery.h

@@ -6,7 +6,6 @@
 
 namespace DB
 {
-
 class AccessRightsElements;
 
 class InterpreterDropAccessEntityQuery : public IInterpreter, WithMutableContext

src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp

@@ -377,12 +377,48 @@ AccessRightsElements InterpreterShowCreateAccessEntityQuery::getRequiredAccess()
     AccessRightsElements res;
     switch (show_query.type)
     {
-        case AccessEntityType::USER: res.emplace_back(AccessType::SHOW_USERS); return res;
+        case AccessEntityType::USER:
-        case AccessEntityType::ROLE: res.emplace_back(AccessType::SHOW_ROLES); return res;
+        {
-        case AccessEntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); return res;
+            res.emplace_back(AccessType::SHOW_USERS);
-        case AccessEntityType::ROW_POLICY: res.emplace_back(AccessType::SHOW_ROW_POLICIES); return res;
+            return res;
-        case AccessEntityType::QUOTA: res.emplace_back(AccessType::SHOW_QUOTAS); return res;
+        }
-        case AccessEntityType::MAX: break;
+        case AccessEntityType::ROLE:
+        {
+            res.emplace_back(AccessType::SHOW_ROLES);
+            return res;
+        }
+        case AccessEntityType::SETTINGS_PROFILE:
+        {
+            res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES);
+            return res;
+        }
+        case AccessEntityType::ROW_POLICY:
+        {
+            if (show_query.row_policy_names)
+            {
+                for (const auto & row_policy_name : show_query.row_policy_names->full_names)
+                    res.emplace_back(AccessType::SHOW_ROW_POLICIES, row_policy_name.database, row_policy_name.table_name);
+            }
+            else if (show_query.database_and_table_name)
+            {
+                if (show_query.database_and_table_name->second.empty())
+                    res.emplace_back(AccessType::SHOW_ROW_POLICIES, show_query.database_and_table_name->first);
+                else
+                    res.emplace_back(AccessType::SHOW_ROW_POLICIES, show_query.database_and_table_name->first, show_query.database_and_table_name->second);
+            }
+            else
+            {
+                res.emplace_back(AccessType::SHOW_ROW_POLICIES);
+            }
+            return res;
+        }
+        case AccessEntityType::QUOTA:
+        {
+            res.emplace_back(AccessType::SHOW_QUOTAS);
+            return res;
+        }
+        case AccessEntityType::MAX:
+            break;
     }
     throw Exception(toString(show_query.type) + ": type is not supported by SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED);
 }

src/Interpreters/ClusterDiscovery.cpp

@@ -229,7 +229,7 @@ ClusterPtr ClusterDiscovery::makeCluster(const ClusterInfo & cluster_info)
 
     bool secure = cluster_info.current_node.secure;
     auto cluster = std::make_shared<Cluster>(
-        context->getSettings(),
+        context->getSettingsRef(),
         shards,
         /* username= */ context->getUserName(),
         /* password= */ "",

src/Interpreters/Context.cpp

@@ -208,6 +208,7 @@ struct ContextSharedPart
     mutable MarkCachePtr index_mark_cache;                  /// Cache of marks in compressed files of MergeTree indices.
     mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads.
     ProcessList process_list;                               /// Executing queries at the moment.
+    GlobalOvercommitTracker global_overcommit_tracker;
     MergeList merge_list;                                   /// The list of executable merge (for (Replicated)?MergeTree)
     ReplicatedFetchList replicated_fetch_list;
     ConfigurationPtr users_config;                          /// Config with the users, profiles and quotas sections.
@@ -275,7 +276,9 @@ struct ContextSharedPart
     Context::ConfigReloadCallback config_reload_callback;
 
     ContextSharedPart()
-        : access_control(std::make_unique<AccessControl>()), macros(std::make_unique<Macros>())
+        : access_control(std::make_unique<AccessControl>())
+        , global_overcommit_tracker(&process_list)
+        , macros(std::make_unique<Macros>())
     {
         /// TODO: make it singleton (?)
         static std::atomic<size_t> num_calls{0};
@@ -474,6 +477,7 @@ std::unique_lock<std::recursive_mutex> Context::getLock() const
 
 ProcessList & Context::getProcessList() { return shared->process_list; }
 const ProcessList & Context::getProcessList() const { return shared->process_list; }
+OvercommitTracker * Context::getGlobalOvercommitTracker() const { return &shared->global_overcommit_tracker; }
 MergeList & Context::getMergeList() { return shared->merge_list; }
 const MergeList & Context::getMergeList() const { return shared->merge_list; }
 ReplicatedFetchList & Context::getReplicatedFetchList() { return shared->replicated_fetch_list; }

src/Interpreters/Context.h

@@ -29,6 +29,7 @@
 namespace Poco::Net { class IPAddress; }
 namespace zkutil { class ZooKeeper; }
 
+struct OvercommitTracker;
 
 namespace DB
 {
@@ -657,6 +658,8 @@ public:
     ProcessList & getProcessList();
     const ProcessList & getProcessList() const;
 
+    OvercommitTracker * getGlobalOvercommitTracker() const;
+
     MergeList & getMergeList();
     const MergeList & getMergeList() const;
 

src/Interpreters/InterpreterCreateQuery.cpp

@@ -2,6 +2,7 @@
 
 #include <filesystem>
 
+#include "Common/Exception.h"
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/escapeForFileName.h>
 #include <Common/typeid_cast.h>
@@ -12,6 +13,7 @@
 
 #include <Core/Defines.h>
 #include <Core/Settings.h>
+#include <Core/SettingsEnums.h>
 
 #include <IO/WriteBufferFromFile.h>
 #include <IO/WriteHelpers.h>
@@ -91,6 +93,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_DATABASE;
     extern const int PATH_ACCESS_DENIED;
     extern const int NOT_IMPLEMENTED;
+    extern const int ENGINE_REQUIRED;
 }
 
 namespace fs = std::filesystem;
@@ -157,6 +160,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
         throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", serializeAST(*create.storage));
     }
 
+    if (create.storage && !create.storage->engine)
+        throw Exception(ErrorCodes::INCORRECT_QUERY, "Database engine must be specified");
+
     if (create.storage->engine->name == "Atomic"
         || create.storage->engine->name == "Replicated"
         || create.storage->engine->name == "MaterializedPostgreSQL")
@@ -581,6 +587,17 @@ ConstraintsDescription InterpreterCreateQuery::getConstraintsDescription(const A
 
 InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create) const
 {
+    /// Set the table engine if it was not specified explicitly.
+    setEngine(create);
+
+    /// We have to check access rights again (in case engine was changed).
+    if (create.storage)
+    {
+        auto source_access_type = StorageFactory::instance().getSourceAccessType(create.storage->engine->name);
+        if (source_access_type != AccessType::NONE)
+            getContext()->checkAccess(source_access_type);
+    }
+
     TableProperties properties;
     TableLockHolder as_storage_lock;
 
@@ -645,7 +662,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
     }
     /// We can have queries like "CREATE TABLE <table> ENGINE=<engine>" if <engine>
     /// supports schema inference (will determine table structure in it's constructor).
-    else if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(create.storage->engine->name))
+    else if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(create.storage->engine->name)) // NOLINT
         throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY);
 
     /// Even if query has list of columns, canonicalize it (unfold Nested columns).
@@ -663,8 +680,6 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
     create.columns_list->setOrReplace(create.columns_list->projections, new_projections);
 
     validateTableStructure(create, properties);
-    /// Set the table engine if it was not specified explicitly.
-    setEngine(create);
 
     assert(as_database_saved.empty() && as_table_saved.empty());
     std::swap(create.as_database, as_database_saved);
@@ -718,30 +733,90 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
     }
 }
 
+String InterpreterCreateQuery::getTableEngineName(DefaultTableEngine default_table_engine)
+{
+    switch (default_table_engine)
+    {
+        case DefaultTableEngine::Log:
+            return "Log";
+
+        case DefaultTableEngine::StripeLog:
+            return "StripeLog";
+
+        case DefaultTableEngine::MergeTree:
+            return "MergeTree";
+
+        case DefaultTableEngine::ReplacingMergeTree:
+            return "ReplacingMergeTree";
+
+        case DefaultTableEngine::ReplicatedMergeTree:
+            return "ReplicatedMergeTree";
+
+        case DefaultTableEngine::ReplicatedReplacingMergeTree:
+            return "ReplicatedReplacingMergeTree";
+
+        case DefaultTableEngine::Memory:
+            return "Memory";
+
+        default:
+            throw Exception("default_table_engine is set to unknown value", ErrorCodes::LOGICAL_ERROR);
+    }
+}
+
+void InterpreterCreateQuery::setDefaultTableEngine(ASTStorage & storage, ContextPtr local_context)
+{
+    if (local_context->getSettingsRef().default_table_engine.value == DefaultTableEngine::None)
+        throw Exception(ErrorCodes::ENGINE_REQUIRED, "Table engine is not specified in CREATE query");
+
+    auto engine_ast = std::make_shared<ASTFunction>();
+    auto default_table_engine = local_context->getSettingsRef().default_table_engine.value;
+    engine_ast->name = getTableEngineName(default_table_engine);
+    engine_ast->no_empty_args = true;
+    storage.set(storage.engine, engine_ast);
+}
+
 void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
 {
     if (create.as_table_function)
         return;
 
-    if (create.storage || create.is_dictionary || create.isView())
+    if (create.is_dictionary || create.is_ordinary_view || create.is_live_view || create.is_window_view)
-    {
+        return;
-        if (create.temporary && create.storage && create.storage->engine && create.storage->engine->name != "Memory")
+
-            throw Exception(ErrorCodes::INCORRECT_QUERY,
+    if (create.is_materialized_view && create.to_table_id)
-                "Temporary tables can only be created with ENGINE = Memory, not {}", create.storage->engine->name);
-
         return;
-    }
 
     if (create.temporary)
     {
+        if (create.storage && create.storage->engine && create.storage->engine->name != "Memory")
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables can only be created with ENGINE = Memory, not {}",
+                create.storage->engine->name);
+
+        /// It's possible if some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not.
+        /// It makes sense when default_table_engine setting is used, but not for temporary tables.
+        /// For temporary tables we ignore this setting to allow CREATE TEMPORARY TABLE query without specifying ENGINE
+        /// even if setting is set to MergeTree or something like that (otherwise MergeTree will be substituted and query will fail).
+        if (create.storage && !create.storage->engine)
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Invalid storage definition for temporary table: must be either ENGINE = Memory or empty");
+
         auto engine_ast = std::make_shared<ASTFunction>();
         engine_ast->name = "Memory";
         engine_ast->no_empty_args = true;
         auto storage_ast = std::make_shared<ASTStorage>();
         storage_ast->set(storage_ast->engine, engine_ast);
         create.set(create.storage, storage_ast);
+        return;
     }
-    else if (!create.as_table.empty())
+
+    if (create.storage)
+    {
+        /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
+        if (!create.storage->engine)
+            setDefaultTableEngine(*create.storage, getContext());
+        return;
+    }
+
+    if (!create.as_table.empty())
     {
         /// NOTE Getting the structure from the table specified in the AS is done not atomically with the creation of the table.
 
@@ -754,24 +829,16 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name);
 
         if (as_create.is_ordinary_view)
-            throw Exception(
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a View", qualified_name);
-                "Cannot CREATE a table AS " + qualified_name + ", it is a View",
-                ErrorCodes::INCORRECT_QUERY);
 
         if (as_create.is_live_view)
-            throw Exception(
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Live View", qualified_name);
-                "Cannot CREATE a table AS " + qualified_name + ", it is a Live View",
-                ErrorCodes::INCORRECT_QUERY);
 
         if (as_create.is_window_view)
-            throw Exception(
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Window View", qualified_name);
-                "Cannot CREATE a table AS " + qualified_name + ", it is a Window View",
-                ErrorCodes::INCORRECT_QUERY);
 
         if (as_create.is_dictionary)
-            throw Exception(
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Dictionary", qualified_name);
-                "Cannot CREATE a table AS " + qualified_name + ", it is a Dictionary",
-                ErrorCodes::INCORRECT_QUERY);
 
         if (as_create.storage)
             create.set(create.storage, as_create.storage->ptr());
@@ -779,7 +846,12 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
             create.as_table_function = as_create.as_table_function->clone();
         else
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot set engine, it's a bug.");
+
+        return;
     }
+
+    create.set(create.storage, std::make_shared<ASTStorage>());
+    setDefaultTableEngine(*create.storage, getContext());
 }
 
 static void generateUUIDForTable(ASTCreateQuery & create)

src/Interpreters/InterpreterCreateQuery.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Core/NamesAndAliases.h>
+#include <Core/SettingsEnums.h>
 #include <Access/Common/AccessRightsElement.h>
 #include <Interpreters/IInterpreter.h>
 #include <Storages/ColumnsDescription.h>
@@ -15,6 +16,7 @@ namespace DB
 class ASTCreateQuery;
 class ASTExpressionList;
 class ASTConstraintDeclaration;
+class ASTStorage;
 class IDatabase;
 using DatabasePtr = std::shared_ptr<IDatabase>;
 
@@ -81,6 +83,8 @@ private:
     /// Calculate list of columns, constraints, indices, etc... of table. Rewrite query in canonical way.
     TableProperties getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create) const;
     void validateTableStructure(const ASTCreateQuery & create, const TableProperties & properties) const;
+    static String getTableEngineName(DefaultTableEngine default_table_engine);
+    static void setDefaultTableEngine(ASTStorage & storage, ContextPtr local_context);
     void setEngine(ASTCreateQuery & create) const;
     AccessRightsElements getRequiredAccess() const;
 

src/Interpreters/ProcessList.cpp

@@ -195,33 +195,21 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
                     ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING);
         }
 
-        auto process_it = processes.emplace(processes.end(),
-            query_context, query_, client_info, priorities.insert(settings.priority), query_kind);
-
-        increaseQueryKindAmount(query_kind);
-
-        res = std::make_shared<Entry>(*this, process_it);
-
         ProcessListForUser & user_process_list = user_to_queries[client_info.current_user];
-        user_process_list.queries.emplace(client_info.current_query_id, &res->get());
-
-        process_it->setUserProcessList(&user_process_list);
-
-        /// Track memory usage for all simultaneously running queries from single user.
-        user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user);
-        user_process_list.user_memory_tracker.setDescription("(for user)");
 
         /// Actualize thread group info
-        if (auto thread_group = CurrentThread::getGroup())
+        auto thread_group = CurrentThread::getGroup();
+        if (thread_group)
         {
             std::lock_guard lock_thread_group(thread_group->mutex);
             thread_group->performance_counters.setParent(&user_process_list.user_performance_counters);
             thread_group->memory_tracker.setParent(&user_process_list.user_memory_tracker);
-            thread_group->query = process_it->query;
+            thread_group->query = query_;
-            thread_group->normalized_query_hash = normalizedQueryHash<false>(process_it->query);
+            thread_group->normalized_query_hash = normalizedQueryHash<false>(query_);
 
             /// Set query-level memory trackers
             thread_group->memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage);
+            thread_group->memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage);
 
             if (query_context->hasTraceCollector())
             {
@@ -236,10 +224,28 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
 
             /// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values
             ///  since allocation and deallocation could happen in different threads
-
-            process_it->thread_group = std::move(thread_group);
         }
 
+        auto process_it = processes.emplace(processes.end(),
+            query_context, query_, client_info, priorities.insert(settings.priority), std::move(thread_group), query_kind);
+
+        increaseQueryKindAmount(query_kind);
+
+        res = std::make_shared<Entry>(*this, process_it);
+
+        process_it->setUserProcessList(&user_process_list);
+
+        {
+            BlockQueryIfMemoryLimit block_query{user_process_list.user_overcommit_tracker};
+            user_process_list.queries.emplace(client_info.current_query_id, &res->get());
+        }
+
+        /// Track memory usage for all simultaneously running queries from single user.
+        user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user);
+        user_process_list.user_memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage_for_user);
+        user_process_list.user_memory_tracker.setDescription("(for user)");
+        user_process_list.user_overcommit_tracker.setMaxWaitTime(settings.memory_usage_overcommit_max_wait_microseconds);
+
         if (!user_process_list.user_throttler)
         {
             if (settings.max_network_bandwidth_for_user)
@@ -268,9 +274,6 @@ ProcessListEntry::~ProcessListEntry()
 
     const QueryStatus * process_list_element_ptr = &*it;
 
-    /// This removes the memory_tracker of one request.
-    parent.processes.erase(it);
-
     auto user_process_list_it = parent.user_to_queries.find(user);
     if (user_process_list_it == parent.user_to_queries.end())
     {
@@ -286,11 +289,15 @@ ProcessListEntry::~ProcessListEntry()
     {
         if (running_query->second == process_list_element_ptr)
         {
+            BlockQueryIfMemoryLimit block_query{user_process_list.user_overcommit_tracker};
             user_process_list.queries.erase(running_query->first);
             found = true;
         }
     }
 
+    /// This removes the memory_tracker of one request.
+    parent.processes.erase(it);
+
     if (!found)
     {
         LOG_ERROR(&Poco::Logger::get("ProcessList"), "Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser");
@@ -312,10 +319,16 @@ ProcessListEntry::~ProcessListEntry()
 
 
 QueryStatus::QueryStatus(
-    ContextPtr context_, const String & query_, const ClientInfo & client_info_, QueryPriorities::Handle && priority_handle_, IAST::QueryKind query_kind_)
+    ContextPtr context_,
+    const String & query_,
+    const ClientInfo & client_info_,
+    QueryPriorities::Handle && priority_handle_,
+    ThreadGroupStatusPtr && thread_group_,
+    IAST::QueryKind query_kind_)
     : WithContext(context_)
     , query(query_)
     , client_info(client_info_)
+    , thread_group(std::move(thread_group_))
     , priority_handle(std::move(priority_handle_))
     , query_kind(query_kind_)
     , num_queries_increment(CurrentMetrics::Query)
@@ -328,6 +341,14 @@ QueryStatus::QueryStatus(
 QueryStatus::~QueryStatus()
 {
     assert(executors.empty());
+
+    if (auto * memory_tracker = getMemoryTracker())
+    {
+        if (user_process_list)
+            user_process_list->user_overcommit_tracker.unsubscribe(memory_tracker);
+        if (auto shared_context = getContext())
+            shared_context->getGlobalOvercommitTracker()->unsubscribe(memory_tracker);
+    }
 }
 
 CancellationCode QueryStatus::cancelQuery(bool)
@@ -481,7 +502,11 @@ ProcessList::Info ProcessList::getInfo(bool get_thread_list, bool get_profile_ev
 }
 
 
-ProcessListForUser::ProcessListForUser() = default;
+ProcessListForUser::ProcessListForUser()
+    : user_overcommit_tracker(this)
+{
+    user_memory_tracker.setOvercommitTracker(&user_overcommit_tracker);
+}
 
 
 ProcessListForUserInfo ProcessListForUser::getInfo(bool get_profile_events) const

src/Interpreters/ProcessList.h

@@ -16,6 +16,7 @@
 #include <Common/ProfileEvents.h>
 #include <Common/Stopwatch.h>
 #include <Common/Throttler.h>
+#include <Common/OvercommitTracker.h>
 
 #include <condition_variable>
 #include <list>
@@ -76,6 +77,7 @@ protected:
     friend class ThreadStatus;
     friend class CurrentThread;
     friend class ProcessListEntry;
+    friend struct ::GlobalOvercommitTracker;
 
     String query;
     ClientInfo client_info;
@@ -132,6 +134,7 @@ public:
         const String & query_,
         const ClientInfo & client_info_,
         QueryPriorities::Handle && priority_handle_,
+        ThreadGroupStatusPtr && thread_group_,
         IAST::QueryKind query_kind_
         );
 
@@ -154,6 +157,13 @@ public:
 
     ThrottlerPtr getUserNetworkThrottler();
 
+    MemoryTracker * getMemoryTracker() const
+    {
+        if (!thread_group)
+            return nullptr;
+        return &thread_group->memory_tracker;
+    }
+
     bool updateProgressIn(const Progress & value)
     {
         CurrentThread::updateProgressIn(value);
@@ -216,6 +226,8 @@ struct ProcessListForUser
     /// Limit and counter for memory of all simultaneously running queries of single user.
     MemoryTracker user_memory_tracker{VariableContext::User};
 
+    UserOvercommitTracker user_overcommit_tracker;
+
     /// Count network usage for all simultaneously running queries of single user.
     ThrottlerPtr user_throttler;
 
@@ -337,6 +349,14 @@ public:
         max_size = max_size_;
     }
 
+    template <typename F>
+    void processEachQueryStatus(F && func) const
+    {
+        std::lock_guard lk(mutex);
+        for (auto && query : processes)
+            func(query);
+    }
+
     void setMaxInsertQueriesAmount(size_t max_insert_queries_amount_)
     {
         std::lock_guard lock(mutex);

src/Parsers/ASTTTLElement.cpp

@@ -1,13 +1,17 @@
-
 #include <Columns/Collator.h>
 #include <Common/quoteString.h>
 #include <Parsers/ASTTTLElement.h>
 #include <IO/Operators.h>
-
+#include <base/EnumReflection.h>
 
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
 ASTPtr ASTTTLElement::clone() const
 {
     auto clone = std::make_shared<ASTTTLElement>(*this);
@@ -29,13 +33,21 @@ ASTPtr ASTTTLElement::clone() const
 void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     ttl()->formatImpl(settings, state, frame);
-    if (mode == TTLMode::MOVE && destination_type == DataDestinationType::DISK)
+    if (mode == TTLMode::MOVE)
     {
-        settings.ostr << " TO DISK " << quoteString(destination_name);
+        if (destination_type == DataDestinationType::DISK)
-    }
+            settings.ostr << " TO DISK ";
-    else if (mode == TTLMode::MOVE && destination_type == DataDestinationType::VOLUME)
+        else if (destination_type == DataDestinationType::VOLUME)
-    {
+            settings.ostr << " TO VOLUME ";
-        settings.ostr << " TO VOLUME " << quoteString(destination_name);
+        else
+            throw Exception(ErrorCodes::LOGICAL_ERROR,
+                "Unsupported destination type {} for TTL MOVE",
+                    magic_enum::enum_name(destination_type));
+
+        if (if_exists)
+            settings.ostr << "IF EXISTS ";
+
+        settings.ostr << quoteString(destination_name);
     }
     else if (mode == TTLMode::GROUP_BY)
     {

src/Parsers/ASTTTLElement.h

@@ -16,16 +16,18 @@ public:
     TTLMode mode;
     DataDestinationType destination_type;
     String destination_name;
+    bool if_exists = false;
 
     ASTs group_by_key;
     ASTs group_by_assignments;
 
     ASTPtr recompression_codec;
 
-    ASTTTLElement(TTLMode mode_, DataDestinationType destination_type_, const String & destination_name_)
+    ASTTTLElement(TTLMode mode_, DataDestinationType destination_type_, const String & destination_name_, bool if_exists_)
         : mode(mode_)
         , destination_type(destination_type_)
         , destination_name(destination_name_)
+        , if_exists(if_exists_)
         , ttl_expr_pos(-1)
         , where_expr_pos(-1)
     {

src/Parsers/ASTTablesInSelectQuery.cpp

@@ -247,10 +247,12 @@ void ASTTableJoin::formatImpl(const FormatSettings & settings, FormatState & sta
 
 void ASTArrayJoin::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
+    std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' ');
     frame.expression_list_prepend_whitespace = true;
 
     settings.ostr << (settings.hilite ? hilite_keyword : "")
         << settings.nl_or_ws
+        << indent_str
         << (kind == Kind::Left ? "LEFT " : "") << "ARRAY JOIN" << (settings.hilite ? hilite_none : "");
 
     settings.one_line

src/Parsers/ExpressionElementParsers.cpp

@@ -2360,6 +2360,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
     ParserKeyword s_to_disk("TO DISK");
     ParserKeyword s_to_volume("TO VOLUME");
+    ParserKeyword s_if_exists("IF EXISTS");
     ParserKeyword s_delete("DELETE");
     ParserKeyword s_where("WHERE");
     ParserKeyword s_group_by("GROUP BY");
@@ -2414,9 +2415,13 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ASTPtr group_by_key;
     ASTPtr recompression_codec;
     ASTPtr group_by_assignments;
+    bool if_exists = false;
 
     if (mode == TTLMode::MOVE)
     {
+        if (s_if_exists.ignore(pos))
+            if_exists = true;
+
         ASTPtr ast_space_name;
         if (!parser_string_literal.parse(pos, ast_space_name, expected))
             return false;
@@ -2448,7 +2453,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             return false;
     }
 
-    auto ttl_element = std::make_shared<ASTTTLElement>(mode, destination_type, destination_name);
+    auto ttl_element = std::make_shared<ASTTTLElement>(mode, destination_type, destination_name, if_exists);
     ttl_element->setTTL(std::move(ttl_expr));
     if (where_expr)
         ttl_element->setWhere(std::move(where_expr));

src/Parsers/ParserCreateQuery.cpp

@@ -18,6 +18,7 @@
 #include <Parsers/ParserSelectWithUnionQuery.h>
 #include <Parsers/ParserSetQuery.h>
 #include <Common/typeid_cast.h>
+#include <Parsers/ASTColumnDeclaration.h>
 
 
 namespace DB
@@ -353,20 +354,26 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ASTPtr ttl_table;
     ASTPtr settings;
 
-    if (!s_engine.ignore(pos, expected))
+    bool storage_like = false;
-        return false;
 
-    s_eq.ignore(pos, expected);
+    if (s_engine.ignore(pos, expected))
+    {
+        s_eq.ignore(pos, expected);
 
-    if (!ident_with_optional_params_p.parse(pos, engine, expected))
+        if (!ident_with_optional_params_p.parse(pos, engine, expected))
-        return false;
+            return false;
+        storage_like = true;
+    }
 
     while (true)
     {
         if (!partition_by && s_partition_by.ignore(pos, expected))
         {
             if (expression_p.parse(pos, partition_by, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -374,7 +381,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (!primary_key && s_primary_key.ignore(pos, expected))
         {
             if (expression_p.parse(pos, primary_key, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -382,7 +392,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (!order_by && s_order_by.ignore(pos, expected))
         {
             if (expression_p.parse(pos, order_by, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -390,7 +403,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (!sample_by && s_sample_by.ignore(pos, expected))
         {
             if (expression_p.parse(pos, sample_by, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -398,7 +414,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (!ttl_table && s_ttl.ignore(pos, expected))
         {
             if (parser_ttl_list.parse(pos, ttl_table, expected))
+            {
+                storage_like = true;
                 continue;
+            }
             else
                 return false;
         }
@@ -407,10 +426,14 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         {
             if (!settings_p.parse(pos, settings, expected))
                 return false;
+            storage_like = true;
         }
 
         break;
     }
+    // If any part of storage definition is found create storage node
+    if (!storage_like)
+        return false;
 
     auto storage = std::make_shared<ASTStorage>();
     storage->set(storage->engine, engine);
@@ -549,13 +572,11 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
 
         if (!storage_parse_result && !is_temporary)
         {
-            if (!s_as.ignore(pos, expected))
+            if (s_as.ignore(pos, expected) && !table_function_p.parse(pos, as_table_function, expected))
                 return false;
-            if (!table_function_p.parse(pos, as_table_function, expected))
-            {
-                return false;
-            }
         }
+
+        /// Will set default table engine if Storage clause was not parsed
     }
     /** Create queries without list of columns:
       *  - CREATE|ATTACH TABLE ... AS ...
@@ -590,10 +611,6 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
                 }
             }
         }
-        else if (!storage)
-        {
-            return false;
-        }
     }
     auto comment = parseComment(pos, expected);
 
@@ -625,12 +642,14 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     if (comment)
         query->set(query->comment, comment);
 
-    if (query->storage && query->columns_list && query->columns_list->primary_key)
+    if (query->columns_list && query->columns_list->primary_key)
     {
-        if (query->storage->primary_key)
+        /// If engine is not set will use default one
-        {
+        if (!query->storage)
+            query->set(query->storage, std::make_shared<ASTStorage>());
+        else if (query->storage->primary_key)
             throw Exception("Multiple primary keys are not allowed.", ErrorCodes::BAD_ARGUMENTS);
-        }
+
         query->storage->primary_key = query->columns_list->primary_key;
     }
 
@@ -1263,8 +1282,8 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     if (is_materialized_view && !to_table)
     {
         /// Internal ENGINE for MATERIALIZED VIEW must be specified.
-        if (!storage_p.parse(pos, storage, expected))
+        /// Actually check it in Interpreter as default_table_engine can be set
-            return false;
+        storage_p.parse(pos, storage, expected);
 
         if (s_populate.ignore(pos, expected))
             is_populate = true;

src/Parsers/ParserCreateQuery.h

@@ -276,7 +276,7 @@ protected:
 class ParserIndexDeclaration : public IParserBase
 {
 public:
-    ParserIndexDeclaration() {}
+    ParserIndexDeclaration() = default;
 
 protected:
     const char * getName() const override { return "index declaration"; }
@@ -336,7 +336,7 @@ protected:
 
 
 /**
-  * ENGINE = name [PARTITION BY expr] [ORDER BY expr] [PRIMARY KEY expr] [SAMPLE BY expr] [SETTINGS name = value, ...]
+  * [ENGINE = name] [PARTITION BY expr] [ORDER BY expr] [PRIMARY KEY expr] [SAMPLE BY expr] [SETTINGS name = value, ...]
   */
 class ParserStorage : public IParserBase
 {
@@ -391,7 +391,7 @@ class ParserTableOverrideDeclaration : public IParserBase
 {
 public:
     const bool is_standalone;
-    ParserTableOverrideDeclaration(bool is_standalone_ = true) : is_standalone(is_standalone_) { }
+    explicit ParserTableOverrideDeclaration(bool is_standalone_ = true) : is_standalone(is_standalone_) { }
 
 protected:
     const char * getName() const override { return "table override declaration"; }

src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp

@@ -356,6 +356,7 @@ void registerInputFormatJSONEachRow(FormatFactory & factory)
     });
 
     factory.registerFileExtension("ndjson", "JSONEachRow");
+    factory.registerFileExtension("jsonl", "JSONEachRow");
 
     factory.registerInputFormat("JSONStringsEachRow", [](
         ReadBuffer & buf,

src/Processors/Formats/Impl/TSKVRowInputFormat.cpp

@@ -20,7 +20,7 @@ namespace ErrorCodes
 
 
 TSKVRowInputFormat::TSKVRowInputFormat(ReadBuffer & in_, Block header_, Params params_, const FormatSettings & format_settings_)
-    : IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), name_map(header_.columns())
+    : IRowInputFormat(std::move(header_), in_, std::move(params_)), format_settings(format_settings_), name_map(getPort().getHeader().columns())
 {
     const auto & sample_block = getPort().getHeader();
     size_t num_columns = sample_block.columns();

src/Processors/QueryPlan/ReadFromMergeTree.cpp

@@ -921,8 +921,8 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
 
         auto reader_settings = getMergeTreeReaderSettings(context);
 
-        bool use_skip_indexes = context->getSettings().use_skip_indexes;
+        bool use_skip_indexes = settings.use_skip_indexes;
-        if (select.final() && !context->getSettings().use_skip_indexes_if_final)
+        if (select.final() && !settings.use_skip_indexes_if_final)
             use_skip_indexes = false;
 
         result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes(

src/Processors/Transforms/CreatingSetsTransform.h

@@ -44,8 +44,8 @@ public:
 private:
     SubqueryForSet subquery;
 
-    std::unique_ptr<PushingPipelineExecutor> executor;
     QueryPipeline table_out;
+    std::unique_ptr<PushingPipelineExecutor> executor;
     UInt64 read_rows = 0;
     Stopwatch watch;
 

src/Processors/Transforms/buildPushingToViewsChain.cpp

@@ -331,7 +331,7 @@ Chain buildPushingToViewsChain(
         {
             auto executing_inner_query = std::make_shared<ExecutingInnerQueryFromViewTransform>(
                 storage_header, views_data->views.back(), views_data);
-            executing_inner_query->setRuntimeData(view_thread_status, elapsed_counter_ms);
+            executing_inner_query->setRuntimeData(view_thread_status, view_counter_ms);
 
             out.addSource(std::move(executing_inner_query));
         }
@@ -381,7 +381,7 @@ Chain buildPushingToViewsChain(
         processors.emplace_front(std::move(copying_data));
         processors.emplace_back(std::move(finalizing_views));
         result_chain = Chain(std::move(processors));
-        result_chain.setNumThreads(max_parallel_streams);
+        result_chain.setNumThreads(std::min(views_data->max_threads, max_parallel_streams));
     }
 
     if (auto * live_view = dynamic_cast<StorageLiveView *>(storage.get()))

src/Processors/Transforms/buildPushingToViewsChain.h

@@ -18,7 +18,7 @@ namespace DB
 
 struct ViewRuntimeData
 {
-    /// A query we should run over inserted block befire pushing into inner storage.
+    /// A query we should run over inserted block before pushing into inner storage.
     const ASTPtr query;
     /// This structure is expected by inner storage. Will convert query result to it.
     Block sample_block;

src/Processors/Transforms/getSourceFromASTInsertQuery.cpp

@@ -59,7 +59,7 @@ InputFormatPtr getInputFormatFromASTInsertQuery(
         : std::make_unique<EmptyReadBuffer>();
 
     /// Create a source from input buffer using format from query
-    auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettings().max_insert_block_size);
+    auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettingsRef().max_insert_block_size);
     source->addBuffer(std::move(input_buffer));
     return source;
 }

src/Server/GRPCServer.cpp

@@ -51,6 +51,7 @@ using GRPCQueryInfo = clickhouse::grpc::QueryInfo;
 using GRPCResult = clickhouse::grpc::Result;
 using GRPCException = clickhouse::grpc::Exception;
 using GRPCProgress = clickhouse::grpc::Progress;
+using GRPCObsoleteTransportCompression = clickhouse::grpc::ObsoleteTransportCompression;
 
 namespace DB
 {
@@ -101,62 +102,6 @@ namespace
         });
     }
 
-    grpc_compression_algorithm parseCompressionAlgorithm(const String & str)
-    {
-        if (str == "none")
-            return GRPC_COMPRESS_NONE;
-        else if (str == "deflate")
-            return GRPC_COMPRESS_DEFLATE;
-        else if (str == "gzip")
-            return GRPC_COMPRESS_GZIP;
-        else if (str == "stream_gzip")
-            return GRPC_COMPRESS_STREAM_GZIP;
-        else
-            throw Exception("Unknown compression algorithm: '" + str + "'", ErrorCodes::INVALID_CONFIG_PARAMETER);
-    }
-
-    grpc_compression_level parseCompressionLevel(const String & str)
-    {
-        if (str == "none")
-            return GRPC_COMPRESS_LEVEL_NONE;
-        else if (str == "low")
-            return GRPC_COMPRESS_LEVEL_LOW;
-        else if (str == "medium")
-            return GRPC_COMPRESS_LEVEL_MED;
-        else if (str == "high")
-            return GRPC_COMPRESS_LEVEL_HIGH;
-        else
-            throw Exception("Unknown compression level: '" + str + "'", ErrorCodes::INVALID_CONFIG_PARAMETER);
-    }
-
-    grpc_compression_algorithm convertCompressionAlgorithm(const ::clickhouse::grpc::CompressionAlgorithm & algorithm)
-    {
-        if (algorithm == ::clickhouse::grpc::NO_COMPRESSION)
-            return GRPC_COMPRESS_NONE;
-        else if (algorithm == ::clickhouse::grpc::DEFLATE)
-            return GRPC_COMPRESS_DEFLATE;
-        else if (algorithm == ::clickhouse::grpc::GZIP)
-            return GRPC_COMPRESS_GZIP;
-        else if (algorithm == ::clickhouse::grpc::STREAM_GZIP)
-            return GRPC_COMPRESS_STREAM_GZIP;
-        else
-            throw Exception("Unknown compression algorithm: '" + ::clickhouse::grpc::CompressionAlgorithm_Name(algorithm) + "'", ErrorCodes::INVALID_GRPC_QUERY_INFO);
-    }
-
-    grpc_compression_level convertCompressionLevel(const ::clickhouse::grpc::CompressionLevel & level)
-    {
-        if (level == ::clickhouse::grpc::COMPRESSION_NONE)
-            return GRPC_COMPRESS_LEVEL_NONE;
-        else if (level == ::clickhouse::grpc::COMPRESSION_LOW)
-            return GRPC_COMPRESS_LEVEL_LOW;
-        else if (level == ::clickhouse::grpc::COMPRESSION_MEDIUM)
-            return GRPC_COMPRESS_LEVEL_MED;
-        else if (level == ::clickhouse::grpc::COMPRESSION_HIGH)
-            return GRPC_COMPRESS_LEVEL_HIGH;
-        else
-            throw Exception("Unknown compression level: '" + ::clickhouse::grpc::CompressionLevel_Name(level) + "'", ErrorCodes::INVALID_GRPC_QUERY_INFO);
-    }
-
     /// Gets file's contents as a string, throws an exception if failed.
     String readFile(const String & filepath)
     {
@@ -193,6 +138,102 @@ namespace
         return grpc::InsecureServerCredentials();
     }
 
+    /// Transport compression makes gRPC library to compress packed Result messages before sending them through network.
+    struct TransportCompression
+    {
+        grpc_compression_algorithm algorithm;
+        grpc_compression_level level;
+
+        /// Extracts the settings of transport compression from a query info if possible.
+        static std::optional<TransportCompression> fromQueryInfo(const GRPCQueryInfo & query_info)
+        {
+            TransportCompression res;
+            if (!query_info.transport_compression_type().empty())
+            {
+                res.setAlgorithm(query_info.transport_compression_type(), ErrorCodes::INVALID_GRPC_QUERY_INFO);
+                res.setLevel(query_info.transport_compression_level(), ErrorCodes::INVALID_GRPC_QUERY_INFO);
+                return res;
+            }
+
+            if (query_info.has_obsolete_result_compression())
+            {
+                switch (query_info.obsolete_result_compression().algorithm())
+                {
+                    case GRPCObsoleteTransportCompression::NO_COMPRESSION: res.algorithm = GRPC_COMPRESS_NONE; break;
+                    case GRPCObsoleteTransportCompression::DEFLATE: res.algorithm = GRPC_COMPRESS_DEFLATE; break;
+                    case GRPCObsoleteTransportCompression::GZIP: res.algorithm = GRPC_COMPRESS_GZIP; break;
+                    case GRPCObsoleteTransportCompression::STREAM_GZIP: res.algorithm = GRPC_COMPRESS_STREAM_GZIP; break;
+                    default: throw Exception(ErrorCodes::INVALID_GRPC_QUERY_INFO, "Unknown compression algorithm: {}", GRPCObsoleteTransportCompression::CompressionAlgorithm_Name(query_info.obsolete_result_compression().algorithm()));
+                }
+
+                switch (query_info.obsolete_result_compression().level())
+                {
+                    case GRPCObsoleteTransportCompression::COMPRESSION_NONE: res.level = GRPC_COMPRESS_LEVEL_NONE; break;
+                    case GRPCObsoleteTransportCompression::COMPRESSION_LOW: res.level = GRPC_COMPRESS_LEVEL_LOW; break;
+                    case GRPCObsoleteTransportCompression::COMPRESSION_MEDIUM: res.level = GRPC_COMPRESS_LEVEL_MED; break;
+                    case GRPCObsoleteTransportCompression::COMPRESSION_HIGH: res.level = GRPC_COMPRESS_LEVEL_HIGH; break;
+                    default: throw Exception(ErrorCodes::INVALID_GRPC_QUERY_INFO, "Unknown compression level: {}", GRPCObsoleteTransportCompression::CompressionLevel_Name(query_info.obsolete_result_compression().level()));
+                }
+                return res;
+            }
+
+            return std::nullopt;
+        }
+
+        /// Extracts the settings of transport compression from the server configuration.
+        static TransportCompression fromConfiguration(const Poco::Util::AbstractConfiguration & config)
+        {
+            TransportCompression res;
+            if (config.has("grpc.transport_compression_type"))
+            {
+                res.setAlgorithm(config.getString("grpc.transport_compression_type"), ErrorCodes::INVALID_CONFIG_PARAMETER);
+                res.setLevel(config.getInt("grpc.transport_compression_level", 0), ErrorCodes::INVALID_CONFIG_PARAMETER);
+            }
+            else
+            {
+                res.setAlgorithm(config.getString("grpc.compression", "none"), ErrorCodes::INVALID_CONFIG_PARAMETER);
+                res.setLevel(config.getString("grpc.compression_level", "none"), ErrorCodes::INVALID_CONFIG_PARAMETER);
+            }
+            return res;
+        }
+
+    private:
+        void setAlgorithm(const String & str, int error_code)
+        {
+            if (str == "none")
+                algorithm = GRPC_COMPRESS_NONE;
+            else if (str == "deflate")
+                algorithm = GRPC_COMPRESS_DEFLATE;
+            else if (str == "gzip")
+                algorithm = GRPC_COMPRESS_GZIP;
+            else if (str == "stream_gzip")
+                algorithm = GRPC_COMPRESS_STREAM_GZIP;
+            else
+                throw Exception(error_code, "Unknown compression algorithm: '{}'", str);
+        }
+
+        void setLevel(const String & str, int error_code)
+        {
+            if (str == "none")
+                level = GRPC_COMPRESS_LEVEL_NONE;
+            else if (str == "low")
+                level = GRPC_COMPRESS_LEVEL_LOW;
+            else if (str == "medium")
+                level = GRPC_COMPRESS_LEVEL_MED;
+            else if (str == "high")
+                level = GRPC_COMPRESS_LEVEL_HIGH;
+            else
+                throw Exception(error_code, "Unknown compression level: '{}'", str);
+        }
+
+        void setLevel(int level_, int error_code)
+        {
+            if (0 <= level_ && level_ < GRPC_COMPRESS_LEVEL_COUNT)
+                level = static_cast<grpc_compression_level>(level_);
+            else
+                throw Exception(error_code, "Compression level {} is out of range 0..{}", level_, GRPC_COMPRESS_LEVEL_COUNT - 1);
+        }
+    };
 
     /// Gets session's timeout from query info or from the server config.
     std::chrono::steady_clock::duration getSessionTimeout(const GRPCQueryInfo & query_info, const Poco::Util::AbstractConfiguration & config)
@@ -293,15 +334,10 @@ namespace
             return std::nullopt;
         }
 
-        void setResultCompression(grpc_compression_algorithm algorithm, grpc_compression_level level)
+        void setTransportCompression(const TransportCompression & transport_compression)
         {
-            grpc_context.set_compression_algorithm(algorithm);
+            grpc_context.set_compression_algorithm(transport_compression.algorithm);
-            grpc_context.set_compression_level(level);
+            grpc_context.set_compression_level(transport_compression.level);
-        }
-
-        void setResultCompression(const ::clickhouse::grpc::Compression & compression)
-        {
-            setResultCompression(convertCompressionAlgorithm(compression.algorithm()), convertCompressionLevel(compression.level()));
         }
 
     protected:
@@ -606,6 +642,9 @@ namespace
         void throwIfFailedToReadQueryInfo();
         bool isQueryCancelled();
 
+        void addQueryDetailsToResult();
+        void addOutputFormatToResult();
+        void addOutputColumnsNamesAndTypesToResult(const Block & headers);
         void addProgressToResult();
         void addTotalsToResult(const Block & totals);
         void addExtremesToResult(const Block & extremes);
@@ -628,10 +667,12 @@ namespace
         ASTInsertQuery * insert_query = nullptr;
         String input_format;
         String input_data_delimiter;
+        CompressionMethod input_compression_method = CompressionMethod::None;
         PODArray<char> output;
         String output_format;
-        CompressionMethod compression_method = CompressionMethod::None;
+        bool send_output_columns_names_and_types = false;
-        int compression_level = 0;
+        CompressionMethod output_compression_method = CompressionMethod::None;
+        int output_compression_level = 0;
 
         uint64_t interactive_delay = 100000;
         bool send_exception_with_stacktrace = true;
@@ -815,9 +856,9 @@ namespace
         if (!query_info.database().empty())
             query_context->setCurrentDatabase(query_info.database());
 
-        /// Apply compression settings for this call.
+        /// Apply transport compression for this call.
-        if (query_info.has_result_compression())
+        if (auto transport_compression = TransportCompression::fromQueryInfo(query_info))
-            responder->setResultCompression(query_info.result_compression());
+            responder->setTransportCompression(*transport_compression);
 
         /// The interactive delay will be used to show progress.
         interactive_delay = settings.interactive_delay;
@@ -851,9 +892,19 @@ namespace
         if (output_format.empty())
             output_format = query_context->getDefaultFormat();
 
+        send_output_columns_names_and_types = query_info.send_output_columns();
+
         /// Choose compression.
-        compression_method = chooseCompressionMethod("", query_info.compression_type());
+        String input_compression_method_str = query_info.input_compression_type();
-        compression_level = query_info.compression_level();
+        if (input_compression_method_str.empty())
+            input_compression_method_str = query_info.obsolete_compression_type();
+        input_compression_method = chooseCompressionMethod("", input_compression_method_str);
+
+        String output_compression_method_str = query_info.output_compression_type();
+        if (output_compression_method_str.empty())
+            output_compression_method_str = query_info.obsolete_compression_type();
+        output_compression_method = chooseCompressionMethod("", output_compression_method_str);
+        output_compression_level = query_info.output_compression_level();
 
         /// Set callback to create and fill external tables
         query_context->setExternalTablesInitializer([this] (ContextPtr context)
@@ -984,7 +1035,7 @@ namespace
             return {nullptr, 0}; /// no more input data
         });
 
-        read_buffer = wrapReadBufferWithCompressionMethod(std::move(read_buffer), compression_method);
+        read_buffer = wrapReadBufferWithCompressionMethod(std::move(read_buffer), input_compression_method);
 
         assert(!pipeline);
         auto source = query_context->getInputFormat(
@@ -1105,6 +1156,9 @@ namespace
 
     void Call::generateOutput()
     {
+        /// We add query_id and time_zone to the first result anyway.
+        addQueryDetailsToResult();
+
         if (!io.pipeline.initialized() || io.pipeline.pushing())
             return;
 
@@ -1112,13 +1166,13 @@ namespace
         if (io.pipeline.pulling())
             header = io.pipeline.getHeader();
 
-        if (compression_method != CompressionMethod::None)
+        if (output_compression_method != CompressionMethod::None)
             output.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data.
         write_buffer = std::make_unique<WriteBufferFromVector<PODArray<char>>>(output);
         nested_write_buffer = static_cast<WriteBufferFromVector<PODArray<char>> *>(write_buffer.get());
-        if (compression_method != CompressionMethod::None)
+        if (output_compression_method != CompressionMethod::None)
         {
-            write_buffer = wrapWriteBufferWithCompressionMethod(std::move(write_buffer), compression_method, compression_level);
+            write_buffer = wrapWriteBufferWithCompressionMethod(std::move(write_buffer), output_compression_method, output_compression_level);
             compressing_write_buffer = write_buffer.get();
         }
 
@@ -1144,6 +1198,9 @@ namespace
                 return true;
             };
 
+            addOutputFormatToResult();
+            addOutputColumnsNamesAndTypesToResult(header);
+
             Block block;
             while (check_for_cancel())
             {
@@ -1394,6 +1451,29 @@ namespace
         return false;
     }
 
+    void Call::addQueryDetailsToResult()
+    {
+        *result.mutable_query_id() = query_context->getClientInfo().current_query_id;
+        *result.mutable_time_zone() = DateLUT::instance().getTimeZone();
+    }
+
+    void Call::addOutputFormatToResult()
+    {
+        *result.mutable_output_format() = output_format;
+    }
+
+    void Call::addOutputColumnsNamesAndTypesToResult(const Block & header)
+    {
+        if (!send_output_columns_names_and_types)
+            return;
+        for (const auto & column : header)
+        {
+            auto & name_and_type = *result.add_output_columns();
+            *name_and_type.mutable_name() = column.name;
+            *name_and_type.mutable_type() = column.type->getName();
+        }
+    }
+
     void Call::addProgressToResult()
     {
         auto values = progress.fetchAndResetPiecewiseAtomically();
@@ -1414,10 +1494,10 @@ namespace
             return;
 
         PODArray<char> memory;
-        if (compression_method != CompressionMethod::None)
+        if (output_compression_method != CompressionMethod::None)
             memory.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data.
         std::unique_ptr<WriteBuffer> buf = std::make_unique<WriteBufferFromVector<PODArray<char>>>(memory);
-        buf = wrapWriteBufferWithCompressionMethod(std::move(buf), compression_method, compression_level);
+        buf = wrapWriteBufferWithCompressionMethod(std::move(buf), output_compression_method, output_compression_level);
         auto format = query_context->getOutputFormat(output_format, *buf, totals);
         format->write(materializeBlock(totals));
         format->finalize();
@@ -1432,10 +1512,10 @@ namespace
             return;
 
         PODArray<char> memory;
-        if (compression_method != CompressionMethod::None)
+        if (output_compression_method != CompressionMethod::None)
             memory.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data.
         std::unique_ptr<WriteBuffer> buf = std::make_unique<WriteBufferFromVector<PODArray<char>>>(memory);
-        buf = wrapWriteBufferWithCompressionMethod(std::move(buf), compression_method, compression_level);
+        buf = wrapWriteBufferWithCompressionMethod(std::move(buf), output_compression_method, output_compression_level);
         auto format = query_context->getOutputFormat(output_format, *buf, extremes);
         format->write(materializeBlock(extremes));
         format->finalize();
@@ -1772,8 +1852,9 @@ void GRPCServer::start()
     builder.RegisterService(&grpc_service);
     builder.SetMaxSendMessageSize(iserver.config().getInt("grpc.max_send_message_size", -1));
     builder.SetMaxReceiveMessageSize(iserver.config().getInt("grpc.max_receive_message_size", -1));
-    builder.SetDefaultCompressionAlgorithm(parseCompressionAlgorithm(iserver.config().getString("grpc.compression", "none")));
+    auto default_transport_compression = TransportCompression::fromConfiguration(iserver.config());
-    builder.SetDefaultCompressionLevel(parseCompressionLevel(iserver.config().getString("grpc.compression_level", "none")));
+    builder.SetDefaultCompressionAlgorithm(default_transport_compression.algorithm);
+    builder.SetDefaultCompressionLevel(default_transport_compression.level);
 
     queue = builder.AddCompletionQueue();
     grpc_server = builder.BuildAndStart();