Merge branch 'master' into UTC_timestamp

.github/workflows/cancel.yml

@@ -6,7 +6,7 @@ env:
 
 on: # yamllint disable-line rule:truthy
   workflow_run:
-    workflows: ["PullRequestCI", "ReleaseCI", "DocsCheck", "BackportPR"]
+    workflows: ["PullRequestCI", "ReleaseBranchCI", "DocsCheck", "BackportPR"]
     types:
       - requested
 jobs:

.github/workflows/nightly.yml

@@ -122,3 +122,58 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  SonarCloud:
+    runs-on: [self-hosted, builder]
+    env:
+      SONAR_SCANNER_VERSION: 4.7.0.2747
+      SONAR_SERVER_URL: "https://sonarcloud.io"
+      BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
+      CC: clang-15
+      CXX: clang++-15
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+          submodules: true
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Download and set up sonar-scanner
+        env:
+          SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
+        run: |
+          mkdir -p "$HOME/.sonar"
+          curl -sSLo "$HOME/.sonar/sonar-scanner.zip" "${{ env.SONAR_SCANNER_DOWNLOAD_URL }}"
+          unzip -o "$HOME/.sonar/sonar-scanner.zip" -d "$HOME/.sonar/"
+          echo "$HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> "$GITHUB_PATH"
+      - name: Download and set up build-wrapper
+        env:
+          BUILD_WRAPPER_DOWNLOAD_URL: ${{ env.SONAR_SERVER_URL }}/static/cpp/build-wrapper-linux-x86.zip
+        run: |
+          curl -sSLo "$HOME/.sonar/build-wrapper-linux-x86.zip" "${{ env.BUILD_WRAPPER_DOWNLOAD_URL }}"
+          unzip -o "$HOME/.sonar/build-wrapper-linux-x86.zip" -d "$HOME/.sonar/"
+          echo "$HOME/.sonar/build-wrapper-linux-x86" >> "$GITHUB_PATH"
+      - name: Set Up Build Tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -yq git cmake ccache python3 ninja-build
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+      - name: Run build-wrapper
+        run: |
+          mkdir build
+          cd build
+          cmake ..
+          cd ..
+          build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} cmake --build build/
+      - name: Run sonar-scanner
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        run: |
+          sonar-scanner \
+            --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" \
+            --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" \
+            --define sonar.projectKey="ClickHouse_ClickHouse" \
+            --define sonar.organization="clickhouse-java" \
+            --define sonar.exclusions="**/*.java,**/*.ts,**/*.js,**/*.css,**/*.sql"

.github/workflows/pull_request.yml

@@ -2023,6 +2023,7 @@ jobs:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
   TestsBugfixCheck:
+    needs: [CheckLabels, StyleCheck]
     runs-on: [self-hosted, stress-tester]
     steps:
       - name: Set envs

.snyk

@@ -0,0 +1,4 @@
+# Snyk (https://snyk.io) policy file
+exclude:
+  global:
+    - tests/**

README.md

@@ -5,7 +5,7 @@ ClickHouse® is an open-source column-oriented database management system that a
 ## Useful Links
 
 * [Official website](https://clickhouse.com/) has a quick high-level overview of ClickHouse on the main page.
-* [ClickHouse Cloud](https://clickhouse.com/cloud) ClickHouse as a service, built by the creators and maintainers.
+* [ClickHouse Cloud](https://clickhouse.cloud) ClickHouse as a service, built by the creators and maintainers.
 * [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster.
 * [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information.
 * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
@@ -16,5 +16,6 @@ ClickHouse® is an open-source column-oriented database management system that a
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
 ## Upcoming events
-* [**v22.10 Release Webinar**](https://clickhouse.com/company/events/v22-10-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
+* [**v22.11 Release Webinar**](https://clickhouse.com/company/events/v22-11-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
-* [**Introducing ClickHouse Cloud**](https://clickhouse.com/company/events/cloud-beta) Introducing ClickHouse as a service, built by creators and maintainers of the fastest OLAP database on earth. Join Tanya Bragin for a detailed walkthrough of ClickHouse Cloud capabilities, as well as a peek behind the curtain to understand the unique architecture that makes our service tick. 
+* [**ClickHouse Meetup at the Deutsche Bank office in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/289311596/) Hear from Deutsche Bank on why they chose ClickHouse for big sensitive data in a regulated environment. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management.
+* [**AWS re:Invent**](https://clickhouse.com/company/events/aws-reinvent) Core members of the ClickHouse team -- including 2 of our founders -- will be at re:Invent from November 29 to December 3. We are available on the show floor, but are also determining interest in holding an event during the time there. 

cmake/clang_tidy.cmake

@@ -3,10 +3,20 @@ option (ENABLE_CLANG_TIDY "Use clang-tidy static analyzer" OFF)
 
 if (ENABLE_CLANG_TIDY)
 
-    find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12")
+    find_program (CLANG_TIDY_CACHE_PATH NAMES "clang-tidy-cache")
+    if (CLANG_TIDY_CACHE_PATH)
+        find_program (_CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12")
+
+        # Why do we use ';' here?
+        # It's a cmake black magic: https://cmake.org/cmake/help/latest/prop_tgt/LANG_CLANG_TIDY.html#prop_tgt:%3CLANG%3E_CLANG_TIDY
+        # The CLANG_TIDY_PATH is passed to CMAKE_CXX_CLANG_TIDY, which follows CXX_CLANG_TIDY syntax.
+        set (CLANG_TIDY_PATH "${CLANG_TIDY_CACHE_PATH};${_CLANG_TIDY_PATH}" CACHE STRING "A combined command to run clang-tidy with caching wrapper")
+    else ()
+        find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12")
+    endif ()
 
     if (CLANG_TIDY_PATH)
-        message(STATUS
+        message (STATUS
             "Using clang-tidy: ${CLANG_TIDY_PATH}.
             The checks will be run during build process.
             See the .clang-tidy file at the root directory to configure the checks.")
@@ -15,11 +25,15 @@ if (ENABLE_CLANG_TIDY)
 
         # clang-tidy requires assertions to guide the analysis
         # Note that NDEBUG is set implicitly by CMake for non-debug builds
-        set(COMPILER_FLAGS "${COMPILER_FLAGS} -UNDEBUG")
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -UNDEBUG")
 
-        # The variable CMAKE_CXX_CLANG_TIDY will be set inside src and base directories with non third-party code.
+        # The variable CMAKE_CXX_CLANG_TIDY will be set inside the following directories with non third-party code.
+        # - base
+        # - programs
+        # - src
+        # - utils
         # set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
     else ()
-        message(${RECONFIGURE_MESSAGE_LEVEL} "clang-tidy is not found")
+        message (${RECONFIGURE_MESSAGE_LEVEL} "clang-tidy is not found")
     endif ()
 endif ()

contrib/cctz

@@ -1 +1 @@
-Subproject commit 7a454c25c7d16053bcd327cdd16329212a08fa4a
+Subproject commit 5c8528fb35e89ee0b3a7157490423fba0d4dd7b5

contrib/llvm-project-cmake/CMakeLists.txt

@@ -21,6 +21,9 @@ set (LLVM_INCLUDE_DIRS
     "${ClickHouse_BINARY_DIR}/contrib/llvm-project/llvm/include"
 )
 set (LLVM_LIBRARY_DIRS "${ClickHouse_BINARY_DIR}/contrib/llvm-project/llvm")
+# NOTE: You should not remove this line since otherwise it will use default 20,
+# and llvm cannot be compiled with bundled libcxx and 20 standard.
+set (CMAKE_CXX_STANDARD 14)
 
 # This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles.
 set (REQUIRED_LLVM_LIBRARIES

docker/packager/binary/Dockerfile

@@ -91,6 +91,9 @@ ENV PATH="$PATH:/usr/local/go/bin"
 ENV GOPATH=/workdir/go
 ENV GOCACHE=/workdir/
 
+RUN curl https://raw.githubusercontent.com/matus-chochlik/ctcache/7fd516e91c17779cbc6fc18bd119313d9532dd90/clang-tidy-cache -Lo /usr/bin/clang-tidy-cache \
+  && chmod +x /usr/bin/clang-tidy-cache
+
 RUN mkdir /workdir && chmod 777 /workdir
 WORKDIR /workdir
 

docker/packager/packager

@@ -258,6 +258,10 @@ def parse_env_variables(
         if clang_tidy:
             # 15G is not enough for tidy build
             cache_maxsize = "25G"
+
+            # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above.
+            # It's there to have the clang-tidy cache embedded into our standard `CCACHE_DIR`
+            result.append("CTCACHE_DIR=/ccache/clang-tidy-cache")
         result.append(f"CCACHE_MAXSIZE={cache_maxsize}")
 
     if distcc_hosts:
@@ -282,9 +286,7 @@ def parse_env_variables(
         cmake_flags.append("-DENABLE_TESTS=1")
 
     if shared_libraries:
-        cmake_flags.append(
+        cmake_flags.append("-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1")
-            "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1"
-        )
         # We can't always build utils because it requires too much space, but
         # we have to build them at least in some way in CI. The shared library
         # build is probably the least heavy disk-wise.

docker/server/Dockerfile.alpine

@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="22.10.1.1877"
+ARG VERSION="22.10.2.11"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # user/group precreated explicitly with fixed uid/gid on purpose.

docker/server/Dockerfile.ubuntu

@@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="22.10.1.1877"
+ARG VERSION="22.10.2.11"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # set non-empty deb_location_url url to create a docker image

docs/README.md

@@ -212,4 +212,4 @@ Templates:
 
 ## How to Build Documentation
 
-You can build your documentation manually by following the instructions in [docs/tools/README.md](../docs/tools/README.md). Also, our CI runs the documentation build after the `documentation` label is added to PR. You can see the results of a build in the GitHub interface. If you have no permissions to add labels, a reviewer of your PR will add it.
+You can build your documentation manually by following the instructions in the docs repo [contrib-writing-guide](https://github.com/ClickHouse/clickhouse-docs/blob/main/contrib-writing-guide.md). Also, our CI runs the documentation build after the `documentation` label is added to PR. You can see the results of a build in the GitHub interface. If you have no permissions to add labels, a reviewer of your PR will add it.

docs/changelogs/v22.10.2.11-stable.md

@@ -0,0 +1,18 @@
+---
+sidebar_position: 1
+sidebar_label: 2022
+---
+
+# 2022 Changelog
+
+### ClickHouse release v22.10.2.11-stable (d2bfcaba002) FIXME as compared to v22.10.1.1877-stable (98ab5a3c189)
+
+#### Bug Fix (user-visible misbehavior in official stable or prestable release)
+
+* Backported in [#42750](https://github.com/ClickHouse/ClickHouse/issues/42750): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} <Fatal> BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} <Fatal> BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} <Fatal> BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} <Fatal> BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} <Fatal> BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} <Fatal> BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} <Fatal> BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)).
+* Backported in [#42793](https://github.com/ClickHouse/ClickHouse/issues/42793): Fix a bug in ParserFunction that could have led to a segmentation fault. [#42724](https://github.com/ClickHouse/ClickHouse/pull/42724) ([Nikolay Degterinsky](https://github.com/evillique)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Always run `BuilderReport` and `BuilderSpecialReport` in all CI types [#42684](https://github.com/ClickHouse/ClickHouse/pull/42684) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+

docs/changelogs/v22.3.14.18-lts.md

@@ -0,0 +1,26 @@
+---
+sidebar_position: 1
+sidebar_label: 2022
+---
+
+# 2022 Changelog
+
+### ClickHouse release v22.3.14.18-lts (642946f61b2) FIXME as compared to v22.3.13.80-lts (e2708b01fba)
+
+#### Bug Fix
+* Backported in [#42432](https://github.com/ClickHouse/ClickHouse/issues/42432): - Choose correct aggregation method for LowCardinality with BigInt. [#42342](https://github.com/ClickHouse/ClickHouse/pull/42342) ([Duc Canh Le](https://github.com/canhld94)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#42328](https://github.com/ClickHouse/ClickHouse/issues/42328): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)).
+* Backported in [#42358](https://github.com/ClickHouse/ClickHouse/issues/42358): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### Bug Fix (user-visible misbehavior in official stable or prestable release)
+
+* Backported in [#42298](https://github.com/ClickHouse/ClickHouse/issues/42298): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#42592](https://github.com/ClickHouse/ClickHouse/issues/42592): This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Add a warning message to release.py script, require release type [#41975](https://github.com/ClickHouse/ClickHouse/pull/41975) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Revert [#27787](https://github.com/ClickHouse/ClickHouse/issues/27787) [#42136](https://github.com/ClickHouse/ClickHouse/pull/42136) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+

docs/changelogs/v22.3.14.23-lts.md

@@ -0,0 +1,29 @@
+---
+sidebar_position: 1
+sidebar_label: 2022
+---
+
+# 2022 Changelog
+
+### ClickHouse release v22.3.14.23-lts (74956bfee4d) FIXME as compared to v22.3.13.80-lts (e2708b01fba)
+
+#### Improvement
+* Backported in [#42527](https://github.com/ClickHouse/ClickHouse/issues/42527): Fix issue with passing MySQL timeouts for MySQL database engine and MySQL table function. Closes [#34168](https://github.com/ClickHouse/ClickHouse/issues/34168)?notification_referrer_id=NT_kwDOAzsV57MzMDMxNjAzNTY5OjU0MjAzODc5. [#40751](https://github.com/ClickHouse/ClickHouse/pull/40751) ([Kseniia Sumarokova](https://github.com/kssenii)).
+
+#### Bug Fix
+* Backported in [#42432](https://github.com/ClickHouse/ClickHouse/issues/42432): - Choose correct aggregation method for LowCardinality with BigInt. [#42342](https://github.com/ClickHouse/ClickHouse/pull/42342) ([Duc Canh Le](https://github.com/canhld94)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#42328](https://github.com/ClickHouse/ClickHouse/issues/42328): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)).
+* Backported in [#42358](https://github.com/ClickHouse/ClickHouse/issues/42358): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### Bug Fix (user-visible misbehavior in official stable or prestable release)
+
+* Backported in [#42298](https://github.com/ClickHouse/ClickHouse/issues/42298): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#42592](https://github.com/ClickHouse/ClickHouse/issues/42592): This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Add a warning message to release.py script, require release type [#41975](https://github.com/ClickHouse/ClickHouse/pull/41975) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Revert [#27787](https://github.com/ClickHouse/ClickHouse/issues/27787) [#42136](https://github.com/ClickHouse/ClickHouse/pull/42136) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+

docs/en/development/build.md

@@ -105,7 +105,7 @@ ninja
 Example for Fedora Rawhide:
 ``` bash
 sudo yum update
-yum --nogpg install git cmake make clang-c++ python3
+sudo yum --nogpg install git cmake make clang python3 ccache
 git clone --recursive https://github.com/ClickHouse/ClickHouse.git
 mkdir build && cd build
 cmake ../ClickHouse

docs/en/engines/table-engines/integrations/s3.md

@@ -139,7 +139,7 @@ The following settings can be specified in configuration file for given endpoint
 -   `use_environment_credentials` — If set to `true`, S3 client will try to obtain credentials from environment variables and [Amazon EC2](https://en.wikipedia.org/wiki/Amazon_Elastic_Compute_Cloud) metadata for given endpoint. Optional, default value is `false`.
 -   `region` — Specifies S3 region name. Optional.
 -   `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Optional, default value is `false`.
--   `header` —  Adds specified HTTP header to a request to given endpoint. Optional, can be speficied multiple times.
+-   `header` —  Adds specified HTTP header to a request to given endpoint. Optional, can be specified multiple times.
 -   `server_side_encryption_customer_key_base64` — If specified, required headers for accessing S3 objects with SSE-C encryption will be set. Optional.
 -   `max_single_read_retries` — The maximum number of attempts during single read. Default value is `4`. Optional.
 

docs/en/getting-started/example-datasets/nyc-taxi.md

@@ -33,7 +33,7 @@ CREATE TABLE trips (
     tip_amount          Float32,
     tolls_amount        Float32,
     total_amount        Float32,
-    payment_type        Enum('CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4),
+    payment_type        Enum('CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4, 'UNK' = 5),
     pickup_ntaname      LowCardinality(String),
     dropoff_ntaname     LowCardinality(String)
 )
@@ -63,7 +63,7 @@ SELECT
     payment_type,
     pickup_ntaname,
     dropoff_ntaname
-FROM url(
+FROM s3(
     'https://datasets-documentation.s3.eu-west-3.amazonaws.com/nyc-taxi/trips_{0..2}.gz',
     'TabSeparatedWithNames'
 )

docs/en/getting-started/install.md

@@ -128,6 +128,24 @@ clickhouse-client # or "clickhouse-client --password" if you set up a password.
 
 </details>
 
+<details>
+<summary>Migration Method for installing the deb-packages</summary>
+
+```bash
+sudo apt-key del E0C56BD4
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
+echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
+    /etc/apt/sources.list.d/clickhouse.list
+sudo apt-get update
+
+sudo apt-get install -y clickhouse-server clickhouse-client
+
+sudo service clickhouse-server start
+clickhouse-client # or "clickhouse-client --password" if you set up a password.
+```
+
+</details>
+
 You can replace `stable` with `lts` to use different [release kinds](/docs/en/faq/operations/production.md) based on your needs.
 
 You can also download and install packages manually from [here](https://packages.clickhouse.com/deb/pool/main/c/).

docs/en/operations/_update.md

@@ -1,10 +1,7 @@
----
-slug: /en/operations/update
-sidebar_position: 47
-sidebar_label: ClickHouse Upgrade
----
 
-# ClickHouse Upgrade
+[//]: # (This file is included in Manage > Updates)
+
+## Self-managed ClickHouse Upgrade
 
 If ClickHouse was installed from `deb` packages, execute the following commands on the server:
 

docs/en/operations/clickhouse-keeper.md

@@ -309,7 +309,7 @@ Sessions with Ephemerals (1):
  /clickhouse/task_queue/ddl
 ```
 
-## [experimental] Migration from ZooKeeper {#migration-from-zookeeper}
+## Migration from ZooKeeper {#migration-from-zookeeper}
 
 Seamlessly migration from ZooKeeper to ClickHouse Keeper is impossible you have to stop your ZooKeeper cluster, convert data and start ClickHouse Keeper. `clickhouse-keeper-converter` tool allows converting ZooKeeper logs and snapshots to ClickHouse Keeper snapshot. It works only with ZooKeeper > 3.4. Steps for migration:
 

docs/en/operations/system-tables/information_schema.md

@@ -178,7 +178,7 @@ Columns:
 -   `view_definition` ([String](../../sql-reference/data-types/string.md)) — `SELECT` query for view.
 -   `check_option` ([String](../../sql-reference/data-types/string.md)) — `NONE`, no checking.
 -   `is_updatable` ([Enum8](../../sql-reference/data-types/enum.md)) — `NO`, the view is not updated.
--   `is_insertable_into` ([Enum8](../../sql-reference/data-types/enum.md)) — Shows whether the created view is [materialized](../../sql-reference/statements/create/view/#materialized). Possible values:
+-   `is_insertable_into` ([Enum8](../../sql-reference/data-types/enum.md)) — Shows whether the created view is [materialized](../../sql-reference/statements/create/view.md/#materialized-view). Possible values:
     -   `NO` — The created view is not materialized.
     -   `YES` — The created view is materialized.
 -   `is_trigger_updatable` ([Enum8](../../sql-reference/data-types/enum.md)) — `NO`, the trigger is not updated.

docs/en/operations/system-tables/replicated_fetches.md

@@ -68,6 +68,5 @@ thread_id:                   54
 
 **See Also**
 
--   [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system/#query-language-system-replicated)
+-   [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md/#managing-replicatedmergetree-tables)
 
-[Original article](https://clickhouse.com/docs/en/operations/system_tables/replicated_fetches) <!--hide-->

docs/en/operations/troubleshooting.md

@@ -28,18 +28,34 @@ sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D7
 sudo apt-get update
 ```
 
-### You Get the Unsupported Architecture Warning with Apt-get {#you-get-the-unsupported-architecture-warning-with-apt-get}
+### You Get Different Warnings with `apt-get update` {#you-get-different-warnings-with-apt-get-update}
 
-- The completed warning message is as follows:
+- The completed warning messages are as one of following:
 
 ```
 N: Skipping acquire of configured file 'main/binary-i386/Packages' as repository 'https://packages.clickhouse.com/deb stable InRelease' doesn't support architecture 'i386'
 ```
 
+```
+E: Failed to fetch https://packages.clickhouse.com/deb/dists/stable/main/binary-amd64/Packages.gz  File has unexpected size (30451 != 28154). Mirror sync in progress?
+```
+
+```
+E: Repository 'https://packages.clickhouse.com/deb stable InRelease' changed its 'Origin' value from 'Artifactory' to 'ClickHouse'
+E: Repository 'https://packages.clickhouse.com/deb stable InRelease' changed its 'Label' value from 'Artifactory' to 'ClickHouse'
+N: Repository 'https://packages.clickhouse.com/deb stable InRelease' changed its 'Suite' value from 'stable' to ''
+N: This must be accepted explicitly before updates for this repository can be applied. See apt-secure(8) manpage for details.
+```
+
+```
+Err:11 https://packages.clickhouse.com/deb stable InRelease
+  400  Bad Request [IP: 172.66.40.249 443]
+```
+
 To resolve the above issue, please use the following script:
 
 ```bash
-sudo rm /var/lib/apt/lists/packages.clickhouse.com_* /var/lib/dpkg/arch
+sudo rm /var/lib/apt/lists/packages.clickhouse.com_* /var/lib/dpkg/arch /var/lib/apt/lists/partial/packages.clickhouse.com_*
 sudo apt-get clean
 sudo apt-get autoclean
 ```

docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md

@@ -303,17 +303,25 @@ or
 CREATE DICTIONARY somedict (
     id UInt64,
     first Date,
-    last Date
+    last Date,
+    advertiser_id UInt64
 )
 PRIMARY KEY id
+SOURCE(CLICKHOUSE(TABLE 'date_table'))
+LIFETIME(MIN 1 MAX 1000)
 LAYOUT(RANGE_HASHED())
 RANGE(MIN first MAX last)
 ```
 
-To work with these dictionaries, you need to pass an additional argument to the `dictGetT` function, for which a range is selected:
+To work with these dictionaries, you need to pass an additional argument to the `dictGet` function, for which a range is selected:
 
 ``` sql
-dictGetT('dict_name', 'attr_name', id, date)
+dictGet('dict_name', 'attr_name', id, date)
+```
+Query example:
+
+``` sql
+SELECT dictGet('somedict', 'advertiser_id', 1, '2022-10-20 23:20:10.000'::DateTime64::UInt64);
 ```
 
 This function returns the value for the specified `id`s and the date range that includes the passed date.

docs/en/sql-reference/functions/date-time-functions.md

@@ -1068,7 +1068,7 @@ Example:
 SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600));
 SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299);
 SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0));
-``` 
+```
 ``` text
 ┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐
 │ ['2012-01-01 12:00:00','2012-01-01 12:30:00']               │
@@ -1244,7 +1244,7 @@ Result:
 └──────────────────────────┘
 ```
 
-When there are two arguments: first is an [Integer](../../sql-reference/data-types/int-uint.md) or [DateTime](../../sql-reference/data-types/datetime.md), second is a constant format string — it acts in the same way as [formatDateTime](#formatdatetime) and return [String](../../sql-reference/data-types/string.md#string) type.
+When there are two or three arguments, the first an [Integer](../../sql-reference/data-types/int-uint.md), [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md), the second a constant format string and the third an optional constant time zone string — it acts in the same way as [formatDateTime](#formatdatetime) and return [String](../../sql-reference/data-types/string.md#string) type.
 
 For example:
 

docs/en/sql-reference/functions/string-functions.md

@@ -571,13 +571,13 @@ Similar to base58Decode, but returns an empty string in case of error.
 
 ## base64Encode(s)
 
-Encodes ‘s’ string into base64
+Encodes ‘s’ FixedString or String into base64.
 
 Alias: `TO_BASE64`.
 
 ## base64Decode(s)
 
-Decode base64-encoded string ‘s’ into original string. In case of failure raises an exception.
+Decode base64-encoded FixedString or String ‘s’ into original string. In case of failure raises an exception.
 
 Alias: `FROM_BASE64`.
 

docs/en/sql-reference/statements/alter/column.md

@@ -107,7 +107,7 @@ ALTER TABLE visits RENAME COLUMN webBrowser TO browser
 CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name
 ```
 
-Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to specify the partition expression](#alter-how-to-specify-part-expr).
+Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to set the partition expression](partition.md#how-to-set-partition-expression).
 
 If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist.
 

docs/en/sql-reference/statements/alter/partition.md

@@ -39,7 +39,7 @@ ALTER TABLE mt DETACH PARTITION '2020-11-21';
 ALTER TABLE mt DETACH PART 'all_2_2_0';
 ```
 
-Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
+Read about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
 
 After the query is executed, you can do whatever you want with the data in the `detached` directory — delete it from the file system, or just leave it.
 
@@ -53,7 +53,7 @@ ALTER TABLE table_name [ON CLUSTER cluster] DROP PARTITION|PART partition_expr
 
 Deletes the specified partition from the table. This query tags the partition as inactive and deletes data completely, approximately in 10 minutes.
 
-Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
+Read about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
 
 The query is replicated – it deletes data on all replicas.
 
@@ -71,7 +71,7 @@ ALTER TABLE table_name [ON CLUSTER cluster] DROP DETACHED PARTITION|PART partiti
 ```
 
 Removes the specified part or all parts of the specified partition from `detached`.
-Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
+Read more about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
 
 ## ATTACH PARTITION\|PART
 
@@ -86,7 +86,7 @@ ALTER TABLE visits ATTACH PARTITION 201901;
 ALTER TABLE visits ATTACH PART 201901_2_2_0;
 ```
 
-Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
+Read more about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
 
 This query is replicated. The replica-initiator checks whether there is data in the `detached` directory.
 If data exists, the query checks its integrity. If everything is correct, the query adds the data to the table.
@@ -166,7 +166,7 @@ This query creates a local backup of a specified partition. If the `PARTITION` c
 The entire backup process is performed without stopping the server.
 :::
 
-Note that for old-styled tables you can specify the prefix of the partition name (for example, `2019`) - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr).
+Note that for old-styled tables you can specify the prefix of the partition name (for example, `2019`) - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
 
 At the time of execution, for a data snapshot, the query creates hardlinks to a table data. Hardlinks are placed in the directory `/var/lib/clickhouse/shadow/N/...`, where:
 

docs/en/sql-reference/statements/alter/projection.md

@@ -7,18 +7,26 @@ title: "Manipulating Projections"
 
 The following operations with [projections](../../../engines/table-engines/mergetree-family/mergetree.md#projections) are available:
 
--   `ALTER TABLE [db].name ADD PROJECTION name ( SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY] )` - Adds projection description to tables metadata.
+## ADD PROJECTION
 
--   `ALTER TABLE [db].name DROP PROJECTION name` - Removes projection description from tables metadata and deletes projection files from disk. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
+`ALTER TABLE [db].name ADD PROJECTION name ( SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY] )` - Adds projection description to tables metadata.
 
--   `ALTER TABLE [db.]table MATERIALIZE PROJECTION name IN PARTITION partition_name` - The query rebuilds the projection `name` in the partition `partition_name`. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
+## DROP PROJECTION
 
--   `ALTER TABLE [db.]table CLEAR PROJECTION name IN PARTITION partition_name` - Deletes projection files from disk without removing description. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
+`ALTER TABLE [db].name DROP PROJECTION name` - Removes projection description from tables metadata and deletes projection files from disk. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
+
+## MATERIALIZE PROJECTION
+
+`ALTER TABLE [db.]table MATERIALIZE PROJECTION name IN PARTITION partition_name` - The query rebuilds the projection `name` in the partition `partition_name`. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
+
+## CLEAR PROJECTION
+
+`ALTER TABLE [db.]table CLEAR PROJECTION name IN PARTITION partition_name` - Deletes projection files from disk without removing description. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
 
 
 The commands `ADD`, `DROP` and `CLEAR` are lightweight in a sense that they only change metadata or remove files.
 
-Also, they are replicated, syncing projections metadata via ZooKeeper.
+Also, they are replicated, syncing projections metadata via ClickHouse Keeper or ZooKeeper.
 
 :::note    
 Projection manipulation is supported only for tables with [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) variants).

docs/en/sql-reference/statements/create/function.md

@@ -4,7 +4,7 @@ sidebar_position: 38
 sidebar_label: FUNCTION
 ---
 
-# CREATE FUNCTION
+# CREATE FUNCTION — user defined function (UDF)
 
 Creates a user defined function from a lambda expression. The expression must consist of function parameters, constants, operators, or other function calls.
 

docs/en/sql-reference/statements/optimize.md

@@ -22,7 +22,7 @@ The `OPTIMIZE` query is supported for [MergeTree](../../engines/table-engines/me
 When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all replicas (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `2`) or on current replica (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `1`).
 
 -   If `OPTIMIZE` does not perform a merge for any reason, it does not notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting.
--   If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter/index.md#alter-how-to-specify-part-expr).
+-   If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](alter/partition.md#how-to-set-partition-expression).
 -   If you specify `FINAL`, optimization is performed even when all the data is already in one part. Also merge is forced even if concurrent merges are performed.
 -   If you specify `DEDUPLICATE`, then completely identical rows (unless by-clause is specified) will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine.
 

docs/en/sql-reference/statements/select/intersect.md

@@ -7,7 +7,7 @@ sidebar_label: INTERSECT
 
 The `INTERSECT` clause returns only those rows that result from both the first and the second queries. The queries must match the number of columns, order, and type. The result of `INTERSECT` can contain duplicate rows.
 
-Multiple `INTERSECT` statements are executes left to right if parenthesis are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clause.
+Multiple `INTERSECT` statements are executed left to right if parentheses are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clauses.
 
 
 ``` sql

docs/ru/sql-reference/functions/date-time-functions.md

@@ -1126,8 +1126,7 @@ SELECT FROM_UNIXTIME(423543535);
 └──────────────────────────┘
 ```
 
-В случае, когда есть два аргумента: первый типа [Integer](../../sql-reference/data-types/int-uint.md) или [DateTime](../../sql-reference/data-types/datetime.md), а второй является строкой постоянного формата — функция работает также, как [formatDateTime](#formatdatetime), и возвращает значение типа [String](../../sql-reference/data-types/string.md#string).
+В случае, когда есть два или три аргумента: первый типа [Integer](../../sql-reference/data-types/int-uint.md), [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) или [DateTime64](../../sql-reference/data-types/datetime64.md), а второй является строкой постоянного формата и третий является строкой постоянной временной зоны — функция работает также, как [formatDateTime](#formatdatetime), и возвращает значение типа [String](../../sql-reference/data-types/string.md#string).
-
 
 Запрос:
 

packages/clickhouse-client.yaml

@@ -37,7 +37,7 @@ deb:
 contents:
 - src: root/etc/clickhouse-client/config.xml
   dst: /etc/clickhouse-client/config.xml
-  type: config
+  type: config|noreplace
 - src: root/usr/bin/clickhouse-benchmark
   dst: /usr/bin/clickhouse-benchmark
 - src: root/usr/bin/clickhouse-compressor

packages/clickhouse-keeper.yaml

@@ -29,7 +29,7 @@ deb:
 contents:
 - src: root/etc/clickhouse-keeper/keeper_config.xml
   dst: /etc/clickhouse-keeper/keeper_config.xml
-  type: config
+  type: config|noreplace
 - src: root/usr/bin/clickhouse-keeper
   dst: /usr/bin/clickhouse-keeper
 # docs

packages/clickhouse-server.yaml

@@ -44,10 +44,10 @@ deb:
 contents:
 - src: root/etc/clickhouse-server/config.xml
   dst: /etc/clickhouse-server/config.xml
-  type: config
+  type: config|noreplace
 - src: root/etc/clickhouse-server/users.xml
   dst: /etc/clickhouse-server/users.xml
-  type: config
+  type: config|noreplace
 - src: clickhouse-server.init
   dst: /etc/init.d/clickhouse-server
 - src: clickhouse-server.service

src/AggregateFunctions/AggregateFunctionAnalysisOfVariance.cpp

@@ -18,8 +18,10 @@ AggregateFunctionPtr createAggregateFunctionAnalysisOfVariance(const std::string
     assertNoParameters(name, parameters);
     assertBinary(name, arguments);
 
-    if (!isNumber(arguments[0]) || !isNumber(arguments[1]))
+    if (!isNumber(arguments[0]))
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} only supports numerical types", name);
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} only supports numerical argument types", name);
+    if (!WhichDataType(arguments[1]).isNativeUInt())
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument of aggregate function {} should be a native unsigned integer", name);
 
     return std::make_shared<AggregateFunctionAnalysisOfVariance>(arguments, parameters);
 }

src/AggregateFunctions/AggregateFunctionAnalysisOfVariance.h

@@ -77,7 +77,7 @@ public:
     void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
     {
         auto f_stat = data(place).getFStatistic();
-        if (std::isinf(f_stat) || isNaN(f_stat))
+        if (std::isinf(f_stat) || isNaN(f_stat) || f_stat < 0)
             throw Exception("F statistic is not defined or infinite for these arguments", ErrorCodes::BAD_ARGUMENTS);
 
         auto p_value = data(place).getPValue(f_stat);

src/AggregateFunctions/Moments.h

@@ -482,6 +482,8 @@ struct ZTestMoments
 template <typename T>
 struct AnalysisOfVarianceMoments
 {
+    constexpr static size_t MAX_GROUPS_NUMBER = 1024 * 1024;
+
     /// Sums of values within a group
     std::vector<T> xs1{};
     /// Sums of squared values within a group
@@ -494,6 +496,10 @@ struct AnalysisOfVarianceMoments
         if (xs1.size() >= possible_size)
             return;
 
+        if (possible_size > MAX_GROUPS_NUMBER)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Too many groups for analysis of variance (should be no more than {}, got {})",
+                            MAX_GROUPS_NUMBER, possible_size);
+
         xs1.resize(possible_size, 0.0);
         xs2.resize(possible_size, 0.0);
         ns.resize(possible_size, 0);

src/Analyzer/Passes/QueryAnalysisPass.cpp

@@ -1621,34 +1621,7 @@ void QueryAnalyzer::validateTableExpressionModifiers(const QueryTreeNodePtr & ta
         table_expression_node->formatASTForErrorMessage(),
         scope.scope_node->formatASTForErrorMessage());
 
-    if (query_node || union_node)
+    if (table_node || table_function_node)
-    {
-        auto table_expression_modifiers = query_node ? query_node->getTableExpressionModifiers() : union_node->getTableExpressionModifiers();
-
-        if (table_expression_modifiers.has_value())
-        {
-            String table_expression_modifiers_error_message;
-
-            if (table_expression_modifiers->hasFinal())
-            {
-                table_expression_modifiers_error_message += "FINAL";
-
-                if (table_expression_modifiers->hasSampleSizeRatio())
-                    table_expression_modifiers_error_message += ", SAMPLE";
-            }
-            else if (table_expression_modifiers->hasSampleSizeRatio())
-            {
-                table_expression_modifiers_error_message += "SAMPLE";
-            }
-
-            throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
-                "Table expression modifiers {} are not supported for subquery {}. In scope {}",
-                table_expression_modifiers_error_message,
-                table_expression_node->formatASTForErrorMessage(),
-                scope.scope_node->formatASTForErrorMessage());
-        }
-    }
-    else if (table_node || table_function_node)
     {
         auto table_expression_modifiers = table_node ? table_node->getTableExpressionModifiers() : table_function_node->getTableExpressionModifiers();
 
@@ -4661,17 +4634,23 @@ void QueryAnalyzer::initializeQueryJoinTreeNode(QueryTreeNodePtr & join_tree_nod
 
                 auto table_expression_modifiers = from_table_identifier.getTableExpressionModifiers();
 
-                if (auto * resolved_identifier_query_node = resolved_identifier->as<QueryNode>())
+                auto * resolved_identifier_query_node = resolved_identifier->as<QueryNode>();
+                auto * resolved_identifier_union_node = resolved_identifier->as<UnionNode>();
+
+                if (resolved_identifier_query_node || resolved_identifier_union_node)
                 {
-                    resolved_identifier_query_node->setIsCTE(false);
+                    if (resolved_identifier_query_node)
+                        resolved_identifier_query_node->setIsCTE(false);
+                    else
+                        resolved_identifier_union_node->setIsCTE(false);
+
                     if (table_expression_modifiers.has_value())
-                        resolved_identifier_query_node->setTableExpressionModifiers(*table_expression_modifiers);
+                    {
-                }
+                        throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
-                else if (auto * resolved_identifier_union_node = resolved_identifier->as<UnionNode>())
+                            "Table expression modifiers {} are not supported for subquery {}",
-                {
+                            table_expression_modifiers->formatForErrorMessage(),
-                    resolved_identifier_union_node->setIsCTE(false);
+                            resolved_identifier->formatASTForErrorMessage());
-                    if (table_expression_modifiers.has_value())
+                    }
-                        resolved_identifier_union_node->setTableExpressionModifiers(*table_expression_modifiers);
                 }
                 else if (auto * resolved_identifier_table_node = resolved_identifier->as<TableNode>())
                 {

src/Analyzer/QueryNode.cpp

@@ -74,12 +74,6 @@ void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s
         buffer << ", constant_value_type: " << constant_value->getType()->getName();
     }
 
-    if (table_expression_modifiers)
-    {
-        buffer << ", ";
-        table_expression_modifiers->dump(buffer);
-    }
-
     if (hasWith())
     {
         buffer << '\n' << std::string(indent + 2, ' ') << "WITH\n";
@@ -195,13 +189,6 @@ bool QueryNode::isEqualImpl(const IQueryTreeNode & rhs) const
     else if (!constant_value && rhs_typed.constant_value)
         return false;
 
-    if (table_expression_modifiers && rhs_typed.table_expression_modifiers && table_expression_modifiers != rhs_typed.table_expression_modifiers)
-        return false;
-    else if (table_expression_modifiers && !rhs_typed.table_expression_modifiers)
-        return false;
-    else if (!table_expression_modifiers && rhs_typed.table_expression_modifiers)
-        return false;
-
     return is_subquery == rhs_typed.is_subquery &&
         is_cte == rhs_typed.is_cte &&
         cte_name == rhs_typed.cte_name &&
@@ -250,9 +237,6 @@ void QueryNode::updateTreeHashImpl(HashState & state) const
         state.update(constant_value_type_name.size());
         state.update(constant_value_type_name);
     }
-
-    if (table_expression_modifiers)
-        table_expression_modifiers->updateTreeHash(state);
 }
 
 QueryTreeNodePtr QueryNode::cloneImpl() const
@@ -270,7 +254,6 @@ QueryTreeNodePtr QueryNode::cloneImpl() const
     result_query_node->cte_name = cte_name;
     result_query_node->projection_columns = projection_columns;
     result_query_node->constant_value = constant_value;
-    result_query_node->table_expression_modifiers = table_expression_modifiers;
 
     return result_query_node;
 }

src/Analyzer/QueryNode.h

@@ -176,24 +176,6 @@ public:
         is_group_by_with_grouping_sets = is_group_by_with_grouping_sets_value;
     }
 
-    /// Return true if query node has table expression modifiers, false otherwise
-    bool hasTableExpressionModifiers() const
-    {
-        return table_expression_modifiers.has_value();
-    }
-
-    /// Get table expression modifiers
-    const std::optional<TableExpressionModifiers> & getTableExpressionModifiers() const
-    {
-        return table_expression_modifiers;
-    }
-
-    /// Set table expression modifiers
-    void setTableExpressionModifiers(TableExpressionModifiers table_expression_modifiers_value)
-    {
-        table_expression_modifiers = std::move(table_expression_modifiers_value);
-    }
-
     /// Returns true if query node WITH section is not empty, false otherwise
     bool hasWith() const
     {
@@ -602,7 +584,6 @@ private:
     std::string cte_name;
     NamesAndTypes projection_columns;
     ConstantValuePtr constant_value;
-    std::optional<TableExpressionModifiers> table_expression_modifiers;
     SettingsChanges settings_changes;
 
     static constexpr size_t with_child_index = 0;

src/Analyzer/QueryTreeBuilder.cpp

@@ -145,12 +145,10 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectWithUnionExpression(const ASTPtr &
     if (select_lists.children.size() == 1)
         return buildSelectOrUnionExpression(select_lists.children[0], is_subquery, cte_name);
 
-    auto union_node = std::make_shared<UnionNode>();
+    auto union_node = std::make_shared<UnionNode>(select_with_union_query_typed.union_mode);
     union_node->setIsSubquery(is_subquery);
     union_node->setIsCTE(!cte_name.empty());
     union_node->setCTEName(cte_name);
-    union_node->setUnionMode(select_with_union_query_typed.union_mode);
-    union_node->setUnionModes(select_with_union_query_typed.list_of_modes);
     union_node->setOriginalAST(select_with_union_query);
 
     size_t select_lists_children_size = select_lists.children.size();
@@ -173,23 +171,22 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr
     if (select_lists.size() == 1)
         return buildSelectExpression(select_lists[0], is_subquery, cte_name);
 
-    auto union_node = std::make_shared<UnionNode>();
+    SelectUnionMode union_mode;
-    union_node->setIsSubquery(is_subquery);
-    union_node->setIsCTE(!cte_name.empty());
-    union_node->setCTEName(cte_name);
-
     if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::INTERSECT_ALL)
-        union_node->setUnionMode(SelectUnionMode::INTERSECT_ALL);
+        union_mode = SelectUnionMode::INTERSECT_ALL;
     else if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::INTERSECT_DISTINCT)
-        union_node->setUnionMode(SelectUnionMode::INTERSECT_DISTINCT);
+        union_mode = SelectUnionMode::INTERSECT_DISTINCT;
     else if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::EXCEPT_ALL)
-        union_node->setUnionMode(SelectUnionMode::EXCEPT_ALL);
+        union_mode = SelectUnionMode::EXCEPT_ALL;
     else if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::EXCEPT_DISTINCT)
-        union_node->setUnionMode(SelectUnionMode::EXCEPT_DISTINCT);
+        union_mode = SelectUnionMode::EXCEPT_DISTINCT;
     else
         throw Exception(ErrorCodes::LOGICAL_ERROR, "UNION type is not initialized");
 
-    union_node->setUnionModes(SelectUnionModes(select_lists.size() - 1, union_node->getUnionMode()));
+    auto union_node = std::make_shared<UnionNode>(union_mode);
+    union_node->setIsSubquery(is_subquery);
+    union_node->setIsCTE(!cte_name.empty());
+    union_node->setCTEName(cte_name);
     union_node->setOriginalAST(select_intersect_except_query);
 
     size_t select_lists_size = select_lists.size();
@@ -676,14 +673,10 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
 
                 if (table_expression_modifiers)
                 {
-                    if (auto * query_node = node->as<QueryNode>())
+                    throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
-                        query_node->setTableExpressionModifiers(*table_expression_modifiers);
+                        "Table expression modifiers {} are not supported for subquery {}",
-                    else if (auto * union_node = node->as<UnionNode>())
+                        table_expression_modifiers->formatForErrorMessage(),
-                        union_node->setTableExpressionModifiers(*table_expression_modifiers);
+                        node->formatASTForErrorMessage());
-                    else
-                        throw Exception(ErrorCodes::LOGICAL_ERROR,
-                            "Unexpected table expression subquery node. Expected union or query. Actual {}",
-                            node->formatASTForErrorMessage());
                 }
 
                 table_expressions.push_back(std::move(node));

src/Analyzer/TableExpressionModifiers.cpp

@@ -5,6 +5,7 @@
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>
+#include <IO/WriteBufferFromString.h>
 
 namespace DB
 {
@@ -39,4 +40,27 @@ void TableExpressionModifiers::updateTreeHash(SipHash & hash_state) const
     }
 }
 
+String TableExpressionModifiers::formatForErrorMessage() const
+{
+    WriteBufferFromOwnString buffer;
+    if (has_final)
+        buffer << "FINAL";
+
+    if (sample_size_ratio)
+    {
+        if (has_final)
+            buffer << ' ';
+        buffer << "SAMPLE " << ASTSampleRatio::toString(*sample_size_ratio);
+    }
+
+    if (sample_offset_ratio)
+    {
+        if (has_final || sample_size_ratio)
+            buffer << ' ';
+        buffer << "OFFSET " << ASTSampleRatio::toString(*sample_offset_ratio);
+    }
+
+    return buffer.str();
+}
+
 }

src/Analyzer/TableExpressionModifiers.h

@@ -58,6 +58,9 @@ public:
     /// Update tree hash
     void updateTreeHash(SipHash & hash_state) const;
 
+    /// Format for error message
+    String formatForErrorMessage() const;
+
 private:
     bool has_final = false;
     std::optional<Rational> sample_size_ratio;

src/Analyzer/UnionNode.cpp

@@ -30,11 +30,18 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int TYPE_MISMATCH;
+    extern const int BAD_ARGUMENTS;
 }
 
-UnionNode::UnionNode()
+UnionNode::UnionNode(SelectUnionMode union_mode_)
     : IQueryTreeNode(children_size)
+    , union_mode(union_mode_)
 {
+    if (union_mode == SelectUnionMode::UNION_DEFAULT ||
+        union_mode == SelectUnionMode::EXCEPT_DEFAULT ||
+        union_mode == SelectUnionMode::INTERSECT_DEFAULT)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "UNION mode {} must be normalized", toString(union_mode));
+
     children[queries_child_index] = std::make_shared<ListNode>();
 }
 
@@ -101,28 +108,8 @@ void UnionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s
         buffer << ", constant_value_type: " << constant_value->getType()->getName();
     }
 
-    if (table_expression_modifiers)
-    {
-        buffer << ", ";
-        table_expression_modifiers->dump(buffer);
-    }
-
     buffer << ", union_mode: " << toString(union_mode);
 
-    size_t union_modes_size = union_modes.size();
-    buffer << '\n' << std::string(indent + 2, ' ') << "UNION MODES " << union_modes_size << '\n';
-
-    for (size_t i = 0; i < union_modes_size; ++i)
-    {
-        buffer << std::string(indent + 4, ' ');
-
-        auto query_union_mode = union_modes[i];
-        buffer << toString(query_union_mode);
-
-        if (i + 1 != union_modes_size)
-            buffer << '\n';
-    }
-
     buffer << '\n' << std::string(indent + 2, ' ') << "QUERIES\n";
     getQueriesNode()->dumpTreeImpl(buffer, format_state, indent + 4);
 }
@@ -137,15 +124,8 @@ bool UnionNode::isEqualImpl(const IQueryTreeNode & rhs) const
     else if (!constant_value && rhs_typed.constant_value)
         return false;
 
-    if (table_expression_modifiers && rhs_typed.table_expression_modifiers && table_expression_modifiers != rhs_typed.table_expression_modifiers)
-        return false;
-    else if (table_expression_modifiers && !rhs_typed.table_expression_modifiers)
-        return false;
-    else if (!table_expression_modifiers && rhs_typed.table_expression_modifiers)
-        return false;
-
     return is_subquery == rhs_typed.is_subquery && is_cte == rhs_typed.is_cte && cte_name == rhs_typed.cte_name &&
-        union_mode == rhs_typed.union_mode && union_modes == rhs_typed.union_modes;
+        union_mode == rhs_typed.union_mode;
 }
 
 void UnionNode::updateTreeHashImpl(HashState & state) const
@@ -158,10 +138,6 @@ void UnionNode::updateTreeHashImpl(HashState & state) const
 
     state.update(static_cast<size_t>(union_mode));
 
-    state.update(union_modes.size());
-    for (const auto & query_union_mode : union_modes)
-        state.update(static_cast<size_t>(query_union_mode));
-
     if (constant_value)
     {
         auto constant_dump = applyVisitor(FieldVisitorToString(), constant_value->getValue());
@@ -172,23 +148,16 @@ void UnionNode::updateTreeHashImpl(HashState & state) const
         state.update(constant_value_type_name.size());
         state.update(constant_value_type_name);
     }
-
-    if (table_expression_modifiers)
-        table_expression_modifiers->updateTreeHash(state);
 }
 
 QueryTreeNodePtr UnionNode::cloneImpl() const
 {
-    auto result_union_node = std::make_shared<UnionNode>();
+    auto result_union_node = std::make_shared<UnionNode>(union_mode);
 
     result_union_node->is_subquery = is_subquery;
     result_union_node->is_cte = is_cte;
     result_union_node->cte_name = cte_name;
-    result_union_node->union_mode = union_mode;
-    result_union_node->union_modes = union_modes;
-    result_union_node->union_modes_set = union_modes_set;
     result_union_node->constant_value = constant_value;
-    result_union_node->table_expression_modifiers = table_expression_modifiers;
 
     return result_union_node;
 }
@@ -197,14 +166,7 @@ ASTPtr UnionNode::toASTImpl() const
 {
     auto select_with_union_query = std::make_shared<ASTSelectWithUnionQuery>();
     select_with_union_query->union_mode = union_mode;
-
+    select_with_union_query->is_normalized = true;
-    if (union_mode != SelectUnionMode::UNION_DEFAULT &&
-        union_mode != SelectUnionMode::EXCEPT_DEFAULT &&
-        union_mode != SelectUnionMode::INTERSECT_DEFAULT)
-        select_with_union_query->is_normalized = true;
-
-    select_with_union_query->list_of_modes = union_modes;
-    select_with_union_query->set_of_modes = union_modes_set;
     select_with_union_query->children.push_back(getQueriesNode()->toAST());
     select_with_union_query->list_of_selects = select_with_union_query->children.back();
 

src/Analyzer/UnionNode.h

@@ -19,6 +19,7 @@ namespace ErrorCodes
 }
 
 /** Union node represents union of queries in query tree.
+  * Union node must be initialized with normalized union mode.
   *
   * Example: (SELECT id FROM test_table) UNION ALL (SELECT id FROM test_table_2);
   * Example: (SELECT id FROM test_table) UNION DISTINCT (SELECT id FROM test_table_2);
@@ -41,7 +42,8 @@ using UnionNodePtr = std::shared_ptr<UnionNode>;
 class UnionNode final : public IQueryTreeNode
 {
 public:
-    explicit UnionNode();
+    /// Construct union node with normalized union mode
+    explicit UnionNode(SelectUnionMode union_mode_);
 
     /// Returns true if union node is subquery, false otherwise
     bool isSubquery() const
@@ -85,25 +87,6 @@ public:
         return union_mode;
     }
 
-    /// Set union mode value
-    void setUnionMode(SelectUnionMode union_mode_value)
-    {
-        union_mode = union_mode_value;
-    }
-
-    /// Get union modes
-    const SelectUnionModes & getUnionModes() const
-    {
-        return union_modes;
-    }
-
-    /// Set union modes value
-    void setUnionModes(const SelectUnionModes & union_modes_value)
-    {
-        union_modes = union_modes_value;
-        union_modes_set = SelectUnionModesSet(union_modes.begin(), union_modes.end());
-    }
-
     /// Get union node queries
     const ListNode & getQueries() const
     {
@@ -128,24 +111,6 @@ public:
         return children[queries_child_index];
     }
 
-    /// Return true if union node has table expression modifiers, false otherwise
-    bool hasTableExpressionModifiers() const
-    {
-        return table_expression_modifiers.has_value();
-    }
-
-    /// Get table expression modifiers
-    const std::optional<TableExpressionModifiers> & getTableExpressionModifiers() const
-    {
-        return table_expression_modifiers;
-    }
-
-    /// Set table expression modifiers
-    void setTableExpressionModifiers(TableExpressionModifiers table_expression_modifiers_value)
-    {
-        table_expression_modifiers = std::move(table_expression_modifiers_value);
-    }
-
     /// Compute union node projection columns
     NamesAndTypes computeProjectionColumns() const;
 
@@ -189,10 +154,7 @@ private:
     bool is_cte = false;
     std::string cte_name;
     SelectUnionMode union_mode;
-    SelectUnionModes union_modes;
-    SelectUnionModesSet union_modes_set;
     ConstantValuePtr constant_value;
-    std::optional<TableExpressionModifiers> table_expression_modifiers;
 
     static constexpr size_t queries_child_index = 0;
     static constexpr size_t children_size = queries_child_index + 1;

src/Analyzer/Utils.cpp

@@ -98,11 +98,6 @@ static ASTPtr convertIntoTableExpressionAST(const QueryTreeNodePtr & table_expre
 
     if (node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION)
     {
-        if (auto * query_node = table_expression_node->as<QueryNode>())
-            table_expression_modifiers = query_node->getTableExpressionModifiers();
-        else if (auto * union_node = table_expression_node->as<UnionNode>())
-            table_expression_modifiers = union_node->getTableExpressionModifiers();
-
         result_table_expression->subquery = result_table_expression->children.back();
     }
     else if (node_type == QueryTreeNodeType::TABLE || node_type == QueryTreeNodeType::IDENTIFIER)

src/Columns/ColumnArray.h

@@ -176,6 +176,9 @@ public:
 
     void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override;
 
+    void finalize() override { data->finalize(); }
+    bool isFinalized() const override { return data->isFinalized(); }
+
     bool isCollationSupported() const override { return getData().isCollationSupported(); }
 
     size_t getNumberOfDimensions() const;

src/Columns/ColumnMap.h

@@ -93,6 +93,8 @@ public:
     bool structureEquals(const IColumn & rhs) const override;
     double getRatioOfDefaultRows(double sample_ratio) const override;
     void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override;
+    void finalize() override { nested->finalize(); }
+    bool isFinalized() const override { return nested->isFinalized(); }
 
     const ColumnArray & getNestedColumn() const { return assert_cast<const ColumnArray &>(*nested); }
     ColumnArray & getNestedColumn() { return assert_cast<ColumnArray &>(*nested); }

src/Columns/ColumnObject.cpp

@@ -732,8 +732,8 @@ void ColumnObject::get(size_t n, Field & res) const
 {
     assert(n < size());
     res = Object();
-
     auto & object = res.get<Object &>();
+
     for (const auto & entry : subcolumns)
     {
         auto it = object.try_emplace(entry->path.getPath()).first;
@@ -744,7 +744,6 @@ void ColumnObject::get(size_t n, Field & res) const
 void ColumnObject::insertFrom(const IColumn & src, size_t n)
 {
     insert(src[n]);
-    finalize();
 }
 
 void ColumnObject::insertRangeFrom(const IColumn & src, size_t start, size_t length)
@@ -792,9 +791,8 @@ MutableColumnPtr ColumnObject::applyForSubcolumns(Func && func) const
 {
     if (!isFinalized())
     {
-        auto finalized = IColumn::mutate(getPtr());
+        auto finalized = cloneFinalized();
         auto & finalized_object = assert_cast<ColumnObject &>(*finalized);
-        finalized_object.finalize();
         return finalized_object.applyForSubcolumns(std::forward<Func>(func));
     }
 

src/Columns/ColumnObject.h

@@ -198,10 +198,6 @@ public:
     Subcolumns & getSubcolumns() { return subcolumns; }
     PathsInData getKeys() const;
 
-    /// Finalizes all subcolumns.
-    void finalize();
-    bool isFinalized() const;
-
     /// Part of interface
 
     const char * getFamilyName() const override { return "Object"; }
@@ -219,12 +215,17 @@ public:
     void popBack(size_t length) override;
     Field operator[](size_t n) const override;
     void get(size_t n, Field & res) const override;
+
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
     ColumnPtr index(const IColumn & indexes, size_t limit) const override;
     ColumnPtr replicate(const Offsets & offsets) const override;
     MutableColumnPtr cloneResized(size_t new_size) const override;
 
+    /// Finalizes all subcolumns.
+    void finalize() override;
+    bool isFinalized() const override;
+
     /// Order of rows in ColumnObject is undefined.
     void getPermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation & res) const override;
     void compareColumn(const IColumn & rhs, size_t rhs_row_num,
@@ -264,9 +265,7 @@ private:
     template <typename Func>
     MutableColumnPtr applyForSubcolumns(Func && func) const;
 
-    /// For given subcolumn return subcolumn from the same Nested type.
     /// It's used to get shared sized of Nested to insert correct default values.
     const Subcolumns::Node * getLeafOfTheSameNested(const Subcolumns::NodePtr & entry) const;
 };
-
 }

src/Columns/ColumnTuple.cpp

@@ -570,4 +570,15 @@ void ColumnTuple::getIndicesOfNonDefaultRows(Offsets & indices, size_t from, siz
     return getIndicesOfNonDefaultRowsImpl<ColumnTuple>(indices, from, limit);
 }
 
+void ColumnTuple::finalize()
+{
+    for (auto & column : columns)
+        column->finalize();
+}
+
+bool ColumnTuple::isFinalized() const
+{
+    return std::all_of(columns.begin(), columns.end(), [](const auto & column) { return column->isFinalized(); });
+}
+
 }

src/Columns/ColumnTuple.h

@@ -103,6 +103,8 @@ public:
     ColumnPtr compress() const override;
     double getRatioOfDefaultRows(double sample_ratio) const override;
     void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override;
+    void finalize() override;
+    bool isFinalized() const override;
 
     size_t tupleSize() const { return columns.size(); }
 

src/Columns/IColumn.h

@@ -85,8 +85,8 @@ public:
     [[nodiscard]] virtual MutablePtr cloneEmpty() const { return cloneResized(0); }
 
     /// Creates column with the same type and specified size.
-    /// If size is less current size, then data is cut.
+    /// If size is less than current size, then data is cut.
-    /// If size is greater, than default values are appended.
+    /// If size is greater, then default values are appended.
     [[nodiscard]] virtual MutablePtr cloneResized(size_t /*size*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot cloneResized() column {}", getName()); }
 
     /// Returns number of values in column.
@@ -453,6 +453,16 @@ public:
         return getPtr();
     }
 
+    /// Some columns may require finalization before using of other operations.
+    virtual void finalize() {}
+    virtual bool isFinalized() const { return true; }
+
+    MutablePtr cloneFinalized() const
+    {
+        auto finalized = IColumn::mutate(getPtr());
+        finalized->finalize();
+        return finalized;
+    }
 
     [[nodiscard]] static MutablePtr mutate(Ptr ptr)
     {

src/Common/JSONParsers/DummyJSONParser.h

@@ -3,6 +3,7 @@
 #include <Common/Exception.h>
 #include <base/types.h>
 #include <base/defines.h>
+#include "ElementTypes.h"
 
 
 namespace DB
@@ -25,6 +26,7 @@ struct DummyJSONParser
     {
     public:
         Element() = default;
+        static ElementType type() { return ElementType::NULL_VALUE; }
         static bool isInt64() { return false; }
         static bool isUInt64() { return false; }
         static bool isDouble() { return false; }

src/Common/JSONParsers/ElementTypes.h

@@ -0,0 +1,17 @@
+#pragma once
+
+namespace DB
+{
+// Enum values match simdjson's for fast conversion
+enum class ElementType
+{
+    ARRAY = '[',
+    OBJECT = '{',
+    INT64 = 'l',
+    UINT64 = 'u',
+    DOUBLE = 'd',
+    STRING = '"',
+    BOOL = 't',
+    NULL_VALUE = 'n'
+};
+}

src/Common/JSONParsers/RapidJSONParser.h

@@ -6,7 +6,7 @@
 #    include <base/types.h>
 #    include <base/defines.h>
 #    include <rapidjson/document.h>
-
+#    include "ElementTypes.h"
 
 namespace DB
 {
@@ -26,6 +26,20 @@ struct RapidJSONParser
         ALWAYS_INLINE Element() = default;
         ALWAYS_INLINE Element(const rapidjson::Value & value_) : ptr(&value_) {} /// NOLINT
 
+        ALWAYS_INLINE ElementType type() const
+        {
+            switch (ptr->GetType())
+            {
+                case rapidjson::kNumberType: return ptr->IsDouble() ? ElementType::DOUBLE : (ptr->IsUint64() ? ElementType::UINT64 : ElementType::INT64);
+                case rapidjson::kStringType: return ElementType::STRING;
+                case rapidjson::kArrayType: return ElementType::ARRAY;
+                case rapidjson::kObjectType: return ElementType::OBJECT;
+                case rapidjson::kTrueType: return ElementType::BOOL;
+                case rapidjson::kFalseType: return ElementType::BOOL;
+                case rapidjson::kNullType: return ElementType::NULL_VALUE;
+            }
+        }
+
         ALWAYS_INLINE bool isInt64() const { return ptr->IsInt64(); }
         ALWAYS_INLINE bool isUInt64() const { return ptr->IsUint64(); }
         ALWAYS_INLINE bool isDouble() const { return ptr->IsDouble(); }

src/Common/JSONParsers/SimdJSONParser.h

@@ -7,7 +7,7 @@
 #    include <Common/Exception.h>
 #    include <base/defines.h>
 #    include <simdjson.h>
-
+#    include "ElementTypes.h"
 
 namespace DB
 {
@@ -31,6 +31,21 @@ struct SimdJSONParser
         ALWAYS_INLINE Element() {} /// NOLINT
         ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) {} /// NOLINT
 
+        ALWAYS_INLINE ElementType type() const
+        {
+            switch (element.type())
+            {
+                case simdjson::dom::element_type::INT64: return ElementType::INT64;
+                case simdjson::dom::element_type::UINT64: return ElementType::UINT64;
+                case simdjson::dom::element_type::DOUBLE: return ElementType::DOUBLE;
+                case simdjson::dom::element_type::STRING: return ElementType::STRING;
+                case simdjson::dom::element_type::ARRAY: return ElementType::ARRAY;
+                case simdjson::dom::element_type::OBJECT: return ElementType::OBJECT;
+                case simdjson::dom::element_type::BOOL: return ElementType::BOOL;
+                case simdjson::dom::element_type::NULL_VALUE: return ElementType::NULL_VALUE;
+            }
+        }
+
         ALWAYS_INLINE bool isInt64() const { return element.type() == simdjson::dom::element_type::INT64; }
         ALWAYS_INLINE bool isUInt64() const { return element.type() == simdjson::dom::element_type::UINT64; }
         ALWAYS_INLINE bool isDouble() const { return element.type() == simdjson::dom::element_type::DOUBLE; }

src/Common/MemoryTracker.cpp

@@ -10,6 +10,7 @@
 #include <Common/ProfileEvents.h>
 #include <Common/thread_local_rng.h>
 #include <Common/OvercommitTracker.h>
+#include <Common/Stopwatch.h>
 #include <Common/logger_useful.h>
 
 #include "config.h"
@@ -86,6 +87,8 @@ inline std::string_view toDescription(OvercommitResult result)
 namespace ProfileEvents
 {
     extern const Event QueryMemoryLimitExceeded;
+    extern const Event MemoryAllocatorPurge;
+    extern const Event MemoryAllocatorPurgeTimeMicroseconds;
 }
 
 using namespace std::chrono_literals;
@@ -229,7 +232,10 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT
         {
             if (free_memory_in_allocator_arenas.exchange(-current_free_memory_in_allocator_arenas) > 0)
             {
+                Stopwatch watch;
                 mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0);
+                ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge);
+                ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds());
             }
         }
 
@@ -432,7 +438,7 @@ void MemoryTracker::reset()
 
 void MemoryTracker::setRSS(Int64 rss_, Int64 free_memory_in_allocator_arenas_)
 {
-    Int64 new_amount = rss_; // - free_memory_in_allocator_arenas_;
+    Int64 new_amount = rss_;
     total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed);
     free_memory_in_allocator_arenas.store(free_memory_in_allocator_arenas_, std::memory_order_relaxed);
 

src/Common/ProfileEvents.cpp

@@ -229,6 +229,8 @@ The server successfully detected this situation and will download merged part fr
     M(UserTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in user space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \
     M(SystemTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in OS kernel space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \
     M(MemoryOvercommitWaitTimeMicroseconds, "Total time spent in waiting for memory to be freed in OvercommitTracker.") \
+    M(MemoryAllocatorPurge, "Total number of times memory allocator purge was requested") \
+    M(MemoryAllocatorPurgeTimeMicroseconds, "Total number of times memory allocator purge was requested") \
     M(SoftPageFaults, "The number of soft page faults in query execution threads. Soft page fault usually means a miss in the memory allocator cache which required a new memory mapping from the OS and subsequent allocation of a page of physical memory.") \
     M(HardPageFaults, "The number of hard page faults in query execution threads. High values indicate either that you forgot to turn off swap on your server, or eviction of memory pages of the ClickHouse binary during very high memory pressure, or successful usage of the 'mmap' read method for the tables data.") \
     \

src/Common/Throttler.cpp

@@ -3,7 +3,6 @@
 #include <Common/Exception.h>
 #include <Common/Stopwatch.h>
 #include <IO/WriteHelpers.h>
-#include <cmath>
 
 namespace ProfileEvents
 {
@@ -21,63 +20,56 @@ namespace ErrorCodes
 /// Just 10^9.
 static constexpr auto NS = 1000000000UL;
 
-/// Tracking window. Actually the size is not really important. We just want to avoid
+static const size_t default_burst_seconds = 1;
-/// throttles when there are no actions for a long period time.
+
-static const double window_ns = 1ULL * NS;
+Throttler::Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent_)
+    : max_speed(max_speed_)
+    , max_burst(max_speed_ * default_burst_seconds)
+    , limit_exceeded_exception_message("")
+    , tokens(max_burst)
+    , parent(parent_)
+{}
+
+Throttler::Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_,
+            const std::shared_ptr<Throttler> & parent_)
+    : max_speed(max_speed_)
+    , max_burst(max_speed_ * default_burst_seconds)
+    , limit(limit_)
+    , limit_exceeded_exception_message(limit_exceeded_exception_message_)
+    , tokens(max_burst)
+    , parent(parent_)
+{}
 
 void Throttler::add(size_t amount)
 {
-    size_t new_count;
+    // Values obtained under lock to be checked after release
-    /// This outer variable is always equal to smoothed_speed.
+    size_t count_value;
-    /// We use to avoid race condition.
+    double tokens_value;
-    double current_speed = 0;
-
     {
         std::lock_guard lock(mutex);
-
         auto now = clock_gettime_ns_adjusted(prev_ns);
-        /// If prev_ns is equal to zero (first `add` call) we known nothing about speed
+        if (max_speed)
-        /// and don't track anything.
-        if (max_speed && prev_ns != 0)
         {
-            /// Time spent to process the amount of bytes
+            double delta_seconds = prev_ns ? static_cast<double>(now - prev_ns) / NS : 0;
-            double time_spent = now - prev_ns;
+            tokens = std::min<double>(tokens + max_speed * delta_seconds - amount, max_burst);
-
-            /// The speed in bytes per second is equal to amount / time_spent in seconds
-            auto new_speed = amount / (time_spent / NS);
-
-            /// We want to make old values of speed less important for our smoothed value
-            /// so we decay it's value with coef.
-            auto decay_coeff = std::pow(0.5, time_spent / window_ns);
-
-            /// Weighted average between previous and new speed
-            smoothed_speed = smoothed_speed * decay_coeff + (1 - decay_coeff) * new_speed;
-            current_speed = smoothed_speed;
         }
-
         count += amount;
-        new_count = count;
+        count_value = count;
+        tokens_value = tokens;
         prev_ns = now;
     }
 
-    if (limit && new_count > limit)
+    if (limit && count_value > limit)
         throw Exception(limit_exceeded_exception_message + std::string(" Maximum: ") + toString(limit), ErrorCodes::LIMIT_EXCEEDED);
 
-    if (max_speed && current_speed > max_speed)
+    /// Wait unless there is positive amount of tokens - throttling
+    if (max_speed && tokens_value < 0)
     {
-        /// If we was too fast then we have to sleep until our smoothed speed became <= max_speed
+        int64_t sleep_time = static_cast<int64_t>(-tokens_value / max_speed * NS);
-        int64_t sleep_time = static_cast<int64_t>(-window_ns * std::log2(max_speed / current_speed));
+        accumulated_sleep += sleep_time;
-
+        sleepForNanoseconds(sleep_time);
-        if (sleep_time > 0)
+        accumulated_sleep -= sleep_time;
-        {
+        ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_time / 1000UL);
-            accumulated_sleep += sleep_time;
-
-            sleepForNanoseconds(sleep_time);
-
-            accumulated_sleep -= sleep_time;
-
-            ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_time / 1000UL);
-        }
     }
 
     if (parent)
@@ -89,9 +81,9 @@ void Throttler::reset()
     std::lock_guard lock(mutex);
 
     count = 0;
-    accumulated_sleep = 0;
+    tokens = max_burst;
-    smoothed_speed = 0;
     prev_ns = 0;
+    // NOTE: do not zero `accumulated_sleep` to avoid races
 }
 
 bool Throttler::isThrottling() const

src/Common/Throttler.h

@@ -10,25 +10,26 @@
 namespace DB
 {
 
-/** Allows you to limit the speed of something (in entities per second) using sleep.
+/** Allows you to limit the speed of something (in tokens per second) using sleep.
-  * Specifics of work:
+  * Implemented using Token Bucket Throttling algorithm.
-  *  Tracks exponentially (pow of 1/2) smoothed speed with hardcoded window.
+  * Also allows you to set a limit on the maximum number of tokens. If exceeded, an exception will be thrown.
-  *  See more comments in .cpp file.
-  *
-  * Also allows you to set a limit on the maximum number of entities. If exceeded, an exception will be thrown.
   */
 class Throttler
 {
 public:
-    explicit Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent_ = nullptr)
+    Throttler(size_t max_speed_, size_t max_burst_, const std::shared_ptr<Throttler> & parent_ = nullptr)
-            : max_speed(max_speed_), limit_exceeded_exception_message(""), parent(parent_) {}
+        : max_speed(max_speed_), max_burst(max_burst_), limit_exceeded_exception_message(""), tokens(max_burst), parent(parent_) {}
+
+    explicit Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent_ = nullptr);
+
+    Throttler(size_t max_speed_, size_t max_burst_, size_t limit_, const char * limit_exceeded_exception_message_,
+              const std::shared_ptr<Throttler> & parent_ = nullptr)
+        : max_speed(max_speed_), max_burst(max_burst_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), tokens(max_burst), parent(parent_) {}
 
     Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_,
-              const std::shared_ptr<Throttler> & parent_ = nullptr)
+              const std::shared_ptr<Throttler> & parent_ = nullptr);
-        : max_speed(max_speed_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), parent(parent_) {}
 
-    /// Calculates the smoothed speed, sleeps if required and throws exception on
+    /// Use `amount` tokens, sleeps if required or throws exception on limit overflow.
-    /// limit overflow.
     void add(size_t amount);
 
     /// Not thread safe
@@ -45,15 +46,14 @@ public:
 
 private:
     size_t count{0};
-    const size_t max_speed{0};
+    const size_t max_speed{0}; /// in tokens per second.
-    const uint64_t limit{0};        /// 0 - not limited.
+    const size_t max_burst{0}; /// in tokens.
+    const uint64_t limit{0}; /// 0 - not limited.
     const char * limit_exceeded_exception_message = nullptr;
     std::mutex mutex;
-    std::atomic<uint64_t> accumulated_sleep{0};
+    std::atomic<uint64_t> accumulated_sleep{0}; // Accumulated sleep time over all waiting threads
-    /// Smoothed value of current speed. Updated in `add` method.
+    double tokens{0}; /// Amount of tokens available in token bucket. Updated in `add` method.
-    double smoothed_speed{0};
+    uint64_t prev_ns{0}; /// Previous `add` call time (in nanoseconds).
-    /// previous `add` call time (in nanoseconds)
-    uint64_t prev_ns{0};
 
     /// Used to implement a hierarchy of throttlers
     std::shared_ptr<Throttler> parent;

src/Common/formatIPv6.h

@@ -5,6 +5,7 @@
 #include <algorithm>
 #include <utility>
 #include <base/range.h>
+#include <base/unaligned.h>
 #include <Common/hex.h>
 #include <Common/StringUtils/StringUtils.h>
 
@@ -55,8 +56,11 @@ inline bool parseIPv4(const char * src, unsigned char * dst)
     }
     if (*(src - 1) != '\0')
         return false;
-
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    reverseMemcpy(dst, &result, sizeof(result));
+#else
     memcpy(dst, &result, sizeof(result));
+#endif
     return true;
 }
 

src/DataTypes/DataTypeArray.h

@@ -48,6 +48,7 @@ public:
     bool textCanContainOnlyValidUTF8() const override { return nested->textCanContainOnlyValidUTF8(); }
     bool isComparable() const override { return nested->isComparable(); }
     bool canBeComparedWithCollation() const override { return nested->canBeComparedWithCollation(); }
+    bool hasDynamicSubcolumns() const override { return nested->hasDynamicSubcolumns(); }
 
     bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override
     {

src/DataTypes/DataTypeMap.cpp

@@ -22,6 +22,27 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
+DataTypeMap::DataTypeMap(const DataTypePtr & nested_)
+    : nested(nested_)
+{
+    const auto * type_array = typeid_cast<const DataTypeArray *>(nested.get());
+    if (!type_array)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            "Expected Array(Tuple(key, value)) type, got {}", nested->getName());
+
+    const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type_array->getNestedType().get());
+    if (!type_tuple)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            "Expected Array(Tuple(key, value)) type, got {}", nested->getName());
+
+    if (type_tuple->getElements().size() != 2)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            "Expected Array(Tuple(key, value)) type, got {}", nested->getName());
+
+    key_type = type_tuple->getElement(0);
+    value_type = type_tuple->getElement(1);
+    assertKeyType();
+}
 
 DataTypeMap::DataTypeMap(const DataTypes & elems_)
 {

src/DataTypes/DataTypeMap.h

@@ -23,6 +23,7 @@ private:
 public:
     static constexpr bool is_parametric = true;
 
+    explicit DataTypeMap(const DataTypePtr & nested_);
     explicit DataTypeMap(const DataTypes & elems);
     DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & value_type_);
 
@@ -40,6 +41,7 @@ public:
     bool isComparable() const override { return key_type->isComparable() && value_type->isComparable(); }
     bool isParametric() const override { return true; }
     bool haveSubtypes() const override { return true; }
+    bool hasDynamicSubcolumns() const override { return nested->hasDynamicSubcolumns(); }
 
     const DataTypePtr & getKeyType() const { return key_type; }
     const DataTypePtr & getValueType() const { return value_type; }

src/DataTypes/DataTypeObject.h

@@ -36,6 +36,7 @@ public:
     bool haveSubtypes() const override { return false; }
     bool equals(const IDataType & rhs) const override;
     bool isParametric() const override { return true; }
+    bool hasDynamicSubcolumns() const override { return true; }
 
     SerializationPtr doGetDefaultSerialization() const override;
 

src/DataTypes/DataTypeTuple.cpp

@@ -247,6 +247,11 @@ bool DataTypeTuple::haveMaximumSizeOfValue() const
     return std::all_of(elems.begin(), elems.end(), [](auto && elem) { return elem->haveMaximumSizeOfValue(); });
 }
 
+bool DataTypeTuple::hasDynamicSubcolumns() const
+{
+    return std::any_of(elems.begin(), elems.end(), [](auto && elem) { return elem->hasDynamicSubcolumns(); });
+}
+
 bool DataTypeTuple::isComparable() const
 {
     return std::all_of(elems.begin(), elems.end(), [](auto && elem) { return elem->isComparable(); });

src/DataTypes/DataTypeTuple.h

@@ -50,6 +50,7 @@ public:
     bool isComparable() const override;
     bool textCanContainOnlyValidUTF8() const override;
     bool haveMaximumSizeOfValue() const override;
+    bool hasDynamicSubcolumns() const override;
     size_t getMaximumSizeOfValueInMemory() const override;
     size_t getSizeOfValueInMemory() const override;
 

src/DataTypes/IDataType.h

@@ -291,6 +291,9 @@ public:
     /// Strings, Numbers, Date, DateTime, Nullable
     virtual bool canBeInsideLowCardinality() const { return false; }
 
+    /// Object, Array(Object), Tuple(..., Object, ...)
+    virtual bool hasDynamicSubcolumns() const { return false; }
+
     /// Updates avg_value_size_hint for newly read column. Uses to optimize deserialization. Zero expected for first column.
     static void updateAvgValueSizeHint(const IColumn & column, double & avg_value_size_hint);
 

src/DataTypes/ObjectUtils.cpp

@@ -1,17 +1,19 @@
-#include <Storages/StorageSnapshot.h>
 #include <DataTypes/ObjectUtils.h>
 #include <DataTypes/DataTypeObject.h>
 #include <DataTypes/DataTypeNothing.h>
 #include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeMap.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeNested.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/getLeastSupertype.h>
 #include <DataTypes/NestedUtils.h>
+#include <Storages/StorageSnapshot.h>
 #include <Columns/ColumnObject.h>
 #include <Columns/ColumnTuple.h>
 #include <Columns/ColumnArray.h>
+#include <Columns/ColumnMap.h>
 #include <Columns/ColumnNullable.h>
 #include <Parsers/ASTSelectQuery.h>
 #include <Parsers/ASTExpressionList.h>
@@ -105,10 +107,11 @@ Array createEmptyArrayField(size_t num_dimensions)
 DataTypePtr getDataTypeByColumn(const IColumn & column)
 {
     auto idx = column.getDataType();
-    if (WhichDataType(idx).isSimple())
+    WhichDataType which(idx);
+    if (which.isSimple())
         return DataTypeFactory::instance().get(String(magic_enum::enum_name(idx)));
 
-    if (WhichDataType(idx).isNothing())
+    if (which.isNothing())
         return std::make_shared<DataTypeNothing>();
 
     if (const auto * column_array = checkAndGetColumn<ColumnArray>(&column))
@@ -132,41 +135,124 @@ static auto extractVector(const std::vector<Tuple> & vec)
     return res;
 }
 
-void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_storage_columns)
+static DataTypePtr recreateTupleWithElements(const DataTypeTuple & type_tuple, const DataTypes & elements)
 {
-    std::unordered_map<String, DataTypePtr> storage_columns_map;
+    return type_tuple.haveExplicitNames()
-    for (const auto & [name, type] : extended_storage_columns)
+        ? std::make_shared<DataTypeTuple>(elements, type_tuple.getElementNames())
-        storage_columns_map[name] = type;
+        : std::make_shared<DataTypeTuple>(elements);
-
-    for (auto & column : block)
-    {
-        if (!isObject(column.type))
-            continue;
-
-        const auto & column_object = assert_cast<const ColumnObject &>(*column.column);
-        if (!column_object.isFinalized())
-            throw Exception(ErrorCodes::LOGICAL_ERROR,
-                "Cannot convert to tuple column '{}' from type {}. Column should be finalized first",
-                column.name, column.type->getName());
-
-        std::tie(column.column, column.type) = unflattenObjectToTuple(column_object);
-
-        auto it = storage_columns_map.find(column.name);
-        if (it == storage_columns_map.end())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Column '{}' not found in storage", column.name);
-
-        /// Check that constructed Tuple type and type in storage are compatible.
-        getLeastCommonTypeForObject({column.type, it->second}, true);
-    }
 }
 
-void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block)
+static std::pair<ColumnPtr, DataTypePtr> convertObjectColumnToTuple(
+    const ColumnObject & column_object, const DataTypeObject & type_object)
 {
-    if (!storage_snapshot->object_columns.empty())
+    if (!column_object.isFinalized())
     {
-        auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects();
+        auto finalized = column_object.cloneFinalized();
-        auto storage_columns = storage_snapshot->getColumns(options);
+        const auto & finalized_object = assert_cast<const ColumnObject &>(*finalized);
-        convertObjectsToTuples(block, storage_columns);
+        return convertObjectColumnToTuple(finalized_object, type_object);
+    }
+
+    const auto & subcolumns = column_object.getSubcolumns();
+
+    PathsInData tuple_paths;
+    DataTypes tuple_types;
+    Columns tuple_columns;
+
+    for (const auto & entry : subcolumns)
+    {
+        tuple_paths.emplace_back(entry->path);
+        tuple_types.emplace_back(entry->data.getLeastCommonType());
+        tuple_columns.emplace_back(entry->data.getFinalizedColumnPtr());
+    }
+
+    return unflattenTuple(tuple_paths, tuple_types, tuple_columns);
+}
+
+static std::pair<ColumnPtr, DataTypePtr> recursivlyConvertDynamicColumnToTuple(
+    const ColumnPtr & column, const DataTypePtr & type)
+{
+    if (!type->hasDynamicSubcolumns())
+        return {column, type};
+
+    if (const auto * type_object = typeid_cast<const DataTypeObject *>(type.get()))
+    {
+        const auto & column_object = assert_cast<const ColumnObject &>(*column);
+        return convertObjectColumnToTuple(column_object, *type_object);
+    }
+
+    if (const auto * type_array = typeid_cast<const DataTypeArray *>(type.get()))
+    {
+        const auto & column_array = assert_cast<const ColumnArray &>(*column);
+        auto [new_column, new_type] = recursivlyConvertDynamicColumnToTuple(
+            column_array.getDataPtr(), type_array->getNestedType());
+
+        return
+        {
+            ColumnArray::create(new_column, column_array.getOffsetsPtr()),
+            std::make_shared<DataTypeArray>(std::move(new_type)),
+        };
+    }
+
+    if (const auto * type_map = typeid_cast<const DataTypeMap *>(type.get()))
+    {
+        const auto & column_map = assert_cast<const ColumnMap &>(*column);
+        auto [new_column, new_type] = recursivlyConvertDynamicColumnToTuple(
+            column_map.getNestedColumnPtr(), type_map->getNestedType());
+
+        return
+        {
+            ColumnMap::create(new_column),
+            std::make_shared<DataTypeMap>(std::move(new_type)),
+        };
+    }
+
+    if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type.get()))
+    {
+        const auto & tuple_columns = assert_cast<const ColumnTuple &>(*column).getColumns();
+        const auto & tuple_types = type_tuple->getElements();
+
+        assert(tuple_columns.size() == tuple_types.size());
+        const size_t tuple_size = tuple_types.size();
+
+        Columns new_tuple_columns(tuple_size);
+        DataTypes new_tuple_types(tuple_size);
+
+        for (size_t i = 0; i < tuple_size; ++i)
+        {
+            std::tie(new_tuple_columns[i], new_tuple_types[i])
+                = recursivlyConvertDynamicColumnToTuple(tuple_columns[i], tuple_types[i]);
+        }
+
+        return
+        {
+            ColumnTuple::create(new_tuple_columns),
+            recreateTupleWithElements(*type_tuple, new_tuple_types)
+        };
+    }
+
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Type {} unexpectedly has dynamic columns", type->getName());
+}
+
+void convertDynamicColumnsToTuples(Block & block, const StorageSnapshotPtr & storage_snapshot)
+{
+    for (auto & column : block)
+    {
+        if (!column.type->hasDynamicSubcolumns())
+            continue;
+
+        std::tie(column.column, column.type)
+            = recursivlyConvertDynamicColumnToTuple(column.column, column.type);
+
+        GetColumnsOptions options(GetColumnsOptions::AllPhysical);
+        auto storage_column = storage_snapshot->tryGetColumn(options, column.name);
+        if (!storage_column)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Column '{}' not found in storage", column.name);
+
+        auto storage_column_concrete = storage_snapshot->getColumn(options.withExtendedObjects(), column.name);
+
+        /// Check that constructed Tuple type and type in storage are compatible.
+        getLeastCommonTypeForDynamicColumns(
+            storage_column->type, {column.type, storage_column_concrete.type}, true);
     }
 }
 
@@ -217,24 +303,8 @@ void checkObjectHasNoAmbiguosPaths(const PathsInData & paths)
     }
 }
 
-DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambiguos_paths)
+static DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambiguos_paths)
 {
-    if (types.empty())
-        return nullptr;
-
-    bool all_equal = true;
-    for (size_t i = 1; i < types.size(); ++i)
-    {
-        if (!types[i]->equals(*types[0]))
-        {
-            all_equal = false;
-            break;
-        }
-    }
-
-    if (all_equal)
-        return types[0];
-
     /// Types of subcolumns by path from all tuples.
     std::unordered_map<PathInData, DataTypes, PathInData::Hash> subcolumns_types;
 
@@ -287,19 +357,139 @@ DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambi
     return unflattenTuple(tuple_paths, tuple_types);
 }
 
-NameSet getNamesOfObjectColumns(const NamesAndTypesList & columns_list)
+static DataTypePtr getLeastCommonTypeForDynamicColumnsImpl(
-{
+    const DataTypePtr & type_in_storage, const DataTypes & concrete_types, bool check_ambiguos_paths);
-    NameSet res;
-    for (const auto & [name, type] : columns_list)
-        if (isObject(type))
-            res.insert(name);
 
-    return res;
+template<typename Type>
+static DataTypePtr getLeastCommonTypeForColumnWithNestedType(
+    const Type & type, const DataTypes & concrete_types, bool check_ambiguos_paths)
+{
+    DataTypes nested_types;
+    nested_types.reserve(concrete_types.size());
+
+    for (const auto & concrete_type : concrete_types)
+    {
+        const auto * type_with_nested_conctete = typeid_cast<const Type *>(concrete_type.get());
+        if (!type_with_nested_conctete)
+            throw Exception(ErrorCodes::TYPE_MISMATCH, "Expected {} type, got {}", demangle(typeid(Type).name()), concrete_type->getName());
+
+        nested_types.push_back(type_with_nested_conctete->getNestedType());
+    }
+
+    return std::make_shared<Type>(
+        getLeastCommonTypeForDynamicColumnsImpl(
+            type.getNestedType(), nested_types, check_ambiguos_paths));
 }
 
-bool hasObjectColumns(const ColumnsDescription & columns)
+static DataTypePtr getLeastCommonTypeForTuple(
+    const DataTypeTuple & type, const DataTypes & concrete_types, bool check_ambiguos_paths)
 {
-    return std::any_of(columns.begin(), columns.end(), [](const auto & column) { return isObject(column.type); });
+    const auto & element_types = type.getElements();
+    DataTypes new_element_types(element_types.size());
+
+    for (size_t i = 0; i < element_types.size(); ++i)
+    {
+        DataTypes concrete_element_types;
+        concrete_element_types.reserve(concrete_types.size());
+
+        for (const auto & type_concrete : concrete_types)
+        {
+            const auto * type_tuple_conctete = typeid_cast<const DataTypeTuple *>(type_concrete.get());
+            if (!type_tuple_conctete)
+                throw Exception(ErrorCodes::TYPE_MISMATCH, "Expected Tuple type, got {}", type_concrete->getName());
+
+            concrete_element_types.push_back(type_tuple_conctete->getElement(i));
+        }
+
+        new_element_types[i] = getLeastCommonTypeForDynamicColumnsImpl(
+            element_types[i], concrete_element_types, check_ambiguos_paths);
+    }
+
+    return recreateTupleWithElements(type, new_element_types);
+}
+
+static DataTypePtr getLeastCommonTypeForDynamicColumnsImpl(
+    const DataTypePtr & type_in_storage, const DataTypes & concrete_types, bool check_ambiguos_paths)
+{
+    if (!type_in_storage->hasDynamicSubcolumns())
+        return type_in_storage;
+
+    if (isObject(type_in_storage))
+        return getLeastCommonTypeForObject(concrete_types, check_ambiguos_paths);
+
+    if (const auto * type_array = typeid_cast<const DataTypeArray *>(type_in_storage.get()))
+        return getLeastCommonTypeForColumnWithNestedType(*type_array, concrete_types, check_ambiguos_paths);
+
+    if (const auto * type_map = typeid_cast<const DataTypeMap *>(type_in_storage.get()))
+        return getLeastCommonTypeForColumnWithNestedType(*type_map, concrete_types, check_ambiguos_paths);
+
+    if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type_in_storage.get()))
+        return getLeastCommonTypeForTuple(*type_tuple, concrete_types, check_ambiguos_paths);
+
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Type {} unexpectedly has dynamic columns", type_in_storage->getName());
+}
+
+DataTypePtr getLeastCommonTypeForDynamicColumns(
+    const DataTypePtr & type_in_storage, const DataTypes & concrete_types, bool check_ambiguos_paths)
+{
+    if (concrete_types.empty())
+        return nullptr;
+
+    bool all_equal = true;
+    for (size_t i = 1; i < concrete_types.size(); ++i)
+    {
+        if (!concrete_types[i]->equals(*concrete_types[0]))
+        {
+            all_equal = false;
+            break;
+        }
+    }
+
+    if (all_equal)
+        return concrete_types[0];
+
+    return getLeastCommonTypeForDynamicColumnsImpl(type_in_storage, concrete_types, check_ambiguos_paths);
+}
+
+DataTypePtr createConcreteEmptyDynamicColumn(const DataTypePtr & type_in_storage)
+{
+    if (!type_in_storage->hasDynamicSubcolumns())
+        return type_in_storage;
+
+    if (isObject(type_in_storage))
+        return std::make_shared<DataTypeTuple>(
+            DataTypes{std::make_shared<DataTypeUInt8>()}, Names{ColumnObject::COLUMN_NAME_DUMMY});
+
+    if (const auto * type_array = typeid_cast<const DataTypeArray *>(type_in_storage.get()))
+        return std::make_shared<DataTypeArray>(
+            createConcreteEmptyDynamicColumn(type_array->getNestedType()));
+
+    if (const auto * type_map = typeid_cast<const DataTypeMap *>(type_in_storage.get()))
+        return std::make_shared<DataTypeMap>(
+            createConcreteEmptyDynamicColumn(type_map->getNestedType()));
+
+    if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type_in_storage.get()))
+    {
+        const auto & elements = type_tuple->getElements();
+        DataTypes new_elements;
+        new_elements.reserve(elements.size());
+
+        for (const auto & element : elements)
+            new_elements.push_back(createConcreteEmptyDynamicColumn(element));
+
+        return recreateTupleWithElements(*type_tuple, new_elements);
+    }
+
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Type {} unexpectedly has dynamic columns", type_in_storage->getName());
+}
+
+bool hasDynamicSubcolumns(const ColumnsDescription & columns)
+{
+    return std::any_of(columns.begin(), columns.end(),
+        [](const auto & column)
+        {
+            return column.type->hasDynamicSubcolumns();
+        });
 }
 
 void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescription & object_columns, bool with_subcolumns)
@@ -320,16 +510,20 @@ void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescript
     columns_list.splice(columns_list.end(), std::move(subcolumns_list));
 }
 
-void updateObjectColumns(ColumnsDescription & object_columns, const NamesAndTypesList & new_columns)
+void updateObjectColumns(
+    ColumnsDescription & object_columns,
+    const ColumnsDescription & storage_columns,
+    const NamesAndTypesList & new_columns)
 {
     for (const auto & new_column : new_columns)
     {
         auto object_column = object_columns.tryGetColumn(GetColumnsOptions::All, new_column.name);
         if (object_column && !object_column->type->equals(*new_column.type))
         {
+            auto storage_column = storage_columns.getColumn(GetColumnsOptions::All, new_column.name);
             object_columns.modify(new_column.name, [&](auto & column)
             {
-                column.type = getLeastCommonTypeForObject({object_column->type, new_column.type});
+                column.type = getLeastCommonTypeForDynamicColumns(storage_column.type, {object_column->type, new_column.type});
             });
         }
     }
@@ -745,13 +939,6 @@ void replaceMissedSubcolumnsByConstants(
             addConstantToWithClause(query, name, type);
 }
 
-void finalizeObjectColumns(const MutableColumns & columns)
-{
-    for (const auto & column : columns)
-        if (auto * column_object = typeid_cast<ColumnObject *>(column.get()))
-            column_object->finalize();
-}
-
 Field FieldVisitorReplaceScalars::operator()(const Array & x) const
 {
     if (num_dimensions_to_keep == 0)
@@ -768,11 +955,13 @@ size_t FieldVisitorToNumberOfDimensions::operator()(const Array & x)
 {
     const size_t size = x.size();
     size_t dimensions = 0;
+
     for (size_t i = 0; i < size; ++i)
     {
         size_t element_dimensions = applyVisitor(*this, x[i]);
         if (i > 0 && element_dimensions != dimensions)
             need_fold_dimension = true;
+
         dimensions = std::max(dimensions, element_dimensions);
     }
 
@@ -783,12 +972,13 @@ Field FieldVisitorFoldDimension::operator()(const Array & x) const
 {
     if (num_dimensions_to_fold == 0)
         return x;
+
     const size_t size = x.size();
     Array res(size);
     for (size_t i = 0; i < size; ++i)
-    {
         res[i] = applyVisitor(FieldVisitorFoldDimension(num_dimensions_to_fold - 1), x[i]);
-    }
+
     return res;
 }
+
 }

src/DataTypes/ObjectUtils.h

@@ -39,27 +39,31 @@ Array createEmptyArrayField(size_t num_dimensions);
 DataTypePtr getDataTypeByColumn(const IColumn & column);
 
 /// Converts Object types and columns to Tuples in @columns_list and @block
-/// and checks that types are consistent with types in @extended_storage_columns.
+/// and checks that types are consistent with types in @storage_snapshot.
-void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_storage_columns);
+void convertDynamicColumnsToTuples(Block & block, const StorageSnapshotPtr & storage_snapshot);
-void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block);
 
 /// Checks that each path is not the prefix of any other path.
 void checkObjectHasNoAmbiguosPaths(const PathsInData & paths);
 
 /// Receives several Tuple types and deduces the least common type among them.
-DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambiguos_paths = false);
+DataTypePtr getLeastCommonTypeForDynamicColumns(
+    const DataTypePtr & type_in_storage, const DataTypes & types, bool check_ambiguos_paths = false);
+
+DataTypePtr createConcreteEmptyDynamicColumn(const DataTypePtr & type_in_storage);
 
 /// Converts types of object columns to tuples in @columns_list
 /// according to @object_columns and adds all tuple's subcolumns if needed.
 void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescription & object_columns, bool with_subcolumns);
 
-NameSet getNamesOfObjectColumns(const NamesAndTypesList & columns_list);
+/// Checks whether @columns contain any column with dynamic subcolumns.
-bool hasObjectColumns(const ColumnsDescription & columns);
+bool hasDynamicSubcolumns(const ColumnsDescription & columns);
-void finalizeObjectColumns(const MutableColumns & columns);
 
 /// Updates types of objects in @object_columns inplace
 /// according to types in new_columns.
-void updateObjectColumns(ColumnsDescription & object_columns, const NamesAndTypesList & new_columns);
+void updateObjectColumns(
+    ColumnsDescription & object_columns,
+    const ColumnsDescription & storage_columns,
+    const NamesAndTypesList & new_columns);
 
 using DataTypeTuplePtr = std::shared_ptr<DataTypeTuple>;
 
@@ -142,13 +146,15 @@ public:
     {
         if (num_dimensions_to_fold == 0)
             return x;
-        Array res(1,x);
+
+        Array res(1, x);
         for (size_t i = 1; i < num_dimensions_to_fold; ++i)
         {
             Array new_res;
             new_res.push_back(std::move(res));
             res = std::move(new_res);
         }
+
         return res;
     }
 
@@ -163,7 +169,7 @@ private:
 /// columns-like objects from entry to which Iterator points.
 /// columns-like object should have fields "name" and "type".
 template <typename Iterator, typename EntryColumnsGetter>
-ColumnsDescription getObjectColumns(
+ColumnsDescription getConcreteObjectColumns(
     Iterator begin, Iterator end,
     const ColumnsDescription & storage_columns,
     EntryColumnsGetter && entry_columns_getter)
@@ -176,14 +182,8 @@ ColumnsDescription getObjectColumns(
     /// dummy column will be removed.
     for (const auto & column : storage_columns)
     {
-        if (isObject(column.type))
+        if (column.type->hasDynamicSubcolumns())
-        {
+            types_in_entries[column.name].push_back(createConcreteEmptyDynamicColumn(column.type));
-            auto tuple_type = std::make_shared<DataTypeTuple>(
-                DataTypes{std::make_shared<DataTypeUInt8>()},
-                Names{ColumnObject::COLUMN_NAME_DUMMY});
-
-            types_in_entries[column.name].push_back(std::move(tuple_type));
-        }
     }
 
     for (auto it = begin; it != end; ++it)
@@ -192,14 +192,17 @@ ColumnsDescription getObjectColumns(
         for (const auto & column : entry_columns)
         {
             auto storage_column = storage_columns.tryGetPhysical(column.name);
-            if (storage_column && isObject(storage_column->type))
+            if (storage_column && storage_column->type->hasDynamicSubcolumns())
                 types_in_entries[column.name].push_back(column.type);
         }
     }
 
     ColumnsDescription res;
     for (const auto & [name, types] : types_in_entries)
-        res.add({name, getLeastCommonTypeForObject(types)});
+    {
+        auto storage_column = storage_columns.getPhysical(name);
+        res.add({name, getLeastCommonTypeForDynamicColumns(storage_column.type, types)});
+    }
 
     return res;
 }

src/DataTypes/Serializations/ISerialization.h

@@ -249,7 +249,9 @@ public:
     };
 
     /// Call before serializeBinaryBulkWithMultipleStreams chain to write something before first mark.
+    /// Column may be used only to retrieve the structure.
     virtual void serializeBinaryBulkStatePrefix(
+        const IColumn & /*column*/,
         SerializeBinaryBulkSettings & /*settings*/,
         SerializeBinaryBulkStatePtr & /*state*/) const {}
 

src/DataTypes/Serializations/SerializationArray.cpp

@@ -246,11 +246,13 @@ void SerializationArray::enumerateStreams(
 }
 
 void SerializationArray::serializeBinaryBulkStatePrefix(
+    const IColumn & column,
     SerializeBinaryBulkSettings & settings,
     SerializeBinaryBulkStatePtr & state) const
 {
     settings.path.push_back(Substream::ArrayElements);
-    nested->serializeBinaryBulkStatePrefix(settings, state);
+    const auto & column_array = assert_cast<const ColumnArray &>(column);
+    nested->serializeBinaryBulkStatePrefix(column_array.getData(), settings, state);
     settings.path.pop_back();
 }
 

src/DataTypes/Serializations/SerializationArray.h

@@ -41,6 +41,7 @@ public:
         const SubstreamData & data) const override;
 
     void serializeBinaryBulkStatePrefix(
+            const IColumn & column,
             SerializeBinaryBulkSettings & settings,
             SerializeBinaryBulkStatePtr & state) const override;
 

src/DataTypes/Serializations/SerializationLowCardinality.cpp

@@ -221,6 +221,7 @@ struct DeserializeStateLowCardinality : public ISerialization::DeserializeBinary
 };
 
 void SerializationLowCardinality::serializeBinaryBulkStatePrefix(
+    const IColumn & /*column*/,
     SerializeBinaryBulkSettings & settings,
     SerializeBinaryBulkStatePtr & state) const
 {

src/DataTypes/Serializations/SerializationLowCardinality.h

@@ -23,6 +23,7 @@ public:
         const SubstreamData & data) const override;
 
     void serializeBinaryBulkStatePrefix(
+            const IColumn & column,
             SerializeBinaryBulkSettings & settings,
             SerializeBinaryBulkStatePtr & state) const override;
 

src/DataTypes/Serializations/SerializationMap.cpp

@@ -270,10 +270,11 @@ void SerializationMap::enumerateStreams(
 }
 
 void SerializationMap::serializeBinaryBulkStatePrefix(
+    const IColumn & column,
     SerializeBinaryBulkSettings & settings,
     SerializeBinaryBulkStatePtr & state) const
 {
-    nested->serializeBinaryBulkStatePrefix(settings, state);
+    nested->serializeBinaryBulkStatePrefix(extractNestedColumn(column), settings, state);
 }
 
 void SerializationMap::serializeBinaryBulkStateSuffix(

src/DataTypes/Serializations/SerializationMap.h

@@ -37,6 +37,7 @@ public:
         const SubstreamData & data) const override;
 
     void serializeBinaryBulkStatePrefix(
+        const IColumn & column,
         SerializeBinaryBulkSettings & settings,
         SerializeBinaryBulkStatePtr & state) const override;
 

src/DataTypes/Serializations/SerializationNamed.cpp

@@ -17,11 +17,12 @@ void SerializationNamed::enumerateStreams(
 }
 
 void SerializationNamed::serializeBinaryBulkStatePrefix(
+    const IColumn & column,
     SerializeBinaryBulkSettings & settings,
     SerializeBinaryBulkStatePtr & state) const
 {
     addToPath(settings.path);
-    nested_serialization->serializeBinaryBulkStatePrefix(settings, state);
+    nested_serialization->serializeBinaryBulkStatePrefix(column, settings, state);
     settings.path.pop_back();
 }
 

src/DataTypes/Serializations/SerializationNamed.h

@@ -31,6 +31,7 @@ public:
         const SubstreamData & data) const override;
 
     void serializeBinaryBulkStatePrefix(
+        const IColumn & column,
         SerializeBinaryBulkSettings & settings,
         SerializeBinaryBulkStatePtr & state) const override;
 

src/DataTypes/Serializations/SerializationNullable.cpp

@@ -70,11 +70,13 @@ void SerializationNullable::enumerateStreams(
 }
 
 void SerializationNullable::serializeBinaryBulkStatePrefix(
+        const IColumn & column,
         SerializeBinaryBulkSettings & settings,
         SerializeBinaryBulkStatePtr & state) const
 {
     settings.path.push_back(Substream::NullableElements);
-    nested->serializeBinaryBulkStatePrefix(settings, state);
+    const auto & column_nullable = assert_cast<const ColumnNullable &>(column);
+    nested->serializeBinaryBulkStatePrefix(column_nullable.getNestedColumn(), settings, state);
     settings.path.pop_back();
 }
 

src/DataTypes/Serializations/SerializationNullable.h

@@ -19,6 +19,7 @@ public:
         const SubstreamData & data) const override;
 
     void serializeBinaryBulkStatePrefix(
+            const IColumn & column,
             SerializeBinaryBulkSettings & settings,
             SerializeBinaryBulkStatePtr & state) const override;
 

src/DataTypes/Serializations/SerializationObject.cpp

@@ -13,8 +13,6 @@
 #include <Columns/ColumnString.h>
 #include <Functions/FunctionsConversion.h>
 
-#include <Common/FieldVisitorToString.h>
-
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <IO/VarInt.h>
@@ -30,6 +28,7 @@ namespace ErrorCodes
     extern const int NOT_IMPLEMENTED;
     extern const int INCORRECT_DATA;
     extern const int CANNOT_READ_ALL_DATA;
+    extern const int ARGUMENT_OUT_OF_BOUND;
     extern const int LOGICAL_ERROR;
 }
 
@@ -141,7 +140,6 @@ void SerializationObject<Parser>::checkSerializationIsSupported(const TSettings
 template <typename Parser>
 struct SerializationObject<Parser>::SerializeStateObject : public ISerialization::SerializeBinaryBulkState
 {
-    bool is_first = true;
     DataTypePtr nested_type;
     SerializationPtr nested_serialization;
     SerializeBinaryBulkStatePtr nested_state;
@@ -158,6 +156,7 @@ struct SerializationObject<Parser>::DeserializeStateObject : public ISerializati
 
 template <typename Parser>
 void SerializationObject<Parser>::serializeBinaryBulkStatePrefix(
+    const IColumn & column,
     SerializeBinaryBulkSettings & settings,
     SerializeBinaryBulkStatePtr & state) const
 {
@@ -166,15 +165,34 @@ void SerializationObject<Parser>::serializeBinaryBulkStatePrefix(
         throw Exception(ErrorCodes::NOT_IMPLEMENTED,
             "DataTypeObject doesn't support serialization with non-trivial state");
 
+    const auto & column_object = assert_cast<const ColumnObject &>(column);
+    if (!column_object.isFinalized())
+    {
+        auto finalized = column_object.cloneFinalized();
+        serializeBinaryBulkStatePrefix(*finalized, settings, state);
+        return;
+    }
+
     settings.path.push_back(Substream::ObjectStructure);
     auto * stream = settings.getter(settings.path);
-    settings.path.pop_back();
 
     if (!stream)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Missing stream for kind of binary serialization");
 
+    auto [tuple_column, tuple_type] = unflattenObjectToTuple(column_object);
+
     writeIntBinary(static_cast<UInt8>(BinarySerializationKind::TUPLE), *stream);
-    state = std::make_shared<SerializeStateObject>();
+    writeStringBinary(tuple_type->getName(), *stream);
+
+    auto state_object = std::make_shared<SerializeStateObject>();
+    state_object->nested_type = tuple_type;
+    state_object->nested_serialization = tuple_type->getDefaultSerialization();
+
+    settings.path.back() = Substream::ObjectData;
+    state_object->nested_serialization->serializeBinaryBulkStatePrefix(*tuple_column, settings, state_object->nested_state);
+
+    state = std::move(state_object);
+    settings.path.pop_back();
 }
 
 template <typename Parser>
@@ -261,33 +279,14 @@ void SerializationObject<Parser>::serializeBinaryBulkWithMultipleStreams(
 
     if (!column_object.isFinalized())
     {
-        auto finalized_object = column_object.clone();
+        auto finalized = column_object.cloneFinalized();
-        assert_cast<ColumnObject &>(*finalized_object).finalize();
+        serializeBinaryBulkWithMultipleStreams(*finalized, offset, limit, settings, state);
-        serializeBinaryBulkWithMultipleStreams(*finalized_object, offset, limit, settings, state);
         return;
     }
 
     auto [tuple_column, tuple_type] = unflattenObjectToTuple(column_object);
 
-    if (state_object->is_first)
+    if (!state_object->nested_type->equals(*tuple_type))
-    {
-        /// Actually it's a part of serializeBinaryBulkStatePrefix,
-        /// but it cannot be done there, because we have to know the
-        /// structure of column.
-
-        settings.path.push_back(Substream::ObjectStructure);
-        if (auto * stream = settings.getter(settings.path))
-            writeStringBinary(tuple_type->getName(), *stream);
-
-        state_object->nested_type = tuple_type;
-        state_object->nested_serialization = tuple_type->getDefaultSerialization();
-        state_object->is_first = false;
-
-        settings.path.back() = Substream::ObjectData;
-        state_object->nested_serialization->serializeBinaryBulkStatePrefix(settings, state_object->nested_state);
-        settings.path.pop_back();
-    }
-    else if (!state_object->nested_type->equals(*tuple_type))
     {
         throw Exception(ErrorCodes::LOGICAL_ERROR,
             "Types of internal column of Object mismatched. Expected: {}, Got: {}",
@@ -411,18 +410,63 @@ void SerializationObject<Parser>::serializeTextImpl(const IColumn & column, size
     writeChar('{', ostr);
     for (auto it = subcolumns.begin(); it != subcolumns.end(); ++it)
     {
+        const auto & entry = *it;
         if (it != subcolumns.begin())
             writeCString(",", ostr);
 
-        writeDoubleQuoted((*it)->path.getPath(), ostr);
+        writeDoubleQuoted(entry->path.getPath(), ostr);
         writeChar(':', ostr);
-
+        serializeTextFromSubcolumn(entry->data, row_num, ostr, settings);
-        auto serialization = (*it)->data.getLeastCommonType()->getDefaultSerialization();
-        serialization->serializeTextJSON((*it)->data.getFinalizedColumn(), row_num, ostr, settings);
     }
     writeChar('}', ostr);
 }
 
+template <typename Parser>
+void SerializationObject<Parser>::serializeTextFromSubcolumn(
+    const ColumnObject::Subcolumn & subcolumn, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
+{
+    const auto & least_common_type = subcolumn.getLeastCommonType();
+
+    if (subcolumn.isFinalized())
+    {
+        const auto & finalized_column = subcolumn.getFinalizedColumn();
+        auto info = least_common_type->getSerializationInfo(finalized_column);
+        auto serialization = least_common_type->getSerialization(*info);
+        serialization->serializeTextJSON(finalized_column, row_num, ostr, settings);
+        return;
+    }
+
+    size_t ind = row_num;
+    if (ind < subcolumn.getNumberOfDefaultsInPrefix())
+    {
+        /// Suboptimal, but it should happen rarely.
+        auto tmp_column = subcolumn.getLeastCommonType()->createColumn();
+        tmp_column->insertDefault();
+
+        auto info = least_common_type->getSerializationInfo(*tmp_column);
+        auto serialization = least_common_type->getSerialization(*info);
+        serialization->serializeTextJSON(*tmp_column, 0, ostr, settings);
+        return;
+    }
+
+    ind -= subcolumn.getNumberOfDefaultsInPrefix();
+    for (const auto & part : subcolumn.getData())
+    {
+        if (ind < part->size())
+        {
+            auto part_type = getDataTypeByColumn(*part);
+            auto info = part_type->getSerializationInfo(*part);
+            auto serialization = part_type->getSerialization(*info);
+            serialization->serializeTextJSON(*part, ind, ostr, settings);
+            return;
+        }
+
+        ind -= part->size();
+    }
+
+    throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Index ({}) for text serialization is out of range", row_num);
+}
+
 template <typename Parser>
 void SerializationObject<Parser>::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {

src/DataTypes/Serializations/SerializationObject.h

@@ -8,7 +8,7 @@ namespace DB
 {
 
 /** Serialization for data type Object.
-  * Supported only test serialization/deserialization.
+  * Supported only text serialization/deserialization.
   * and binary bulk serialization/deserialization without position independent
   * encoding, i.e. serialization/deserialization into Native format.
   */
@@ -31,6 +31,7 @@ public:
       */
 
     void serializeBinaryBulkStatePrefix(
+        const IColumn & column,
         SerializeBinaryBulkSettings & settings,
         SerializeBinaryBulkStatePtr & state) const override;
 
@@ -104,6 +105,7 @@ private:
     void deserializeTextImpl(IColumn & column, Reader && reader) const;
 
     void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const;
+    void serializeTextFromSubcolumn(const ColumnObject::Subcolumn & subcolumn, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const;
 
     /// Pool of parser objects to make SerializationObject thread safe.
     mutable SimpleObjectPool<Parser> parsers_pool;

src/DataTypes/Serializations/SerializationSparse.cpp

@@ -178,11 +178,16 @@ void SerializationSparse::enumerateStreams(
 }
 
 void SerializationSparse::serializeBinaryBulkStatePrefix(
+    const IColumn & column,
     SerializeBinaryBulkSettings & settings,
     SerializeBinaryBulkStatePtr & state) const
 {
     settings.path.push_back(Substream::SparseElements);
-    nested->serializeBinaryBulkStatePrefix(settings, state);
+    if (const auto * column_sparse = typeid_cast<const ColumnSparse *>(&column))
+        nested->serializeBinaryBulkStatePrefix(column_sparse->getValuesColumn(), settings, state);
+    else
+        nested->serializeBinaryBulkStatePrefix(column, settings, state);
+
     settings.path.pop_back();
 }
 

src/DataTypes/Serializations/SerializationSparse.h

@@ -33,6 +33,7 @@ public:
         const SubstreamData & data) const override;
 
     void serializeBinaryBulkStatePrefix(
+        const IColumn & column,
         SerializeBinaryBulkSettings & settings,
         SerializeBinaryBulkStatePtr & state) const override;
 

src/DataTypes/Serializations/SerializationTuple.cpp

@@ -314,6 +314,7 @@ struct DeserializeBinaryBulkStateTuple : public ISerialization::DeserializeBinar
 
 
 void SerializationTuple::serializeBinaryBulkStatePrefix(
+    const IColumn & column,
     SerializeBinaryBulkSettings & settings,
     SerializeBinaryBulkStatePtr & state) const
 {
@@ -321,7 +322,7 @@ void SerializationTuple::serializeBinaryBulkStatePrefix(
     tuple_state->states.resize(elems.size());
 
     for (size_t i = 0; i < elems.size(); ++i)
-        elems[i]->serializeBinaryBulkStatePrefix(settings, tuple_state->states[i]);
+        elems[i]->serializeBinaryBulkStatePrefix(extractElementColumn(column, i), settings, tuple_state->states[i]);
 
     state = std::move(tuple_state);
 }

src/DataTypes/Serializations/SerializationTuple.h

@@ -39,6 +39,7 @@ public:
         const SubstreamData & data) const override;
 
     void serializeBinaryBulkStatePrefix(
+            const IColumn & column,
             SerializeBinaryBulkSettings & settings,
             SerializeBinaryBulkStatePtr & state) const override;
 

src/DataTypes/Serializations/SerializationWrapper.cpp

@@ -13,10 +13,11 @@ void SerializationWrapper::enumerateStreams(
 }
 
 void SerializationWrapper::serializeBinaryBulkStatePrefix(
+    const IColumn & column,
     SerializeBinaryBulkSettings & settings,
     SerializeBinaryBulkStatePtr & state) const
 {
-    nested_serialization->serializeBinaryBulkStatePrefix(settings, state);
+    nested_serialization->serializeBinaryBulkStatePrefix(column, settings, state);
 }
 
 void SerializationWrapper::serializeBinaryBulkStateSuffix(

src/DataTypes/Serializations/SerializationWrapper.h

@@ -26,6 +26,7 @@ public:
         const SubstreamData & data) const override;
 
     void serializeBinaryBulkStatePrefix(
+        const IColumn & column,
         SerializeBinaryBulkSettings & settings,
         SerializeBinaryBulkStatePtr & state) const override;
 

src/DataTypes/Serializations/tests/gtest_object_serialization.cpp

@@ -31,7 +31,7 @@ TEST(SerializationObject, FromString)
         settings.getter = [&out](const auto &) { return &out; };
 
         writeIntBinary(static_cast<UInt8>(1), out);
-        serialization->serializeBinaryBulkStatePrefix(settings, state);
+        serialization->serializeBinaryBulkStatePrefix(*column_string, settings, state);
         serialization->serializeBinaryBulkWithMultipleStreams(*column_string, 0, column_string->size(), settings, state);
         serialization->serializeBinaryBulkStateSuffix(settings, state);
     }

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp

@@ -141,7 +141,7 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
     return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(finalize_callback), object.absolute_path);
 }
 
-void AzureObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
+void AzureObjectStorage::findAllFiles(const std::string & path, RelativePathsWithSize & children) const
 {
     auto client_ptr = client.get();
 

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h

@@ -84,7 +84,7 @@ public:
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
         const WriteSettings & write_settings = {}) override;
 
-    void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
+    void findAllFiles(const std::string & path, RelativePathsWithSize & children) const override;
 
     /// Remove file. Throws exception if file doesn't exists or it's a directory.
     void removeObject(const StoredObject & object) override;

src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp

@@ -282,9 +282,9 @@ std::unique_ptr<IObjectStorage> CachedObjectStorage::cloneObjectStorage(
     return object_storage->cloneObjectStorage(new_namespace, config, config_prefix, context);
 }
 
-void CachedObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
+void CachedObjectStorage::findAllFiles(const std::string & path, RelativePathsWithSize & children) const
 {
-    object_storage->listPrefix(path, children);
+    object_storage->findAllFiles(path, children);
 }
 
 ObjectMetadata CachedObjectStorage::getObjectMetadata(const std::string & path) const

src/Disks/ObjectStorages/Cached/CachedObjectStorage.h

@@ -72,7 +72,7 @@ public:
         const std::string & config_prefix,
         ContextPtr context) override;
 
-    void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
+    void findAllFiles(const std::string & path, RelativePathsWithSize & children) const override;
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 

src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp

@@ -390,7 +390,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage *
     };
 
     RelativePathsWithSize children;
-    source_object_storage->listPrefix(restore_information.source_path, children);
+    source_object_storage->findAllFiles(restore_information.source_path, children);
 
     restore_files(children);
 
@@ -540,7 +540,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject
     };
 
     RelativePathsWithSize children;
-    source_object_storage->listPrefix(restore_information.source_path + "operations/", children);
+    source_object_storage->findAllFiles(restore_information.source_path + "operations/", children);
     restore_file_operations(children);
 
     if (restore_information.detached)