Merge branch 'master' into LessReadInOrder

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,7 +12,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py
 - Backward Incompatible Change
 - Build/Testing/Packaging Improvement
 - Documentation (changelog entry is not required)
-- Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+- Critical Bug Fix (crash, data loss, RBAC)
 - Bug Fix (user-visible misbehavior in an official stable release)
 - CI Fix or Improvement (changelog entry is not required)
 - Not for changelog (changelog entry is not required)

README.md

@@ -47,6 +47,7 @@ Upcoming meetups
 * [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21
 * [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26
 * [Amsterdam Meetup](https://www.meetup.com/clickhouse-netherlands-user-group/events/303638814) - December 3
+* [Stockholm Meetup](https://www.meetup.com/clickhouse-stockholm-user-group/events/304382411) - December 9
 * [New York Meetup](https://www.meetup.com/clickhouse-new-york-user-group/events/304268174) - December 9
 * [San Francisco Meetup](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/304286951/) - December 12
 

contrib/SimSIMD

@@ -1 +1 @@
-Subproject commit ee3c9c9c00b51645f62a1a9e99611b78c0052a21
+Subproject commit fa60f1b8e3582c50978f0ae86c2ebb6c9af957f3

docker/keeper/Dockerfile

@@ -1,7 +1,7 @@
 # The Dockerfile.ubuntu exists for the tests/ci/docker_server.py script
 # If the image is built from Dockerfile.alpine, then the `-alpine` suffix is added automatically,
 # so the only purpose of Dockerfile.ubuntu is to push `latest`, `head` and so on w/o suffixes
-FROM ubuntu:20.04 AS glibc-donor
+FROM ubuntu:22.04 AS glibc-donor
 ARG TARGETARCH
 
 RUN arch=${TARGETARCH:-amd64} \
@@ -9,7 +9,11 @@ RUN arch=${TARGETARCH:-amd64} \
         amd64) rarch=x86_64 ;; \
         arm64) rarch=aarch64 ;; \
     esac \
-    && ln -s "${rarch}-linux-gnu" /lib/linux-gnu
+    && ln -s "${rarch}-linux-gnu" /lib/linux-gnu \
+    && case $arch in \
+        amd64) ln /lib/linux-gnu/ld-linux-x86-64.so.2 /lib/linux-gnu/ld-2.35.so ;; \
+        arm64) ln /lib/linux-gnu/ld-linux-aarch64.so.1 /lib/linux-gnu/ld-2.35.so ;; \
+    esac
 
 
 FROM alpine
@@ -20,7 +24,7 @@ ENV LANG=en_US.UTF-8 \
     TZ=UTC \
     CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml
 
-COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/
+COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.35.so /lib/
 COPY --from=glibc-donor /etc/nsswitch.conf /etc/
 COPY entrypoint.sh /entrypoint.sh
 

docker/keeper/entrypoint.sh

@@ -1,21 +1,31 @@
 #!/bin/bash
 
-set +x
 set -eo pipefail
 shopt -s nullglob
 
 DO_CHOWN=1
-if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then
+if [[ "${CLICKHOUSE_RUN_AS_ROOT:=0}" = "1" || "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]]; then
     DO_CHOWN=0
 fi
 
-CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
-CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+# CLICKHOUSE_UID and CLICKHOUSE_GID are kept for backward compatibility, but deprecated
+# One must use either "docker run --user" or CLICKHOUSE_RUN_AS_ROOT=1 to run the process as
+# FIXME: Remove ALL CLICKHOUSE_UID CLICKHOUSE_GID before 25.3
+if [[ "${CLICKHOUSE_UID:-}" || "${CLICKHOUSE_GID:-}" ]]; then
+    echo 'WARNING: Support for CLICKHOUSE_UID/CLICKHOUSE_GID will be removed in a couple of releases.' >&2
+    echo 'WARNING: Either use a proper "docker run --user=xxx:xxxx" argument instead of CLICKHOUSE_UID/CLICKHOUSE_GID' >&2
+    echo 'WARNING: or set "CLICKHOUSE_RUN_AS_ROOT=1" ENV to run the clickhouse-server as root:root' >&2
+fi
 
-# support --user
-if [ "$(id -u)" = "0" ]; then
-    USER=$CLICKHOUSE_UID
-    GROUP=$CLICKHOUSE_GID
+# support `docker run --user=xxx:xxxx`
+if [[ "$(id -u)" = "0" ]]; then
+    if [[ "$CLICKHOUSE_RUN_AS_ROOT" = 1 ]]; then
+        USER=0
+        GROUP=0
+    else
+        USER="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
+        GROUP="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+    fi
     if command -v gosu &> /dev/null; then
         gosu="gosu $USER:$GROUP"
     elif command -v su-exec &> /dev/null; then
@@ -82,11 +92,11 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
 
     # There is a config file. It is already tested with gosu (if it is readably by keeper user)
     if [ -f "$KEEPER_CONFIG" ]; then
-        exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@"
+        exec $gosu clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@"
     fi
 
     # There is no config file. Will use embedded one
-    exec $gosu /usr/bin/clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
+    exec $gosu clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
 fi
 
 # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image

docker/server/Dockerfile.ubuntu

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # see https://github.com/moby/moby/issues/4032#issuecomment-192327844
 # It could be removed after we move on a version 23:04+
@@ -88,34 +88,32 @@ RUN if [ -n "${single_binary_location_url}" ]; then \
 #docker-official-library:on
 
 # A fallback to installation from ClickHouse repository
-RUN if ! clickhouse local -q "SELECT ''" > /dev/null 2>&1; then \
-        apt-get update \
-        && apt-get install --yes --no-install-recommends \
-            apt-transport-https \
-            dirmngr \
-            gnupg2 \
-        && mkdir -p /etc/apt/sources.list.d \
-        && GNUPGHOME=$(mktemp -d) \
-        && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
-            --keyring /usr/share/keyrings/clickhouse-keyring.gpg \
-            --keyserver hkp://keyserver.ubuntu.com:80 \
-            --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
-        && rm -rf "$GNUPGHOME" \
-        && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
-        && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
-        && echo "installing from repository: ${REPOSITORY}" \
-        && apt-get update \
-        && for package in ${PACKAGES}; do \
-            packages="${packages} ${package}=${VERSION}" \
-        ; done \
-        && apt-get install --allow-unauthenticated --yes --no-install-recommends ${packages} || exit 1 \
-        && rm -rf \
-            /var/lib/apt/lists/* \
-            /var/cache/debconf \
-            /tmp/* \
-        && apt-get autoremove --purge -yq libksba8 \
-        && apt-get autoremove -yq \
-    ; fi
+# It works unless the clickhouse binary already exists
+RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \
+    ; apt-get update \
+    && apt-get install --yes --no-install-recommends \
+        dirmngr \
+        gnupg2 \
+    && mkdir -p /etc/apt/sources.list.d \
+    && GNUPGHOME=$(mktemp -d) \
+    && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
+        --keyring /usr/share/keyrings/clickhouse-keyring.gpg \
+        --keyserver hkp://keyserver.ubuntu.com:80 \
+        --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
+    && rm -rf "$GNUPGHOME" \
+    && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
+    && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
+    && echo "installing from repository: ${REPOSITORY}" \
+    && apt-get update \
+    && for package in ${PACKAGES}; do \
+        packages="${packages} ${package}=${VERSION}" \
+    ; done \
+    && apt-get install --yes --no-install-recommends ${packages} || exit 1 \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /var/cache/debconf \
+        /tmp/* \
+    && apt-get autoremove --purge -yq dirmngr gnupg2
 
 # post install
 # we need to allow "others" access to clickhouse folder, because docker container
@@ -126,8 +124,6 @@ RUN clickhouse-local -q 'SELECT * FROM system.build_options' \
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
 ENV TZ UTC
 
 RUN mkdir /docker-entrypoint-initdb.d

docker/server/README.md

@@ -1,3 +1,11 @@
+<!---
+The README.md is generated by README.sh from the following sources:
+- README.src/content.md
+- README.src/license.md
+
+If you want to change it, edit these files
+-->
+
 # ClickHouse Server Docker Image
 
 ## What is ClickHouse?
@@ -8,6 +16,7 @@ ClickHouse works 100-1000x faster than traditional database management systems,
 
 For more information and documentation see https://clickhouse.com/.
 
+<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
 ## Versions
 
 -	The `latest` tag points to the latest release of the latest stable branch.
@@ -16,10 +25,12 @@ For more information and documentation see https://clickhouse.com/.
 -	The tag `head` is built from the latest commit to the default branch.
 -	Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
 
+<!-- REMOVE UNTIL HERE -->
 ### Compatibility
 
 -	The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
 -	The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
+-	Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run --security-opt seccomp=unconfined` instead, however that has security implications.
 
 ## How to use this image
 
@@ -29,7 +40,7 @@ For more information and documentation see https://clickhouse.com/.
 docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```
 
-By default, ClickHouse will be accessible only via the Docker network. See the [networking section below](#networking).
+By default, ClickHouse will be accessible only via the Docker network. See the **networking** section below.
 
 By default, starting above server instance will be run as the `default` user without password.
 
@@ -46,7 +57,7 @@ More information about the [ClickHouse client](https://clickhouse.com/docs/en/in
 ### connect to it using curl
 
 ```bash
-echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server curlimages/curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
+echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server buildpack-deps:curl curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
 ```
 
 More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/).
@@ -69,7 +80,7 @@ echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @-
 
 `22.6.3.35`
 
-or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
+Or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
 
 ```bash
 docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
@@ -87,8 +98,8 @@ Typically you may want to mount the following folders inside your container to a
 
 ```bash
 docker run -d \
-    -v $(realpath ./ch_data):/var/lib/clickhouse/ \
-    -v $(realpath ./ch_logs):/var/log/clickhouse-server/ \
+    -v "$PWD/ch_data:/var/lib/clickhouse/" \
+    -v "$PWD/ch_logs:/var/log/clickhouse-server/" \
     --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```
 
@@ -110,6 +121,8 @@ docker run -d \
     --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```
 
+Read more in [knowledge base](https://clickhouse.com/docs/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker).
+
 ## Configuration
 
 The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/).
@@ -125,8 +138,8 @@ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /pa
 ### Start server as custom user
 
 ```bash
-# $(pwd)/data/clickhouse should exist and be owned by current user
-docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
+# $PWD/data/clickhouse should exist and be owned by current user
+docker run --rm --user "${UID}:${GID}" --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
 ```
 
 When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start.
@@ -134,7 +147,7 @@ When you use the image with local directories mounted, you probably want to spec
 ### Start server from root (useful in case of enabled user namespace)
 
 ```bash
-docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
+docker run --rm -e CLICKHOUSE_RUN_AS_ROOT=1 --name clickhouse-server-userns -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
 ```
 
 ### How to create default database and user on starting

docker/server/README.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -ueo pipefail
+
+# A script to generate README.sh close to as it done in https://github.com/docker-library/docs
+
+WORKDIR=$(dirname "$0")
+SCRIPT_NAME=$(basename "$0")
+CONTENT=README.src/content.md
+LICENSE=README.src/license.md
+cd "$WORKDIR"
+
+R=README.md
+
+cat > "$R" <<EOD
+<!---
+The $R is generated by $SCRIPT_NAME from the following sources:
+- $CONTENT
+- $LICENSE
+
+If you want to change it, edit these files
+-->
+
+EOD
+
+cat "$CONTENT" >> "$R"
+
+cat >> "$R" <<EOD
+
+## License
+
+$(cat $LICENSE)
+EOD
+
+# Remove %%LOGO%% from the file with one line below
+sed -i '/^%%LOGO%%/,+1d' "$R"
+
+# Replace each %%IMAGE%% with our `clickhouse/clickhouse-server`
+sed -i '/%%IMAGE%%/s:%%IMAGE%%:clickhouse/clickhouse-server:g' $R

docker/server/README.src/README-short.txt

@@ -0,0 +1 @@
+ClickHouse is the fastest and most resource efficient OSS database for real-time apps and analytics.

docker/server/README.src/content.md

@@ -0,0 +1,170 @@
+# ClickHouse Server Docker Image
+
+## What is ClickHouse?
+
+%%LOGO%%
+
+ClickHouse is an open-source column-oriented DBMS (columnar database management system) for online analytical processing (OLAP) that allows users to generate analytical reports using SQL queries in real-time.
+
+ClickHouse works 100-1000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second. With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance.
+
+For more information and documentation see https://clickhouse.com/.
+
+<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
+## Versions
+
+-	The `latest` tag points to the latest release of the latest stable branch.
+-	Branch tags like `22.2` point to the latest release of the corresponding branch.
+-	Full version tags like `22.2.3.5` point to the corresponding release.
+-	The tag `head` is built from the latest commit to the default branch.
+-	Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
+
+<!-- REMOVE UNTIL HERE -->
+### Compatibility
+
+-	The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
+-	The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
+-	Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run --security-opt seccomp=unconfined` instead, however that has security implications.
+
+## How to use this image
+
+### start server instance
+
+```bash
+docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+```
+
+By default, ClickHouse will be accessible only via the Docker network. See the **networking** section below.
+
+By default, starting above server instance will be run as the `default` user without password.
+
+### connect to it from a native client
+
+```bash
+docker run -it --rm --link some-clickhouse-server:clickhouse-server --entrypoint clickhouse-client %%IMAGE%% --host clickhouse-server
+# OR
+docker exec -it some-clickhouse-server clickhouse-client
+```
+
+More information about the [ClickHouse client](https://clickhouse.com/docs/en/interfaces/cli/).
+
+### connect to it using curl
+
+```bash
+echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server buildpack-deps:curl curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
+```
+
+More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/).
+
+### stopping / removing the container
+
+```bash
+docker stop some-clickhouse-server
+docker rm some-clickhouse-server
+```
+
+### networking
+
+You can expose your ClickHouse running in docker by [mapping a particular port](https://docs.docker.com/config/containers/container-networking/) from inside the container using host ports:
+
+```bash
+docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @-
+```
+
+`22.6.3.35`
+
+Or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
+
+```bash
+docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @-
+```
+
+`22.6.3.35`
+
+### Volumes
+
+Typically you may want to mount the following folders inside your container to achieve persistency:
+
+-	`/var/lib/clickhouse/` - main folder where ClickHouse stores the data
+-	`/var/log/clickhouse-server/` - logs
+
+```bash
+docker run -d \
+    -v "$PWD/ch_data:/var/lib/clickhouse/" \
+    -v "$PWD/ch_logs:/var/log/clickhouse-server/" \
+    --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+```
+
+You may also want to mount:
+
+-	`/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments
+-	`/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments
+-	`/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below).
+
+### Linux capabilities
+
+ClickHouse has some advanced functionality, which requires enabling several [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html).
+
+They are optional and can be enabled using the following [docker command-line arguments](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities):
+
+```bash
+docker run -d \
+    --cap-add=SYS_NICE --cap-add=NET_ADMIN --cap-add=IPC_LOCK \
+    --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+```
+
+Read more in [knowledge base](https://clickhouse.com/docs/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker).
+
+## Configuration
+
+The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/).
+
+ClickHouse configuration is represented with a file "config.xml" ([documentation](https://clickhouse.com/docs/en/operations/configuration_files/))
+
+### Start server instance with custom configuration
+
+```bash
+docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml %%IMAGE%%
+```
+
+### Start server as custom user
+
+```bash
+# $PWD/data/clickhouse should exist and be owned by current user
+docker run --rm --user "${UID}:${GID}" --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" %%IMAGE%%
+```
+
+When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start.
+
+### Start server from root (useful in case of enabled user namespace)
+
+```bash
+docker run --rm -e CLICKHOUSE_RUN_AS_ROOT=1 --name clickhouse-server-userns -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" %%IMAGE%%
+```
+
+### How to create default database and user on starting
+
+Sometimes you may want to create a user (user named `default` is used by default) and database on a container start. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`:
+
+```bash
+docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp %%IMAGE%%
+```
+
+## How to extend this image
+
+To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service.  
+Also, you can provide environment variables `CLICKHOUSE_USER` & `CLICKHOUSE_PASSWORD` that will be used for clickhouse-client during initialization.
+
+For example, to add an additional user and database, add the following to `/docker-entrypoint-initdb.d/init-db.sh`:
+
+```bash
+#!/bin/bash
+set -e
+
+clickhouse client -n <<-EOSQL
+    CREATE DATABASE docker;
+    CREATE TABLE docker.docker (x Int32) ENGINE = Log;
+EOSQL
+```

docker/server/README.src/github-repo

@@ -0,0 +1 @@
+https://github.com/ClickHouse/ClickHouse

docker/server/README.src/license.md

@@ -0,0 +1 @@
+View [license information](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) for the software contained in this image.

docker/server/README.src/logo.svg

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 616 616">
+  <defs>
+    <style>
+      .cls-1 {
+        clip-path: url(#clippath);
+      }
+
+      .cls-2 {
+        fill: none;
+      }
+
+      .cls-2, .cls-3, .cls-4 {
+        stroke-width: 0px;
+      }
+
+      .cls-3 {
+        fill: #1e1e1e;
+      }
+
+      .cls-4 {
+        fill: #faff69;
+      }
+    </style>
+    <clipPath id="clippath">
+      <rect class="cls-2" x="83.23" y="71.73" width="472.55" height="472.55"/>
+    </clipPath>
+  </defs>
+  <g id="Layer_2" data-name="Layer 2">
+    <rect class="cls-4" width="616" height="616"/>
+  </g>
+  <g id="Layer_1" data-name="Layer 1">
+    <g class="cls-1">
+      <g>
+        <path class="cls-3" d="m120.14,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
+        <path class="cls-3" d="m208.75,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
+        <path class="cls-3" d="m297.35,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
+        <path class="cls-3" d="m385.94,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
+        <path class="cls-3" d="m474.56,268.36c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.65,2.09,4.65,4.66v79.28c0,2.57-2.09,4.66-4.65,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66v-79.28Z"/>
+      </g>
+    </g>
+  </g>
+</svg>

docker/server/README.src/maintainer.md

@@ -0,0 +1 @@
+[ClickHouse Inc.](%%GITHUB-REPO%%)

docker/server/README.src/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

docker/server/entrypoint.sh

@@ -4,17 +4,28 @@ set -eo pipefail
 shopt -s nullglob
 
 DO_CHOWN=1
-if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then
+if [[ "${CLICKHOUSE_RUN_AS_ROOT:=0}" = "1" || "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]]; then
     DO_CHOWN=0
 fi
 
-CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
-CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+# CLICKHOUSE_UID and CLICKHOUSE_GID are kept for backward compatibility, but deprecated
+# One must use either "docker run --user" or CLICKHOUSE_RUN_AS_ROOT=1 to run the process as
+# FIXME: Remove ALL CLICKHOUSE_UID CLICKHOUSE_GID before 25.3
+if [[ "${CLICKHOUSE_UID:-}" || "${CLICKHOUSE_GID:-}" ]]; then
+    echo 'WARNING: Support for CLICKHOUSE_UID/CLICKHOUSE_GID will be removed in a couple of releases.' >&2
+    echo 'WARNING: Either use a proper "docker run --user=xxx:xxxx" argument instead of CLICKHOUSE_UID/CLICKHOUSE_GID' >&2
+    echo 'WARNING: or set "CLICKHOUSE_RUN_AS_ROOT=1" ENV to run the clickhouse-server as root:root' >&2
+fi
 
-# support --user
-if [ "$(id -u)" = "0" ]; then
-    USER=$CLICKHOUSE_UID
-    GROUP=$CLICKHOUSE_GID
+# support `docker run --user=xxx:xxxx`
+if [[ "$(id -u)" = "0" ]]; then
+    if [[ "$CLICKHOUSE_RUN_AS_ROOT" = 1 ]]; then
+        USER=0
+        GROUP=0
+    else
+        USER="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
+        GROUP="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+    fi
 else
     USER="$(id -u)"
     GROUP="$(id -g)"
@@ -55,14 +66,14 @@ function create_directory_and_do_chown() {
     [ -z "$dir" ] && return
     # ensure directories exist
     if [ "$DO_CHOWN" = "1" ]; then
-        mkdir="mkdir"
+        mkdir=( mkdir )
     else
         # if DO_CHOWN=0 it means that the system does not map root user to "admin" permissions
         # it mainly happens on NFS mounts where root==nobody for security reasons
         # thus mkdir MUST run with user id/gid and not from nobody that has zero permissions
-        mkdir="/usr/bin/clickhouse su "${USER}:${GROUP}" mkdir"
+        mkdir=( clickhouse su "${USER}:${GROUP}" mkdir )
     fi
-    if ! $mkdir -p "$dir"; then
+    if ! "${mkdir[@]}" -p "$dir"; then
         echo "Couldn't create necessary directory: $dir"
         exit 1
     fi
@@ -143,7 +154,7 @@ if [ -n "${RUN_INITDB_SCRIPTS}" ]; then
         fi
 
         # Listen only on localhost until the initialization is done
-        /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
+        clickhouse su "${USER}:${GROUP}" clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
         pid="$!"
 
         # check if clickhouse is ready to accept connections
@@ -203,18 +214,8 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
     CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0}
     export CLICKHOUSE_WATCHDOG_ENABLE
 
-    # An option for easy restarting and replacing clickhouse-server in a container, especially in Kubernetes.
-    # For example, you can replace the clickhouse-server binary to another and restart it while keeping the container running.
-    if [[ "${CLICKHOUSE_DOCKER_RESTART_ON_EXIT:-0}" -eq "1" ]]; then
-        while true; do
-            # This runs the server as a child process of the shell script:
-            /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@" ||:
-            echo >&2 'ClickHouse Server exited, and the environment variable CLICKHOUSE_DOCKER_RESTART_ON_EXIT is set to 1. Restarting the server.'
-        done
-    else
-        # This replaces the shell script with the server:
-        exec /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@"
-    fi
+    # This replaces the shell script with the server:
+    exec clickhouse su "${USER}:${GROUP}" clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@"
 fi
 
 # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image

docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile

@@ -1,16 +0,0 @@
-# Since right now we can't set volumes to the docker during build, we split building container in stages:
-# 1. build base container
-# 2. run base conatiner with mounted volumes
-# 3. commit container as image
-FROM ubuntu:20.04 as clickhouse-test-runner-base
-
-# A volume where directory with clickhouse packages to be mounted,
-# for later installing.
-VOLUME /packages
-
-CMD apt-get update ;\
-    DEBIAN_FRONTEND=noninteractive \
-    apt install -y /packages/clickhouse-common-static_*.deb \
-        /packages/clickhouse-client_*.deb \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*

docs/changelogs/v24.3.13.40-lts.md

@@ -0,0 +1,31 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.3.13.40-lts (7acabd77389) FIXME as compared to v24.3.12.75-lts (7cb5dff8019)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#63976](https://github.com/ClickHouse/ClickHouse/issues/63976): Fix intersect parts when restart after drop range. [#63202](https://github.com/ClickHouse/ClickHouse/pull/63202) ([Han Fei](https://github.com/hanfei1991)).
+* Backported in [#71482](https://github.com/ClickHouse/ClickHouse/issues/71482): Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Backported in [#70451](https://github.com/ClickHouse/ClickHouse/issues/70451): Fix vrash during insertion into FixedString column in PostgreSQL engine. [#69584](https://github.com/ClickHouse/ClickHouse/pull/69584) ([Pavel Kruglov](https://github.com/Avogar)).
+* Backported in [#70619](https://github.com/ClickHouse/ClickHouse/issues/70619): Fix server segfault on creating a materialized view with two selects and an `INTERSECT`, e.g. `CREATE MATERIALIZED VIEW v0 AS (SELECT 1) INTERSECT (SELECT 1);`. [#70264](https://github.com/ClickHouse/ClickHouse/pull/70264) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Backported in [#70877](https://github.com/ClickHouse/ClickHouse/issues/70877): Fix table creation with `CREATE ... AS table_function()` with database `Replicated` and unavailable table function source on secondary replica. [#70511](https://github.com/ClickHouse/ClickHouse/pull/70511) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#70571](https://github.com/ClickHouse/ClickHouse/issues/70571): Ignore all output on async insert with `wait_for_async_insert=1`. Closes [#62644](https://github.com/ClickHouse/ClickHouse/issues/62644). [#70530](https://github.com/ClickHouse/ClickHouse/pull/70530) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Backported in [#71146](https://github.com/ClickHouse/ClickHouse/issues/71146): Ignore frozen_metadata.txt while traversing shadow directory from system.remote_data_paths. [#70590](https://github.com/ClickHouse/ClickHouse/pull/70590) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Backported in [#70682](https://github.com/ClickHouse/ClickHouse/issues/70682): Fix creation of stateful window functions on misaligned memory. [#70631](https://github.com/ClickHouse/ClickHouse/pull/70631) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#71113](https://github.com/ClickHouse/ClickHouse/issues/71113): Fix a crash and a leak in AggregateFunctionGroupArraySorted. [#70820](https://github.com/ClickHouse/ClickHouse/pull/70820) ([Michael Kolupaev](https://github.com/al13n321)).
+* Backported in [#70990](https://github.com/ClickHouse/ClickHouse/issues/70990): Fix a logical error due to negative zeros in the two-level hash table. This closes [#70973](https://github.com/ClickHouse/ClickHouse/issues/70973). [#70979](https://github.com/ClickHouse/ClickHouse/pull/70979) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#71246](https://github.com/ClickHouse/ClickHouse/issues/71246): Fixed named sessions not being closed and hanging on forever under certain circumstances. [#70998](https://github.com/ClickHouse/ClickHouse/pull/70998) ([Márcio Martins](https://github.com/marcio-absmartly)).
+* Backported in [#71371](https://github.com/ClickHouse/ClickHouse/issues/71371): Add try/catch to data parts destructors to avoid terminate. [#71364](https://github.com/ClickHouse/ClickHouse/pull/71364) ([alesapin](https://github.com/alesapin)).
+* Backported in [#71594](https://github.com/ClickHouse/ClickHouse/issues/71594): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#71022](https://github.com/ClickHouse/ClickHouse/issues/71022): Fix dropping of file cache in CHECK query in case of enabled transactions. [#69256](https://github.com/ClickHouse/ClickHouse/pull/69256) ([Anton Popov](https://github.com/CurtizJ)).
+* Backported in [#70384](https://github.com/ClickHouse/ClickHouse/issues/70384): CI: Enable Integration Tests for backport PRs. [#70329](https://github.com/ClickHouse/ClickHouse/pull/70329) ([Max Kainov](https://github.com/maxknv)).
+* Backported in [#70538](https://github.com/ClickHouse/ClickHouse/issues/70538): Remove slow poll() logs in keeper. [#70508](https://github.com/ClickHouse/ClickHouse/pull/70508) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#70971](https://github.com/ClickHouse/ClickHouse/issues/70971): Limiting logging some lines about configs. [#70879](https://github.com/ClickHouse/ClickHouse/pull/70879) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+

docs/en/engines/table-engines/integrations/embedded-rocksdb.md

@@ -4,9 +4,13 @@ sidebar_position: 50
 sidebar_label: EmbeddedRocksDB
 ---
 
+import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
+
 # EmbeddedRocksDB Engine
 
-This engine allows integrating ClickHouse with [rocksdb](http://rocksdb.org/).
+<CloudNotSupportedBadge />
+
+This engine allows integrating ClickHouse with [RocksDB](http://rocksdb.org/).
 
 ## Creating a Table {#creating-a-table}
 

docs/en/engines/table-engines/mergetree-family/annindexes.md

@@ -54,7 +54,7 @@ Parameters:
 - `distance_function`: either `L2Distance` (the [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance) - the length of a
   line between two points in Euclidean space), or `cosineDistance` (the [cosine
   distance](https://en.wikipedia.org/wiki/Cosine_similarity#Cosine_distance)- the angle between two non-zero vectors).
-- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing the vector with reduced precision (optional, default: `bf16`)
+- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing vectors with reduced precision (optional, default: `bf16`)
 - `hnsw_max_connections_per_layer`: the number of neighbors per HNSW graph node, also known as `M` in the [HNSW
   paper](https://doi.org/10.1109/TPAMI.2018.2889473) (optional, default: 32)
 - `hnsw_candidate_list_size_for_construction`: the size of the dynamic candidate list when constructing the HNSW graph, also known as
@@ -92,8 +92,8 @@ Vector similarity indexes currently support two distance functions:
 - `cosineDistance`, also called cosine similarity, is the cosine of the angle between two (non-zero) vectors
   ([Wikipedia](https://en.wikipedia.org/wiki/Cosine_similarity)).
 
-Vector similarity indexes allows storing the vectors in reduced precision formats. Supported scalar kinds are `f64`, `f32`, `f16` or `i8`.
-If no scalar kind was specified during index creation, `f16` is used as default.
+Vector similarity indexes allows storing the vectors in reduced precision formats. Supported scalar kinds are `f64`, `f32`, `f16`, `bf16`,
+and `i8`. If no scalar kind was specified during index creation, `bf16` is used as default.
 
 For normalized data, `L2Distance` is usually a better choice, otherwise `cosineDistance` is recommended to compensate for scale. If no
 distance function was specified during index creation, `L2Distance` is used as default.

docs/en/getting-started/example-datasets/tpch.md

@@ -33,6 +33,21 @@ Then, generate the data. Parameter `-s` specifies the scale factor. For example,
 ./dbgen -s 100
 ```
 
+Detailed table sizes with scale factor 100:
+
+| Table    | size (in rows) | size (compressed in ClickHouse) |
+|----------|----------------|---------------------------------|
+| nation   | 25             | 2 kB                            |
+| region   | 5              | 1 kB                            |
+| part     | 20.000.000     | 895 MB                          |
+| supplier | 1.000.000      | 75 MB                           |
+| partsupp | 80.000.000     | 4.37 GB                         |
+| customer | 15.000.000     | 1.19 GB                         |
+| orders   | 150.000.000    | 6.15 GB                         |
+| lineitem | 600.00.00      | 26.69 GB                        |
+
+(The table sizes in ClickHouse are taken from `system.tables.total_bytes` and based on below table definitions.
+
 Now create tables in ClickHouse.
 
 We stick as closely as possible to the rules of the TPC-H specification:
@@ -151,10 +166,37 @@ clickhouse-client --format_csv_delimiter '|' --query "INSERT INTO orders FORMAT
 clickhouse-client --format_csv_delimiter '|' --query "INSERT INTO lineitem FORMAT CSV" < lineitem.tbl
 ```
 
-The queries are generated by `./qgen -s <scaling_factor>`. Example queries for `s = 100`:
+:::note
+Instead of using tpch-kit and generating the tables by yourself, you can alternatively import the data from a public S3 bucket. Make sure
+to create empty tables first using above `CREATE` statements.
+
+```sql
+-- Scaling factor 1
+INSERT INTO nation SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/nation.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO region SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/region.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO part SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/part.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO supplier SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/supplier.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO partsupp SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/partsupp.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO customer SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/customer.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO orders SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/orders.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO lineitem SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/lineitem.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+
+-- Scaling factor 100
+INSERT INTO nation SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/nation.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO region SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/region.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO part SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/part.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO supplier SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/supplier.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO partsupp SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/partsupp.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO customer SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/customer.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO orders SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/orders.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO lineitem SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/lineitem.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+````
+:::
 
 ## Queries
 
+The queries are generated by `./qgen -s <scaling_factor>`. Example queries for `s = 100`:
+
 **Correctness**
 
 The result of the queries agrees with the official results unless mentioned otherwise. To verify, generate a TPC-H database with scale

docs/en/operations/_troubleshooting.md

@@ -65,6 +65,34 @@ sudo rm -f /etc/yum.repos.d/clickhouse.repo
 
 After that follow the [install guide](../getting-started/install.md#from-rpm-packages)
 
+### You Can't Run Docker Container
+
+You are running a simple `docker run clickhouse/clickhouse-server` and it crashes with a stack trace similar to following:
+
+```
+$ docker run -it clickhouse/clickhouse-server
+........
+2024.11.06 21:04:48.912036 [ 1 ] {} <Information> SentryWriter: Sending crash reports is disabled
+Poco::Exception. Code: 1000, e.code() = 0, System exception: cannot start thread, Stack trace (when copying this message, always include the lines below):
+
+0. Poco::ThreadImpl::startImpl(Poco::SharedPtr<Poco::Runnable, Poco::ReferenceCounter, Poco::ReleasePolicy<Poco::Runnable>>) @ 0x00000000157c7b34
+1. Poco::Thread::start(Poco::Runnable&) @ 0x00000000157c8a0e
+2. BaseDaemon::initializeTerminationAndSignalProcessing() @ 0x000000000d267a14
+3. BaseDaemon::initialize(Poco::Util::Application&) @ 0x000000000d2652cb
+4. DB::Server::initialize(Poco::Util::Application&) @ 0x000000000d128b38
+5. Poco::Util::Application::run() @ 0x000000001581cfda
+6. DB::Server::run() @ 0x000000000d1288f0
+7. Poco::Util::ServerApplication::run(int, char**) @ 0x0000000015825e27
+8. mainEntryClickHouseServer(int, char**) @ 0x000000000d125b38
+9. main @ 0x0000000007ea4eee
+10. ? @ 0x00007f67ff946d90
+11. ? @ 0x00007f67ff946e40
+12. _start @ 0x00000000062e802e
+ (version 24.10.1.2812 (official build))
+```
+
+The reason is an old docker daemon with version lower than `20.10.10`. A way to fix it either upgrading it, or running `docker run [--privileged | --security-opt seccomp=unconfined]`. The latter has security implications.
+
 ## Connecting to the Server {#troubleshooting-accepts-no-connections}
 
 Possible issues:

docs/en/operations/query-cache.md

@@ -25,9 +25,10 @@ Query caches can generally be viewed as transactionally consistent or inconsiste
   slowly enough that the database only needs to compute the report once (represented by the first `SELECT` query). Further queries can be
   served directly from the query cache. In this example, a reasonable validity period could be 30 min.
 
-Transactionally inconsistent caching is traditionally provided by client tools or proxy packages interacting with the database. As a result,
-the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side.
-This reduces maintenance effort and avoids redundancy.
+Transactionally inconsistent caching is traditionally provided by client tools or proxy packages (e.g.
+[chproxy](https://www.chproxy.org/configuration/caching/)) interacting with the database. As a result, the same caching logic and
+configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side. This reduces maintenance
+effort and avoids redundancy.
 
 ## Configuration Settings and Usage
 
@@ -138,7 +139,10 @@ is only cached if the query runs longer than 5 seconds. It is also possible to s
 cached - for that use setting [query_cache_min_query_runs](settings/settings.md#query-cache-min-query-runs).
 
 Entries in the query cache become stale after a certain time period (time-to-live). By default, this period is 60 seconds but a different
-value can be specified at session, profile or query level using setting [query_cache_ttl](settings/settings.md#query-cache-ttl).
+value can be specified at session, profile or query level using setting [query_cache_ttl](settings/settings.md#query-cache-ttl). The query
+cache evicts entries "lazily", i.e. when an entry becomes stale, it is not immediately removed from the cache. Instead, when a new entry
+is to be inserted into the query cache, the database checks whether the cache has enough free space for the new entry. If this is not the
+case, the database tries to remove all stale entries. If the cache still has not enough free space, the new entry is not inserted.
 
 Entries in the query cache are compressed by default. This reduces the overall memory consumption at the cost of slower writes into / reads
 from the query cache. To disable compression, use setting [query_cache_compress_entries](settings/settings.md#query-cache-compress-entries).
@@ -188,14 +192,9 @@ Also, results of queries with non-deterministic functions are not cached by defa
 To force caching of results of queries with non-deterministic functions regardless, use setting
 [query_cache_nondeterministic_function_handling](settings/settings.md#query-cache-nondeterministic-function-handling).
 
-Results of queries that involve system tables, e.g. `system.processes` or `information_schema.tables`, are not cached by default. To force
-caching of results of queries with system tables regardless, use setting
-[query_cache_system_table_handling](settings/settings.md#query-cache-system-table-handling).
-
-:::note
-Prior to ClickHouse v23.11, setting 'query_cache_store_results_of_queries_with_nondeterministic_functions = 0 / 1' controlled whether
-results of queries with non-deterministic results were cached. In newer ClickHouse versions, this setting is obsolete and has no effect.
-:::
+Results of queries that involve system tables (e.g. [system.processes](system-tables/processes.md)` or
+[information_schema.tables](system-tables/information_schema.md)) are not cached by default. To force caching of results of queries with
+system tables regardless, use setting [query_cache_system_table_handling](settings/settings.md#query-cache-system-table-handling).
 
 Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
 row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can

docs/en/operations/server-configuration-parameters/settings.md

@@ -131,16 +131,6 @@ Type: UInt64
 
 Default: 8
 
-## background_pool_size
-
-Sets the number of threads performing background merges and mutations for tables with MergeTree engines. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance.
-
-Before changing it, please also take a look at related MergeTree settings, such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation`.
-
-Type: UInt64
-
-Default: 16
-
 ## background_schedule_pool_size
 
 The maximum number of threads that will be used for constantly executing some lightweight periodic operations for replicated tables, Kafka streaming, and DNS cache updates.

docs/en/sql-reference/data-types/dynamic.md

@@ -512,6 +512,8 @@ The result of operator `<` for values `d1` with underlying type `T1` and `d2` wi
 - If `T1 = T2 = T`, the result will be `d1.T < d2.T` (underlying values will be compared).
 - If `T1 != T2`, the result will be `T1 < T2` (type names will be compared).
 
+By default `Dynamic` type is not allowed in `GROUP BY`/`ORDER BY` keys, if you want to use it consider its special comparison rule and enable `allow_suspicious_types_in_group_by`/`allow_suspicious_types_in_order_by` settings.
+
 Examples:
 ```sql
 CREATE TABLE test (d Dynamic) ENGINE=Memory;
@@ -535,7 +537,7 @@ SELECT d, dynamicType(d) FROM test;
 ```
 
 ```sql
-SELECT d, dynamicType(d) FROM test ORDER BY d;
+SELECT d, dynamicType(d) FROM test ORDER BY d SETTINGS allow_suspicious_types_in_order_by=1;
 ```
 
 ```sql
@@ -557,7 +559,7 @@ Example:
 ```sql
 CREATE TABLE test (d Dynamic) ENGINE=Memory;
 INSERT INTO test VALUES (1::UInt32), (1::Int64), (100::UInt32), (100::Int64);
-SELECT d, dynamicType(d) FROM test ORDER by d;
+SELECT d, dynamicType(d) FROM test ORDER BY d SETTINGS allow_suspicious_types_in_order_by=1;
 ```
 
 ```text
@@ -570,7 +572,7 @@ SELECT d, dynamicType(d) FROM test ORDER by d;
 ```
 
 ```sql
-SELECT d, dynamicType(d) FROM test GROUP by d;
+SELECT d, dynamicType(d) FROM test GROUP by d SETTINGS allow_suspicious_types_in_group_by=1;
 ```
 
 ```text
@@ -582,7 +584,7 @@ SELECT d, dynamicType(d) FROM test GROUP by d;
 └─────┴────────────────┘
 ```
 
-**Note**: the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type
+**Note:** the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type
 
 ## Reaching the limit in number of different data types stored inside Dynamic
 

docs/en/sql-reference/data-types/newjson.md

@@ -58,10 +58,10 @@ SELECT json FROM test;
 └───────────────────────────────────┘
 ```
 
-Using CAST from 'String':
+Using CAST from `String`:
 
 ```sql
-SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON as json;
+SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON AS json;
 ```
 
 ```text
@@ -70,7 +70,47 @@ SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON as json
 └────────────────────────────────────────────────┘
 ```
 
-CAST from `JSON`, named `Tuple`, `Map` and `Object('json')` to `JSON` type will be supported later.
+Using CAST from `Tuple`:
+
+```sql
+SELECT (tuple(42 AS b) AS a, [1, 2, 3] AS c, 'Hello, World!' AS d)::JSON AS json;
+```
+
+```text
+┌─json───────────────────────────────────────────┐
+│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │
+└────────────────────────────────────────────────┘
+```
+
+Using CAST from `Map`:
+
+```sql
+SELECT map('a', map('b', 42), 'c', [1,2,3], 'd', 'Hello, World!')::JSON AS json;
+```
+
+```text
+┌─json───────────────────────────────────────────┐
+│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │
+└────────────────────────────────────────────────┘
+```
+
+Using CAST from deprecated `Object('json')`:
+
+```sql
+ SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::Object('json')::JSON AS json;
+ ```
+
+```text
+┌─json───────────────────────────────────────────┐
+│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │
+└────────────────────────────────────────────────┘
+```
+
+:::note
+CAST from `Tuple`/`Map`/`Object('json')` to `JSON` is implemented via serializing the column into `String` column containing JSON objects and deserializing it back to `JSON` type column. 
+:::
+
+CAST between `JSON` types with different arguments will be supported later.
 
 ## Reading JSON paths as subcolumns
 
@@ -630,6 +670,28 @@ SELECT arrayJoin(distinctJSONPathsAndTypes(json)) FROM s3('s3://clickhouse-publi
 └─arrayJoin(distinctJSONPathsAndTypes(json))──────────────────┘
 ```
 
+## ALTER MODIFY COLUMN to JSON type
+
+It's possible to alter an existing table and change the type of the column to the new `JSON` type. Right now only alter from `String` type is supported.
+
+**Example**
+
+```sql
+CREATE TABLE test (json String) ENGINE=MergeTree ORDeR BY tuple();
+INSERT INTO test VALUES ('{"a" : 42}'), ('{"a" : 43, "b" : "Hello"}'), ('{"a" : 44, "b" : [1, 2, 3]}')), ('{"c" : "2020-01-01"}');
+ALTER TABLE test MODIFY COLUMN json JSON;
+SELECT json, json.a, json.b, json.c FROM test;
+```
+
+```text
+┌─json─────────────────────────┬─json.a─┬─json.b──┬─json.c─────┐
+│ {"a":"42"}                   │ 42     │ ᴺᵁᴸᴸ    │ ᴺᵁᴸᴸ       │
+│ {"a":"43","b":"Hello"}       │ 43     │ Hello   │ ᴺᵁᴸᴸ       │
+│ {"a":"44","b":["1","2","3"]} │ 44     │ [1,2,3] │ ᴺᵁᴸᴸ       │
+│ {"c":"2020-01-01"}           │ ᴺᵁᴸᴸ   │ ᴺᵁᴸᴸ    │ 2020-01-01 │
+└──────────────────────────────┴────────┴─────────┴────────────┘
+```
+
 ## Tips for better usage of the JSON type
 
 Before creating `JSON` column and loading data into it, consider the following tips:

docs/en/sql-reference/data-types/variant.md

@@ -441,6 +441,8 @@ SELECT v, variantType(v) FROM test ORDER by v;
 └─────┴────────────────┘
 ```
 
+**Note** by default `Variant` type is not allowed in `GROUP BY`/`ORDER BY` keys, if you want to use it consider its special comparison rule and enable `allow_suspicious_types_in_group_by`/`allow_suspicious_types_in_order_by` settings.
+
 ## JSONExtract functions with Variant
 
 All `JSONExtract*` functions support `Variant` type:

docs/en/sql-reference/statements/select/order-by.md

@@ -291,7 +291,7 @@ All missed values of `expr` column will be filled sequentially and other columns
 To fill multiple columns, add `WITH FILL` modifier with optional parameters after each field name in `ORDER BY` section.
 
 ``` sql
-ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr]
+ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr] [STALENESS const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] [STALENESS numeric_expr]
 [INTERPOLATE [(col [AS expr], ... colN [AS exprN])]]
 ```
 
@@ -300,6 +300,7 @@ When `FROM const_expr` not defined sequence of filling use minimal `expr` field
 When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`.
 When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types, as `days` for Date type, as `seconds` for DateTime type. It also supports [INTERVAL](https://clickhouse.com/docs/en/sql-reference/data-types/special-data-types/interval/) data type representing time and date intervals.
 When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type.
+When `STALENESS const_numeric_expr` is defined, the query will generate rows until the difference from the previous row in the original data exceeds `const_numeric_expr`.
 `INTERPOLATE` can be applied to columns not participating in `ORDER BY WITH FILL`. Such columns are filled based on previous fields values by applying `expr`. If `expr` is not present will repeat previous value. Omitted list will result in including all allowed columns.
 
 Example of a query without `WITH FILL`:
@@ -497,6 +498,64 @@ Result:
 └────────────┴────────────┴──────────┘
 ```
 
+Example of a query without `STALENESS`:
+
+``` sql
+SELECT number as key, 5 * number value, 'original' AS source
+FROM numbers(16) WHERE key % 5 == 0
+ORDER BY key WITH FILL;
+```
+
+Result:
+
+``` text
+    ┌─key─┬─value─┬─source───┐
+ 1. │   0 │     0 │ original │
+ 2. │   1 │     0 │          │
+ 3. │   2 │     0 │          │
+ 4. │   3 │     0 │          │
+ 5. │   4 │     0 │          │
+ 6. │   5 │    25 │ original │
+ 7. │   6 │     0 │          │
+ 8. │   7 │     0 │          │
+ 9. │   8 │     0 │          │
+10. │   9 │     0 │          │
+11. │  10 │    50 │ original │
+12. │  11 │     0 │          │
+13. │  12 │     0 │          │
+14. │  13 │     0 │          │
+15. │  14 │     0 │          │
+16. │  15 │    75 │ original │
+    └─────┴───────┴──────────┘
+```
+
+Same query after applying `STALENESS 3`:
+
+``` sql
+SELECT number as key, 5 * number value, 'original' AS source
+FROM numbers(16) WHERE key % 5 == 0
+ORDER BY key WITH FILL STALENESS 3;
+```
+
+Result:
+
+``` text
+    ┌─key─┬─value─┬─source───┐
+ 1. │   0 │     0 │ original │
+ 2. │   1 │     0 │          │
+ 3. │   2 │     0 │          │
+ 4. │   5 │    25 │ original │
+ 5. │   6 │     0 │          │
+ 6. │   7 │     0 │          │
+ 7. │  10 │    50 │ original │
+ 8. │  11 │     0 │          │
+ 9. │  12 │     0 │          │
+10. │  15 │    75 │ original │
+11. │  16 │     0 │          │
+12. │  17 │     0 │          │
+    └─────┴───────┴──────────┘
+```
+
 Example of a query without `INTERPOLATE`:
 
 ``` sql

programs/client/Client.cpp

@@ -431,7 +431,7 @@ catch (const Exception & e)
     bool need_print_stack_trace = config().getBool("stacktrace", false) && e.code() != ErrorCodes::NETWORK_ERROR;
     std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl << std::endl;
     /// If exception code isn't zero, we should return non-zero return code anyway.
-    return e.code() ? e.code() : -1;
+    return static_cast<UInt8>(e.code()) ? e.code() : -1;
 }
 catch (...)
 {
@@ -1390,7 +1390,8 @@ int mainEntryClickHouseClient(int argc, char ** argv)
     catch (const DB::Exception & e)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
-        return 1;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
     catch (const boost::program_options::error & e)
     {
@@ -1399,7 +1400,8 @@ int mainEntryClickHouseClient(int argc, char ** argv)
     }
     catch (...)
     {
-        std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
-        return 1;
+        std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }

programs/compressor/Compressor.cpp

@@ -9,6 +9,7 @@
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadBufferFromFile.h>
 #include <Compression/CompressedWriteBuffer.h>
+#include <Compression/ParallelCompressedWriteBuffer.h>
 #include <Compression/CompressedReadBuffer.h>
 #include <Compression/CompressedReadBufferFromFile.h>
 #include <IO/WriteHelpers.h>
@@ -17,6 +18,8 @@
 #include <Parsers/ExpressionElementParsers.h>
 #include <Compression/CompressionFactory.h>
 #include <Common/TerminalSize.h>
+#include <Common/ThreadPool.h>
+#include <Common/CurrentMetrics.h>
 #include <Core/Defines.h>
 
 
@@ -29,6 +32,13 @@ namespace DB
     }
 }
 
+namespace CurrentMetrics
+{
+    extern const Metric LocalThread;
+    extern const Metric LocalThreadActive;
+    extern const Metric LocalThreadScheduled;
+}
+
 
 namespace
 {
@@ -77,11 +87,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
             ("decompress,d", "decompress")
             ("offset-in-compressed-file", po::value<size_t>()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)")
             ("offset-in-decompressed-block", po::value<size_t>()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)")
-            ("block-size,b", po::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
+            ("block-size,b", po::value<size_t>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
             ("hc", "use LZ4HC instead of LZ4")
             ("zstd", "use ZSTD instead of LZ4")
             ("codec", po::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
             ("level", po::value<int>(), "compression level for codecs specified via flags")
+            ("threads", po::value<size_t>()->default_value(1), "number of threads for parallel compression")
             ("none", "use no compression instead of LZ4")
             ("stat", "print block statistics of compressed data")
             ("stacktrace", "print stacktrace of exception")
@@ -109,7 +120,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
         bool stat_mode = options.count("stat");
         bool use_none = options.count("none");
         print_stacktrace = options.count("stacktrace");
-        unsigned block_size = options["block-size"].as<unsigned>();
+        size_t block_size = options["block-size"].as<size_t>();
+        size_t num_threads = options["threads"].as<size_t>();
         std::vector<std::string> codecs;
         if (options.count("codec"))
             codecs = options["codec"].as<std::vector<std::string>>();
@@ -117,6 +129,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
         if ((use_lz4hc || use_zstd || use_none) && !codecs.empty())
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong options, codec flags like --zstd and --codec options are mutually exclusive");
 
+        if (num_threads < 1)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid value of `threads` parameter");
+
+        if (num_threads > 1 && decompress)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Parallel mode is only implemented for compression (not for decompression)");
+
         if (!codecs.empty() && options.count("level"))
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong options, --level is not compatible with --codec list");
 
@@ -145,7 +163,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
         else
             codec = CompressionCodecFactory::instance().get(method_family, level);
 
-
         std::unique_ptr<ReadBufferFromFileBase> rb;
         std::unique_ptr<WriteBufferFromFileBase> wb;
 
@@ -186,9 +203,20 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
         else
         {
             /// Compression
-            CompressedWriteBuffer to(*wb, codec, block_size);
-            copyData(*rb, to);
-            to.finalize();
+
+            if (num_threads == 1)
+            {
+                CompressedWriteBuffer to(*wb, codec, block_size);
+                copyData(*rb, to);
+                to.finalize();
+            }
+            else
+            {
+                ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads);
+                ParallelCompressedWriteBuffer to(*wb, codec, block_size, num_threads, pool);
+                copyData(*rb, to);
+                to.finalize();
+            }
         }
     }
     catch (...)

programs/disks/DisksApp.cpp

@@ -546,16 +546,18 @@ int mainEntryClickHouseDisks(int argc, char ** argv)
     catch (const DB::Exception & e)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
-        return 0;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
     catch (const boost::program_options::error & e)
     {
         std::cerr << "Bad arguments: " << e.what() << std::endl;
-        return 0;
+        return DB::ErrorCodes::BAD_ARGUMENTS;
     }
     catch (...)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
-        return 0;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }

programs/keeper-client/KeeperClient.cpp

@@ -448,7 +448,8 @@ int mainEntryClickHouseKeeperClient(int argc, char ** argv)
     catch (const DB::Exception & e)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
-        return 1;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
     catch (const boost::program_options::error & e)
     {
@@ -458,6 +459,7 @@ int mainEntryClickHouseKeeperClient(int argc, char ** argv)
     catch (...)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
-        return 1;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }

programs/keeper/Keeper.cpp

@@ -81,7 +81,7 @@ int mainEntryClickHouseKeeper(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }
 
@@ -672,7 +672,7 @@ catch (...)
     /// Poco does not provide stacktrace.
     tryLogCurrentException("Application");
     auto code = getCurrentExceptionCode();
-    return code ? code : -1;
+    return static_cast<UInt8>(code) ? code : -1;
 }
 
 

programs/library-bridge/LibraryBridge.cpp

@@ -13,7 +13,7 @@ int mainEntryClickHouseLibraryBridge(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }
 

programs/local/LocalServer.cpp

@@ -22,7 +22,6 @@
 #include <Interpreters/ProcessList.h>
 #include <Interpreters/loadMetadata.h>
 #include <Interpreters/registerInterpreters.h>
-#include <base/getFQDNOrHostName.h>
 #include <Access/AccessControl.h>
 #include <Common/PoolId.h>
 #include <Common/Exception.h>
@@ -31,7 +30,6 @@
 #include <Common/ThreadStatus.h>
 #include <Common/TLDListsHolder.h>
 #include <Common/quoteString.h>
-#include <Common/randomSeed.h>
 #include <Common/ThreadPool.h>
 #include <Common/CurrentMetrics.h>
 #include <Loggers/OwnFormattingChannel.h>
@@ -50,7 +48,6 @@
 #include <Dictionaries/registerDictionaries.h>
 #include <Disks/registerDisks.h>
 #include <Formats/registerFormats.h>
-#include <boost/algorithm/string/replace.hpp>
 #include <boost/program_options/options_description.hpp>
 #include <base/argsToConfig.h>
 #include <filesystem>
@@ -71,9 +68,11 @@ namespace CurrentMetrics
 
 namespace DB
 {
+
 namespace Setting
 {
     extern const SettingsBool allow_introspection_functions;
+    extern const SettingsBool implicit_select;
     extern const SettingsLocalFSReadMethod storage_file_read_method;
 }
 
@@ -126,6 +125,7 @@ void applySettingsOverridesForLocal(ContextMutablePtr context)
 
     settings[Setting::allow_introspection_functions] = true;
     settings[Setting::storage_file_read_method] = LocalFSReadMethod::mmap;
+    settings[Setting::implicit_select] = true;
 
     context->setSettings(settings);
 }
@@ -615,12 +615,14 @@ catch (const DB::Exception & e)
 {
     bool need_print_stack_trace = getClientConfiguration().getBool("stacktrace", false);
     std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl;
-    return e.code() ? e.code() : -1;
+    auto code = DB::getCurrentExceptionCode();
+    return static_cast<UInt8>(code) ? code : 1;
 }
 catch (...)
 {
-    std::cerr << getCurrentExceptionMessage(false) << std::endl;
-    return getCurrentExceptionCode();
+    std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
+    auto code = DB::getCurrentExceptionCode();
+    return static_cast<UInt8>(code) ? code : 1;
 }
 
 void LocalServer::updateLoggerLevel(const String & logs_level)
@@ -1029,7 +1031,7 @@ int mainEntryClickHouseLocal(int argc, char ** argv)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
     catch (const boost::program_options::error & e)
     {
@@ -1040,6 +1042,6 @@ int mainEntryClickHouseLocal(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }

programs/obfuscator/Obfuscator.cpp

@@ -1480,5 +1480,5 @@ catch (...)
 {
     std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
     auto code = DB::getCurrentExceptionCode();
-    return code ? code : 1;
+    return static_cast<UInt8>(code) ? code : 1;
 }

programs/odbc-bridge/ODBCBridge.cpp

@@ -13,7 +13,7 @@ int mainEntryClickHouseODBCBridge(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }
 

programs/server/Server.cpp

@@ -343,7 +343,7 @@ int mainEntryClickHouseServer(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }
 
@@ -2537,7 +2537,7 @@ catch (...)
     /// Poco does not provide stacktrace.
     tryLogCurrentException("Application");
     auto code = getCurrentExceptionCode();
-    return code ? code : -1;
+    return static_cast<UInt8>(code) ? code : -1;
 }
 
 std::unique_ptr<TCPProtocolStackFactory> Server::buildProtocolStackFromConfig(

programs/su/su.cpp

@@ -59,7 +59,13 @@ void setUserAndGroup(std::string arg_uid, std::string arg_gid)
                 throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid);
 
             if (!result)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid);
+            {
+                if (0 != getgrgid_r(gid, &entry, buf.get(), buf_size, &result))
+                    throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid);
+
+                if (!result)
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid);
+            }
 
             gid = entry.gr_gid;
         }
@@ -84,7 +90,13 @@ void setUserAndGroup(std::string arg_uid, std::string arg_gid)
                 throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid);
 
             if (!result)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid);
+            {
+                if (0 != getpwuid_r(uid, &entry, buf.get(), buf_size, &result))
+                    throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getpwuid_r' to obtain uid from user name ({})", uid);
+
+                if (!result)
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid);
+            }
 
             uid = entry.pw_uid;
         }

src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.cpp

@@ -387,7 +387,7 @@ template <typename Value, bool return_float, bool interpolated>
 using FuncQuantileExactWeighted = AggregateFunctionQuantile<
     Value,
     QuantileExactWeighted<Value, interpolated>,
-    NameQuantileExactWeighted,
+    std::conditional_t<interpolated, NameQuantileExactWeightedInterpolated, NameQuantileExactWeighted>,
     true,
     std::conditional_t<return_float, Float64, void>,
     false,
@@ -396,7 +396,7 @@ template <typename Value, bool return_float, bool interpolated>
 using FuncQuantilesExactWeighted = AggregateFunctionQuantile<
     Value,
     QuantileExactWeighted<Value, interpolated>,
-    NameQuantilesExactWeighted,
+    std::conditional_t<interpolated, NameQuantilesExactWeightedInterpolated, NameQuantilesExactWeighted>,
     true,
     std::conditional_t<return_float, Float64, void>,
     true,

src/Analyzer/QueryNode.h

@@ -602,9 +602,21 @@ public:
         return projection_columns;
     }
 
+    /// Returns true if query node is resolved, false otherwise
+    bool isResolved() const
+    {
+        return !projection_columns.empty();
+    }
+
     /// Resolve query node projection columns
     void resolveProjectionColumns(NamesAndTypes projection_columns_value);
 
+    /// Clear query node projection columns
+    void clearProjectionColumns()
+    {
+        projection_columns.clear();
+    }
+
     /// Remove unused projection columns
     void removeUnusedProjectionColumns(const std::unordered_set<std::string> & used_projection_columns);
 

src/Analyzer/QueryTreeBuilder.cpp

@@ -498,6 +498,8 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express
             sort_node->getFillTo() = buildExpression(order_by_element.getFillTo(), context);
         if (order_by_element.getFillStep())
             sort_node->getFillStep() = buildExpression(order_by_element.getFillStep(), context);
+        if (order_by_element.getFillStaleness())
+            sort_node->getFillStaleness() = buildExpression(order_by_element.getFillStaleness(), context);
 
         list_node->getNodes().push_back(std::move(sort_node));
     }

src/Analyzer/Resolve/QueryAnalyzer.cpp

@@ -51,7 +51,6 @@
 #include <Analyzer/ArrayJoinNode.h>
 #include <Analyzer/JoinNode.h>
 #include <Analyzer/UnionNode.h>
-#include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/QueryTreeBuilder.h>
 #include <Analyzer/IQueryTreeNode.h>
 #include <Analyzer/Identifier.h>
@@ -103,6 +102,8 @@ namespace Setting
     extern const SettingsBool single_join_prefer_left_table;
     extern const SettingsBool transform_null_in;
     extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions;
+    extern const SettingsBool allow_suspicious_types_in_group_by;
+    extern const SettingsBool allow_suspicious_types_in_order_by;
     extern const SettingsBool use_concurrency_control;
 }
 
@@ -437,8 +438,13 @@ ProjectionName QueryAnalyzer::calculateWindowProjectionName(const QueryTreeNodeP
     return buffer.str();
 }
 
-ProjectionName QueryAnalyzer::calculateSortColumnProjectionName(const QueryTreeNodePtr & sort_column_node, const ProjectionName & sort_expression_projection_name,
-    const ProjectionName & fill_from_expression_projection_name, const ProjectionName & fill_to_expression_projection_name, const ProjectionName & fill_step_expression_projection_name)
+ProjectionName QueryAnalyzer::calculateSortColumnProjectionName(
+    const QueryTreeNodePtr & sort_column_node,
+    const ProjectionName & sort_expression_projection_name,
+    const ProjectionName & fill_from_expression_projection_name,
+    const ProjectionName & fill_to_expression_projection_name,
+    const ProjectionName & fill_step_expression_projection_name,
+    const ProjectionName & fill_staleness_expression_projection_name)
 {
     auto & sort_node_typed = sort_column_node->as<SortNode &>();
 
@@ -468,6 +474,9 @@ ProjectionName QueryAnalyzer::calculateSortColumnProjectionName(const QueryTreeN
 
         if (sort_node_typed.hasFillStep())
             sort_column_projection_name_buffer << " STEP " << fill_step_expression_projection_name;
+
+        if (sort_node_typed.hasFillStaleness())
+            sort_column_projection_name_buffer << " STALENESS " << fill_staleness_expression_projection_name;
     }
 
     return sort_column_projection_name_buffer.str();
@@ -2958,27 +2967,29 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
             /// Replace storage with values storage of insertion block
             if (StoragePtr storage = scope.context->getViewSource())
             {
-                QueryTreeNodePtr table_expression;
-                /// Process possibly nested sub-selects
-                for (auto * query_node = in_second_argument->as<QueryNode>(); query_node; query_node = table_expression->as<QueryNode>())
-                    table_expression = extractLeftTableExpression(query_node->getJoinTree());
+                QueryTreeNodePtr table_expression = in_second_argument;
 
-                if (table_expression)
+                /// Process possibly nested sub-selects
+                while (table_expression)
                 {
-                    if (auto * query_table_node = table_expression->as<TableNode>())
-                    {
-                        if (query_table_node->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted())
-                        {
-                            auto replacement_table_expression = std::make_shared<TableNode>(storage, scope.context);
-                            if (std::optional<TableExpressionModifiers> table_expression_modifiers = query_table_node->getTableExpressionModifiers())
-                                replacement_table_expression->setTableExpressionModifiers(*table_expression_modifiers);
-                            in_second_argument = in_second_argument->cloneAndReplace(table_expression, std::move(replacement_table_expression));
-                        }
-                    }
+                    if (auto * query_node = table_expression->as<QueryNode>())
+                        table_expression = extractLeftTableExpression(query_node->getJoinTree());
+                    else if (auto * union_node = table_expression->as<UnionNode>())
+                        table_expression = union_node->getQueries().getNodes().at(0);
+                    else
+                        break;
+                }
+
+                TableNode * table_expression_table_node = table_expression ? table_expression->as<TableNode>() : nullptr;
+
+                if (table_expression_table_node &&
+                    table_expression_table_node->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted())
+                {
+                    auto replacement_table_expression_table_node = table_expression_table_node->clone();
+                    replacement_table_expression_table_node->as<TableNode &>().updateStorage(storage, scope.context);
+                    in_second_argument = in_second_argument->cloneAndReplace(table_expression, std::move(replacement_table_expression_table_node));
                 }
             }
-
-            resolveExpressionNode(in_second_argument, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/);
         }
 
         /// Edge case when the first argument of IN is scalar subquery.
@@ -3011,9 +3022,10 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
         argument_column.name = arguments_projection_names[function_argument_index];
 
         /** If function argument is lambda, save lambda argument index and initialize argument type as DataTypeFunction
-          * where function argument types are initialized with empty array of lambda arguments size.
+          * where function argument types are initialized with empty arrays of lambda arguments size.
           */
-        if (const auto * lambda_node = function_argument->as<const LambdaNode>())
+        const auto * lambda_node = function_argument->as<const LambdaNode>();
+        if (lambda_node)
         {
             size_t lambda_arguments_size = lambda_node->getArguments().getNodes().size();
             argument_column.type = std::make_shared<DataTypeFunction>(DataTypes(lambda_arguments_size, nullptr), nullptr);
@@ -3485,15 +3497,11 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
         else
             function_base = function->build(argument_columns);
 
-        /// Do not constant fold get scalar functions
-        // bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" ||
-        //     function_name == "shardCount" || function_name == "hostName" || function_name == "tcpPort";
-
         /** If function is suitable for constant folding try to convert it to constant.
           * Example: SELECT plus(1, 1);
           * Result: SELECT 2;
           */
-        if (function_base->isSuitableForConstantFolding()) // && !disable_constant_folding)
+        if (function_base->isSuitableForConstantFolding())
         {
             auto result_type = function_base->getResultType();
             auto executable_function = function_base->prepare(argument_columns);
@@ -3502,7 +3510,9 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
 
             if (all_arguments_constants)
             {
-                size_t num_rows = function_arguments.empty() ? 0 : argument_columns.front().column->size();
+                size_t num_rows = 0;
+                if (!argument_columns.empty())
+                    num_rows = argument_columns.front().column->size();
                 column = executable_function->execute(argument_columns, result_type, num_rows, true);
             }
             else
@@ -3998,6 +4008,7 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_
     ProjectionNames fill_from_expression_projection_names;
     ProjectionNames fill_to_expression_projection_names;
     ProjectionNames fill_step_expression_projection_names;
+    ProjectionNames fill_staleness_expression_projection_names;
 
     auto & sort_node_list_typed = sort_node_list->as<ListNode &>();
     for (auto & node : sort_node_list_typed.getNodes())
@@ -4019,6 +4030,8 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_
             sort_node.getExpression() = sort_column_list_node->getNodes().front();
         }
 
+        validateSortingKeyType(sort_node.getExpression()->getResultType(), scope);
+
         size_t sort_expression_projection_names_size = sort_expression_projection_names.size();
         if (sort_expression_projection_names_size != 1)
             throw Exception(ErrorCodes::LOGICAL_ERROR,
@@ -4088,11 +4101,38 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_
                     fill_step_expression_projection_names_size);
         }
 
+        if (sort_node.hasFillStaleness())
+        {
+            fill_staleness_expression_projection_names = resolveExpressionNode(sort_node.getFillStaleness(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
+
+            const auto * constant_node = sort_node.getFillStaleness()->as<ConstantNode>();
+            if (!constant_node)
+                throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
+                    "Sort FILL STALENESS expression must be constant with numeric or interval type. Actual {}. In scope {}",
+                    sort_node.getFillStaleness()->formatASTForErrorMessage(),
+                    scope.scope_node->formatASTForErrorMessage());
+
+            bool is_number = isColumnedAsNumber(constant_node->getResultType());
+            bool is_interval = WhichDataType(constant_node->getResultType()).isInterval();
+            if (!is_number && !is_interval)
+                throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
+                    "Sort FILL STALENESS expression must be constant with numeric or interval type. Actual {}. In scope {}",
+                    sort_node.getFillStaleness()->formatASTForErrorMessage(),
+                    scope.scope_node->formatASTForErrorMessage());
+
+            size_t fill_staleness_expression_projection_names_size = fill_staleness_expression_projection_names.size();
+            if (fill_staleness_expression_projection_names_size != 1)
+                throw Exception(ErrorCodes::LOGICAL_ERROR,
+                    "Sort FILL STALENESS expression expected 1 projection name. Actual {}",
+                    fill_staleness_expression_projection_names_size);
+        }
+
         auto sort_column_projection_name = calculateSortColumnProjectionName(node,
             sort_expression_projection_names[0],
             fill_from_expression_projection_names.empty() ? "" : fill_from_expression_projection_names.front(),
             fill_to_expression_projection_names.empty() ? "" : fill_to_expression_projection_names.front(),
-            fill_step_expression_projection_names.empty() ? "" : fill_step_expression_projection_names.front());
+            fill_step_expression_projection_names.empty() ? "" : fill_step_expression_projection_names.front(),
+            fill_staleness_expression_projection_names.empty() ? "" : fill_staleness_expression_projection_names.front());
 
         result_projection_names.push_back(std::move(sort_column_projection_name));
 
@@ -4100,11 +4140,32 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_
         fill_from_expression_projection_names.clear();
         fill_to_expression_projection_names.clear();
         fill_step_expression_projection_names.clear();
+        fill_staleness_expression_projection_names.clear();
     }
 
     return result_projection_names;
 }
 
+void QueryAnalyzer::validateSortingKeyType(const DataTypePtr & sorting_key_type, const IdentifierResolveScope & scope) const
+{
+    if (scope.context->getSettingsRef()[Setting::allow_suspicious_types_in_order_by])
+        return;
+
+    auto check = [](const IDataType & type)
+    {
+        if (isDynamic(type) || isVariant(type))
+            throw Exception(
+                ErrorCodes::ILLEGAL_COLUMN,
+                "Data types Variant/Dynamic are not allowed in ORDER BY keys, because it can lead to unexpected results. "
+                "Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if "
+                "its a JSON path subcolumn) or casting this column to a specific data type. "
+                "Set setting allow_suspicious_types_in_order_by = 1 in order to allow it");
+    };
+
+    check(*sorting_key_type);
+    sorting_key_type->forEachChild(check);
+}
+
 namespace
 {
 
@@ -4144,11 +4205,12 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR
             expandTuplesInList(group_by_list);
         }
 
-        if (scope.group_by_use_nulls)
+        for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes())
         {
-            for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes())
+            for (const auto & group_by_elem : grouping_set->as<ListNode>()->getNodes())
             {
-                for (const auto & group_by_elem : grouping_set->as<ListNode>()->getNodes())
+                validateGroupByKeyType(group_by_elem->getResultType(), scope);
+                if (scope.group_by_use_nulls)
                     scope.nullable_group_by_keys.insert(group_by_elem);
             }
         }
@@ -4164,14 +4226,37 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR
         auto & group_by_list = query_node_typed.getGroupBy().getNodes();
         expandTuplesInList(group_by_list);
 
-        if (scope.group_by_use_nulls)
+        for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes())
         {
-            for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes())
+            validateGroupByKeyType(group_by_elem->getResultType(), scope);
+            if (scope.group_by_use_nulls)
                 scope.nullable_group_by_keys.insert(group_by_elem);
         }
     }
 }
 
+/** Validate data types of GROUP BY key.
+  */
+void QueryAnalyzer::validateGroupByKeyType(const DataTypePtr & group_by_key_type, const IdentifierResolveScope & scope) const
+{
+    if (scope.context->getSettingsRef()[Setting::allow_suspicious_types_in_group_by])
+        return;
+
+    auto check = [](const IDataType & type)
+    {
+        if (isDynamic(type) || isVariant(type))
+            throw Exception(
+                ErrorCodes::ILLEGAL_COLUMN,
+                "Data types Variant/Dynamic are not allowed in GROUP BY keys, because it can lead to unexpected results. "
+                "Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if "
+                "its a JSON path subcolumn) or casting this column to a specific data type. "
+                "Set setting allow_suspicious_types_in_group_by = 1 in order to allow it");
+    };
+
+    check(*group_by_key_type);
+    group_by_key_type->forEachChild(check);
+}
+
 /** Resolve interpolate columns nodes list.
   */
 void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpolate_node_list, IdentifierResolveScope & scope)
@@ -5310,6 +5395,16 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
 
     auto & query_node_typed = query_node->as<QueryNode &>();
 
+    /** It is unsafe to call resolveQuery on already resolved query node, because during identifier resolution process
+      * we replace identifiers with expressions without aliases, also at the end of resolveQuery all aliases from all nodes will be removed.
+      * For subsequent resolveQuery executions it is possible to have wrong projection header, because for nodes
+      * with aliases projection name is alias.
+      *
+      * If for client it is necessary to resolve query node after clone, client must clear projection columns from query node before resolve.
+      */
+    if (query_node_typed.isResolved())
+        return;
+
     if (query_node_typed.isCTE())
         ctes_in_resolve_process.insert(query_node_typed.getCTEName());
 
@@ -5448,16 +5543,13 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
       */
     scope.use_identifier_lookup_to_result_cache = false;
 
-    if (query_node_typed.getJoinTree())
-    {
-        TableExpressionsAliasVisitor table_expressions_visitor(scope);
-        table_expressions_visitor.visit(query_node_typed.getJoinTree());
+    TableExpressionsAliasVisitor table_expressions_visitor(scope);
+    table_expressions_visitor.visit(query_node_typed.getJoinTree());
 
-        initializeQueryJoinTreeNode(query_node_typed.getJoinTree(), scope);
-        scope.aliases.alias_name_to_table_expression_node.clear();
+    initializeQueryJoinTreeNode(query_node_typed.getJoinTree(), scope);
+    scope.aliases.alias_name_to_table_expression_node.clear();
 
-        resolveQueryJoinTreeNode(query_node_typed.getJoinTree(), scope, visitor);
-    }
+    resolveQueryJoinTreeNode(query_node_typed.getJoinTree(), scope, visitor);
 
     if (!scope.group_by_use_nulls)
         scope.use_identifier_lookup_to_result_cache = true;
@@ -5675,6 +5767,9 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier
 {
     auto & union_node_typed = union_node->as<UnionNode &>();
 
+    if (union_node_typed.isResolved())
+        return;
+
     if (union_node_typed.isCTE())
         ctes_in_resolve_process.insert(union_node_typed.getCTEName());
 

src/Analyzer/Resolve/QueryAnalyzer.h

@@ -140,7 +140,8 @@ private:
         const ProjectionName & sort_expression_projection_name,
         const ProjectionName & fill_from_expression_projection_name,
         const ProjectionName & fill_to_expression_projection_name,
-        const ProjectionName & fill_step_expression_projection_name);
+        const ProjectionName & fill_step_expression_projection_name,
+        const ProjectionName & fill_staleness_expression_projection_name);
 
     QueryTreeNodePtr tryGetLambdaFromSQLUserDefinedFunctions(const std::string & function_name, ContextPtr context);
 
@@ -219,8 +220,12 @@ private:
 
     ProjectionNames resolveSortNodeList(QueryTreeNodePtr & sort_node_list, IdentifierResolveScope & scope);
 
+    void validateSortingKeyType(const DataTypePtr & sorting_key_type, const IdentifierResolveScope & scope) const;
+
     void resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope);
 
+    void validateGroupByKeyType(const DataTypePtr & group_by_key_type, const IdentifierResolveScope & scope) const;
+
     void resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpolate_node_list, IdentifierResolveScope & scope);
 
     void resolveWindowNodeList(QueryTreeNodePtr & window_node_list, IdentifierResolveScope & scope);

src/Analyzer/SortNode.cpp

@@ -69,6 +69,12 @@ void SortNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, si
         buffer << '\n' << std::string(indent + 2, ' ') << "FILL STEP\n";
         getFillStep()->dumpTreeImpl(buffer, format_state, indent + 4);
     }
+
+    if (hasFillStaleness())
+    {
+        buffer << '\n' << std::string(indent + 2, ' ') << "FILL STALENESS\n";
+        getFillStaleness()->dumpTreeImpl(buffer, format_state, indent + 4);
+    }
 }
 
 bool SortNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const
@@ -132,6 +138,8 @@ ASTPtr SortNode::toASTImpl(const ConvertToASTOptions & options) const
         result->setFillTo(getFillTo()->toAST(options));
     if (hasFillStep())
         result->setFillStep(getFillStep()->toAST(options));
+    if (hasFillStaleness())
+        result->setFillStaleness(getFillStaleness()->toAST(options));
 
     return result;
 }

src/Analyzer/SortNode.h

@@ -105,6 +105,24 @@ public:
         return children[fill_step_child_index];
     }
 
+    /// Returns true if sort node has fill staleness, false otherwise
+    bool hasFillStaleness() const
+    {
+        return children[fill_staleness_child_index] != nullptr;
+    }
+
+    /// Get fill staleness
+    const QueryTreeNodePtr & getFillStaleness() const
+    {
+        return children[fill_staleness_child_index];
+    }
+
+    /// Get fill staleness
+    QueryTreeNodePtr & getFillStaleness()
+    {
+        return children[fill_staleness_child_index];
+    }
+
     /// Get collator
     const std::shared_ptr<Collator> & getCollator() const
     {
@@ -144,7 +162,8 @@ private:
     static constexpr size_t fill_from_child_index = 1;
     static constexpr size_t fill_to_child_index = 2;
     static constexpr size_t fill_step_child_index = 3;
-    static constexpr size_t children_size = fill_step_child_index + 1;
+    static constexpr size_t fill_staleness_child_index = 4;
+    static constexpr size_t children_size = fill_staleness_child_index + 1;
 
     SortDirection sort_direction = SortDirection::ASCENDING;
     std::optional<SortDirection> nulls_sort_direction;

src/Analyzer/UnionNode.cpp

@@ -35,6 +35,7 @@ namespace ErrorCodes
 {
     extern const int TYPE_MISMATCH;
     extern const int BAD_ARGUMENTS;
+    extern const int LOGICAL_ERROR;
 }
 
 UnionNode::UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_)
@@ -50,6 +51,26 @@ UnionNode::UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_)
     children[queries_child_index] = std::make_shared<ListNode>();
 }
 
+bool UnionNode::isResolved() const
+{
+    for (const auto & query_node : getQueries().getNodes())
+    {
+        bool is_resolved = false;
+
+        if (auto * query_node_typed = query_node->as<QueryNode>())
+            is_resolved = query_node_typed->isResolved();
+        else if (auto * union_node_typed = query_node->as<UnionNode>())
+            is_resolved = union_node_typed->isResolved();
+        else
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected query tree node type in UNION node");
+
+        if (!is_resolved)
+            return false;
+    }
+
+    return true;
+}
+
 NamesAndTypes UnionNode::computeProjectionColumns() const
 {
     if (recursive_cte_table)

src/Analyzer/UnionNode.h

@@ -163,6 +163,9 @@ public:
         return children[queries_child_index];
     }
 
+    /// Returns true if union node is resolved, false otherwise
+    bool isResolved() const;
+
     /// Compute union node projection columns
     NamesAndTypes computeProjectionColumns() const;
 

src/Client/ClientBase.cpp

@@ -1650,6 +1650,11 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
     if (!parsed_insert_query)
         return;
 
+    /// If it's clickhouse-local, and the input data reading is already baked into the query pipeline,
+    /// don't read the data again here. This happens in some cases (e.g. input() table function) but not others (e.g. INFILE).
+    if (!connection->isSendDataNeeded())
+        return;
+
     bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && isStdinNotEmptyAndValid(std_in);
 
     if (need_render_progress)
@@ -2674,7 +2679,10 @@ void ClientBase::runInteractive()
 #if USE_REPLXX
     replxx::Replxx::highlighter_callback_t highlight_callback{};
     if (getClientConfiguration().getBool("highlight", true))
-        highlight_callback = highlight;
+        highlight_callback = [this](const String & query, std::vector<replxx::Replxx::Color> & colors)
+        {
+            highlight(query, colors, *client_context);
+        };
 
     ReplxxLineReader lr(
         *suggest,

src/Client/ClientBaseHelpers.cpp

@@ -5,6 +5,8 @@
 #include <Parsers/ParserQuery.h>
 #include <Parsers/parseQuery.h>
 #include <Common/UTF8Helpers.h>
+#include <Core/Settings.h>
+#include <Interpreters/Context.h>
 
 #include <iostream>
 
@@ -12,6 +14,11 @@
 namespace DB
 {
 
+namespace Setting
+{
+    extern const SettingsBool implicit_select;
+}
+
 /// Should we celebrate a bit?
 bool isNewYearMode()
 {
@@ -95,7 +102,7 @@ bool isChineseNewYearMode(const String & local_tz)
 }
 
 #if USE_REPLXX
-void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors)
+void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors, const Context & context)
 {
     using namespace replxx;
 
@@ -135,13 +142,27 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
 
     /// Currently we highlight only the first query in the multi-query mode.
 
-    ParserQuery parser(end);
+    ParserQuery parser(end, false, context.getSettingsRef()[Setting::implicit_select]);
     ASTPtr ast;
     bool parse_res = false;
 
     try
     {
-        parse_res = parser.parse(token_iterator, ast, expected);
+        while (!token_iterator->isEnd())
+        {
+            parse_res = parser.parse(token_iterator, ast, expected);
+            if (!parse_res)
+                break;
+
+            if (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon)
+            {
+                parse_res = false;
+                break;
+            }
+
+            while (token_iterator->type == TokenType::Semicolon)
+                ++token_iterator;
+        }
     }
     catch (...)
     {
@@ -175,7 +196,7 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
 
     /// Highlight the last error in red. If the parser failed or the lexer found an invalid token,
     /// or if it didn't parse all the data (except, the data for INSERT query, which is legitimately unparsed)
-    if ((!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon))
+    if ((!parse_res || last_token.isError())
         && !(insert_data && expected.max_parsed_pos >= insert_data)
         && expected.max_parsed_pos >= prev)
     {

src/Client/ClientBaseHelpers.h

@@ -11,13 +11,15 @@
 namespace DB
 {
 
+class Context;
+
 /// Should we celebrate a bit?
 bool isNewYearMode();
 
 bool isChineseNewYearMode(const String & local_tz);
 
 #if USE_REPLXX
-void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors);
+void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors, const Context & context);
 #endif
 
 }

src/Client/IServerConnection.h

@@ -109,6 +109,10 @@ public:
     /// Send block of data; if name is specified, server will write it to external (temporary) table of that name.
     virtual void sendData(const Block & block, const String & name, bool scalar) = 0;
 
+    /// Whether the client needs to read and send the data for the INSERT.
+    /// False if the server will read the data through other means (in particular if clickhouse-local added input reading step directly into the query pipeline).
+    virtual bool isSendDataNeeded() const { return true; }
+
     /// Send all contents of external (temporary) tables.
     virtual void sendExternalTablesData(ExternalTablesData & data) = 0;
 

src/Client/LocalConnection.cpp

@@ -328,6 +328,11 @@ void LocalConnection::sendData(const Block & block, const String &, bool)
         sendProfileEvents();
 }
 
+bool LocalConnection::isSendDataNeeded() const
+{
+    return !state || state->input_pipeline == nullptr;
+}
+
 void LocalConnection::sendCancel()
 {
     state->is_cancelled = true;

src/Client/LocalConnection.h

@@ -120,6 +120,8 @@ public:
 
     void sendData(const Block & block, const String & name/* = "" */, bool scalar/* = false */) override;
 
+    bool isSendDataNeeded() const override;
+
     void sendExternalTablesData(ExternalTablesData &) override;
 
     void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) override;

src/Columns/ColumnArray.h

@@ -196,6 +196,13 @@ public:
     bool hasDynamicStructure() const override { return getData().hasDynamicStructure(); }
     void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override;
 
+    bool dynamicStructureEquals(const IColumn & rhs) const override
+    {
+        if (const auto * rhs_concrete = typeid_cast<const ColumnArray *>(&rhs))
+            return data->dynamicStructureEquals(*rhs_concrete->data);
+        return false;
+    }
+
 private:
     WrappedPtr data;
     WrappedPtr offsets;

src/Columns/ColumnDynamic.cpp

@@ -1208,6 +1208,15 @@ void ColumnDynamic::prepareVariantsForSquashing(const Columns & source_columns)
     }
 }
 
+bool ColumnDynamic::dynamicStructureEquals(const IColumn & rhs) const
+{
+    if (const auto * rhs_concrete = typeid_cast<const ColumnDynamic *>(&rhs))
+        return max_dynamic_types == rhs_concrete->max_dynamic_types && global_max_dynamic_types == rhs_concrete->global_max_dynamic_types
+            && variant_info.variant_name == rhs_concrete->variant_info.variant_name
+            && variant_column->dynamicStructureEquals(*rhs_concrete->variant_column);
+    return false;
+}
+
 void ColumnDynamic::takeDynamicStructureFromSourceColumns(const Columns & source_columns)
 {
     if (!empty())

src/Columns/ColumnDynamic.h

@@ -376,6 +376,7 @@ public:
     bool addNewVariant(const DataTypePtr & new_variant) { return addNewVariant(new_variant, new_variant->getName()); }
 
     bool hasDynamicStructure() const override { return true; }
+    bool dynamicStructureEquals(const IColumn & rhs) const override;
     void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override;
 
     const StatisticsPtr & getStatistics() const { return statistics; }

src/Columns/ColumnFunction.cpp

@@ -72,6 +72,26 @@ ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
     return ColumnFunction::create(length, function, capture, is_short_circuit_argument, is_function_compiled);
 }
 
+Field ColumnFunction::operator[](size_t n) const
+{
+    Field res;
+    get(n, res);
+    return res;
+}
+
+void ColumnFunction::get(size_t n, Field & res) const
+{
+    const size_t tuple_size = captured_columns.size();
+
+    res = Tuple();
+    Tuple & res_tuple = res.safeGet<Tuple &>();
+    res_tuple.reserve(tuple_size);
+
+    for (size_t i = 0; i < tuple_size; ++i)
+        res_tuple.push_back((*captured_columns[i].column)[n]);
+}
+
+
 #if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnFunction::insertFrom(const IColumn & src, size_t n)
 #else

src/Columns/ColumnFunction.h

@@ -60,15 +60,9 @@ public:
     void appendArguments(const ColumnsWithTypeAndName & columns);
     ColumnWithTypeAndName reduce() const;
 
-    Field operator[](size_t) const override
-    {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName());
-    }
+    Field operator[](size_t n) const override;
 
-    void get(size_t, Field &) const override
-    {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName());
-    }
+    void get(size_t n, Field & res) const override;
 
     StringRef getDataAt(size_t) const override
     {

src/Columns/ColumnMap.cpp

@@ -345,6 +345,13 @@ bool ColumnMap::structureEquals(const IColumn & rhs) const
     return false;
 }
 
+bool ColumnMap::dynamicStructureEquals(const IColumn & rhs) const
+{
+    if (const auto * rhs_map = typeid_cast<const ColumnMap *>(&rhs))
+        return nested->dynamicStructureEquals(*rhs_map->nested);
+    return false;
+}
+
 ColumnPtr ColumnMap::compress() const
 {
     auto compressed = nested->compress();

src/Columns/ColumnMap.h

@@ -123,6 +123,7 @@ public:
     ColumnPtr compress() const override;
 
     bool hasDynamicStructure() const override { return nested->hasDynamicStructure(); }
+    bool dynamicStructureEquals(const IColumn & rhs) const override;
     void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override;
 };
 

src/Columns/ColumnObject.cpp

@@ -1415,6 +1415,31 @@ void ColumnObject::prepareForSquashing(const std::vector<ColumnPtr> & source_col
     }
 }
 
+bool ColumnObject::dynamicStructureEquals(const IColumn & rhs) const
+{
+    const auto * rhs_object = typeid_cast<const ColumnObject *>(&rhs);
+    if (!rhs_object || typed_paths.size() != rhs_object->typed_paths.size()
+        || global_max_dynamic_paths != rhs_object->global_max_dynamic_paths || max_dynamic_types != rhs_object->max_dynamic_types
+        || dynamic_paths.size() != rhs_object->dynamic_paths.size())
+        return false;
+
+    for (const auto & [path, column] : typed_paths)
+    {
+        auto it = rhs_object->typed_paths.find(path);
+        if (it == rhs_object->typed_paths.end() || !it->second->dynamicStructureEquals(*column))
+            return false;
+    }
+
+    for (const auto & [path, column] : dynamic_paths)
+    {
+        auto it = rhs_object->dynamic_paths.find(path);
+        if (it == rhs_object->dynamic_paths.end() || !it->second->dynamicStructureEquals(*column))
+            return false;
+    }
+
+    return true;
+}
+
 void ColumnObject::takeDynamicStructureFromSourceColumns(const DB::Columns & source_columns)
 {
     if (!empty())

src/Columns/ColumnObject.h

@@ -177,6 +177,7 @@ public:
     bool isFinalized() const override;
 
     bool hasDynamicStructure() const override { return true; }
+    bool dynamicStructureEquals(const IColumn & rhs) const override;
     void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override;
 
     const PathToColumnMap & getTypedPaths() const { return typed_paths; }
@@ -227,6 +228,7 @@ public:
     void setDynamicPaths(const std::vector<String> & paths);
     void setDynamicPaths(const std::vector<std::pair<String, ColumnPtr>> & paths);
     void setMaxDynamicPaths(size_t max_dynamic_paths_);
+    void setGlobalMaxDynamicPaths(size_t global_max_dynamic_paths_);
     void setStatistics(const StatisticsPtr & statistics_) { statistics = statistics_; }
 
     void serializePathAndValueIntoSharedData(ColumnString * shared_data_paths, ColumnString * shared_data_values, std::string_view path, const IColumn & column, size_t n);

src/Columns/ColumnTuple.cpp

@@ -757,6 +757,26 @@ bool ColumnTuple::hasDynamicStructure() const
     return false;
 }
 
+bool ColumnTuple::dynamicStructureEquals(const IColumn & rhs) const
+{
+    if (const auto * rhs_tuple = typeid_cast<const ColumnTuple *>(&rhs))
+    {
+        const size_t tuple_size = columns.size();
+        if (tuple_size != rhs_tuple->columns.size())
+            return false;
+
+        for (size_t i = 0; i < tuple_size; ++i)
+            if (!columns[i]->dynamicStructureEquals(*rhs_tuple->columns[i]))
+                return false;
+
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
 void ColumnTuple::takeDynamicStructureFromSourceColumns(const Columns & source_columns)
 {
     std::vector<Columns> nested_source_columns;

src/Columns/ColumnTuple.h

@@ -141,6 +141,7 @@ public:
     ColumnPtr & getColumnPtr(size_t idx) { return columns[idx]; }
 
     bool hasDynamicStructure() const override;
+    bool dynamicStructureEquals(const IColumn & rhs) const override;
     void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override;
 
     /// Empty tuple needs a public method to manage its size.

src/Columns/ColumnVariant.cpp

@@ -952,7 +952,7 @@ ColumnPtr ColumnVariant::permute(const Permutation & perm, size_t limit) const
     if (hasOnlyNulls())
     {
         if (limit)
-            return cloneResized(limit);
+            return cloneResized(limit ? std::min(size(), limit) : size());
 
         /// If no limit, we can just return current immutable column.
         return this->getPtr();
@@ -1409,6 +1409,23 @@ bool ColumnVariant::structureEquals(const IColumn & rhs) const
     return true;
 }
 
+bool ColumnVariant::dynamicStructureEquals(const IColumn & rhs) const
+{
+    const auto * rhs_variant = typeid_cast<const ColumnVariant *>(&rhs);
+    if (!rhs_variant)
+        return false;
+
+    const size_t num_variants = variants.size();
+    if (num_variants != rhs_variant->variants.size())
+        return false;
+
+    for (size_t i = 0; i < num_variants; ++i)
+        if (!variants[i]->dynamicStructureEquals(rhs_variant->getVariantByGlobalDiscriminator(globalDiscriminatorByLocal(i))))
+            return false;
+
+    return true;
+}
+
 ColumnPtr ColumnVariant::compress() const
 {
     ColumnPtr local_discriminators_compressed = local_discriminators->compress();

src/Columns/ColumnVariant.h

@@ -336,6 +336,7 @@ public:
     void extend(const std::vector<Discriminator> & old_to_new_global_discriminators, std::vector<std::pair<MutableColumnPtr, Discriminator>> && new_variants_and_discriminators);
 
     bool hasDynamicStructure() const override;
+    bool dynamicStructureEquals(const IColumn & rhs) const override;
     void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override;
 
 private:

src/Columns/IColumn.h

@@ -635,6 +635,9 @@ public:
 
     /// Checks if column has dynamic subcolumns.
     virtual bool hasDynamicStructure() const { return false; }
+
+    /// For columns with dynamic subcolumns checks if columns have equal dynamic structure.
+    [[nodiscard]] virtual bool dynamicStructureEquals(const IColumn & rhs) const { return structureEquals(rhs); }
     /// For columns with dynamic subcolumns this method takes dynamic structure from source columns
     /// and creates proper resulting dynamic structure in advance for merge of these source columns.
     virtual void takeDynamicStructureFromSourceColumns(const std::vector<Ptr> & /*source_columns*/) {}

src/Common/CurrentMetrics.cpp

@@ -41,6 +41,10 @@
     M(PostgreSQLConnection, "Number of client connections using PostgreSQL protocol") \
     M(OpenFileForRead, "Number of files open for reading") \
     M(OpenFileForWrite, "Number of files open for writing") \
+    M(Compressing, "Number of compress operations using internal compression codecs") \
+    M(Decompressing, "Number of decompress operations using internal compression codecs") \
+    M(ParallelCompressedWriteBufferThreads, "Number of threads in all instances of ParallelCompressedWriteBuffer - these threads are doing parallel compression and writing") \
+    M(ParallelCompressedWriteBufferWait, "Number of threads in all instances of ParallelCompressedWriteBuffer that are currently waiting for buffer to become available for writing") \
     M(TotalTemporaryFiles, "Number of temporary files created") \
     M(TemporaryFilesForSort, "Number of temporary files created for external sorting") \
     M(TemporaryFilesForAggregation, "Number of temporary files created for external aggregation") \
@@ -99,6 +103,9 @@
     M(IOThreads, "Number of threads in the IO thread pool.") \
     M(IOThreadsActive, "Number of threads in the IO thread pool running a task.") \
     M(IOThreadsScheduled, "Number of queued or active jobs in the IO thread pool.") \
+    M(CompressionThread, "Number of threads in compression thread pools.") \
+    M(CompressionThreadActive, "Number of threads in compression thread pools running a task.") \
+    M(CompressionThreadScheduled, "Number of queued or active jobs in compression thread pools.") \
     M(ThreadPoolRemoteFSReaderThreads, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool.") \
     M(ThreadPoolRemoteFSReaderThreadsActive, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool running a task.") \
     M(ThreadPoolRemoteFSReaderThreadsScheduled, "Number of queued or active jobs in the thread pool for remote_filesystem_read_method=threadpool.") \

src/Common/FieldVisitorScale.cpp

@@ -0,0 +1,30 @@
+#include <Common/FieldVisitorScale.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+FieldVisitorScale::FieldVisitorScale(Int32 rhs_) : rhs(rhs_) {}
+
+void FieldVisitorScale::operator() (Int64 & x) const { x *= rhs; }
+void FieldVisitorScale::operator() (UInt64 & x) const { x *= rhs; }
+void FieldVisitorScale::operator() (Float64 & x) const { x *= rhs; }
+void FieldVisitorScale::operator() (Null &) const { /*Do not scale anything*/ }
+
+void FieldVisitorScale::operator() (String &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Strings"); }
+void FieldVisitorScale::operator() (Array &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Arrays"); }
+void FieldVisitorScale::operator() (Tuple &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Tuples"); }
+void FieldVisitorScale::operator() (Map &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Maps"); }
+void FieldVisitorScale::operator() (Object &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Objects"); }
+void FieldVisitorScale::operator() (UUID &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale UUIDs"); }
+void FieldVisitorScale::operator() (IPv4 &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale IPv4s"); }
+void FieldVisitorScale::operator() (IPv6 &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale IPv6s"); }
+void FieldVisitorScale::operator() (CustomType & x) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale custom type {}", x.getTypeName()); }
+void FieldVisitorScale::operator() (AggregateFunctionStateData &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale AggregateFunctionStates"); }
+void FieldVisitorScale::operator() (bool &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Bools"); }
+
+}

src/Common/FieldVisitorScale.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <Common/FieldVisitors.h>
+#include <Common/FieldVisitorConvertToNumber.h>
+
+namespace DB
+{
+
+/** Implements `*=` operation by number
+ */
+class FieldVisitorScale : public StaticVisitor<void>
+{
+private:
+    Int32 rhs;
+
+public:
+    explicit FieldVisitorScale(Int32 rhs_);
+
+    void operator() (Int64 & x) const;
+    void operator() (UInt64 & x) const;
+    void operator() (Float64 & x) const;
+    void operator() (Null &) const;
+    [[noreturn]] void operator() (String &) const;
+    [[noreturn]] void operator() (Array &) const;
+    [[noreturn]] void operator() (Tuple &) const;
+    [[noreturn]] void operator() (Map &) const;
+    [[noreturn]] void operator() (Object &) const;
+    [[noreturn]] void operator() (UUID &) const;
+    [[noreturn]] void operator() (IPv4 &) const;
+    [[noreturn]] void operator() (IPv6 &) const;
+    [[noreturn]] void operator() (AggregateFunctionStateData &) const;
+    [[noreturn]] void operator() (CustomType &) const;
+    [[noreturn]] void operator() (bool &) const;
+
+    template <typename T>
+    void operator() (DecimalField<T> & x) const { x = DecimalField<T>(x.getValue() * T(rhs), x.getScale()); }
+
+    template <typename T>
+    requires is_big_int_v<T>
+    void operator() (T & x) const { x *= rhs; }
+};
+
+}

src/Common/HashTable/HashTable.h

@@ -658,16 +658,11 @@ protected:
     {
         if (!std::is_trivially_destructible_v<Cell>)
         {
-            for (iterator it = begin(), it_end = end(); it != it_end; ++it)
+            for (iterator it = begin(), it_end = end(); it != it_end;)
             {
-                it.ptr->~Cell();
-                /// In case of poison_in_dtor=1 it will be poisoned,
-                /// but it maybe used later, during iteration.
-                ///
-                /// NOTE, that technically this is UB [1], but OK for now.
-                ///
-                ///   [1]: https://github.com/google/sanitizers/issues/854#issuecomment-329661378
-                __msan_unpoison(it.ptr, sizeof(*it.ptr));
+                auto ptr = it.ptr;
+                ++it;
+                ptr->~Cell();
             }
 
             /// Everything had been destroyed in the loop above, reset the flag

src/Common/ThreadPool.h

@@ -122,7 +122,7 @@ public:
     void scheduleOrThrowOnError(Job job, Priority priority = {});
 
     /// Similar to scheduleOrThrowOnError(...). Wait for specified amount of time and schedule a job or return false.
-    bool trySchedule(Job job, Priority priority = {}, uint64_t wait_microseconds = 0) noexcept;
+    [[nodiscard]] bool trySchedule(Job job, Priority priority = {}, uint64_t wait_microseconds = 0) noexcept;
 
     /// Similar to scheduleOrThrowOnError(...). Wait for specified amount of time and schedule a job or throw an exception.
     void scheduleOrThrow(Job job, Priority priority = {}, uint64_t wait_microseconds = 0, bool propagate_opentelemetry_tracing_context = true);
@@ -142,7 +142,7 @@ public:
 
     /// Returns true if the pool already terminated
     /// (and any further scheduling will produce CANNOT_SCHEDULE_TASK exception)
-    bool finished() const;
+    [[nodiscard]] bool finished() const;
 
     void setMaxThreads(size_t value);
     void setMaxFreeThreads(size_t value);

src/Compression/CompressedWriteBuffer.cpp

@@ -2,7 +2,6 @@
 #include <cstring>
 
 #include <base/types.h>
-#include <base/unaligned.h>
 #include <base/defines.h>
 
 #include <IO/WriteHelpers.h>

src/Compression/ICompressionCodec.cpp

@@ -5,11 +5,18 @@
 #include <Parsers/ASTFunction.h>
 #include <base/unaligned.h>
 #include <Common/Exception.h>
+#include <Common/CurrentMetrics.h>
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Compression/CompressionCodecMultiple.h>
 
 
+namespace CurrentMetrics
+{
+    extern const Metric Compressing;
+    extern const Metric Decompressing;
+}
+
 namespace DB
 {
 
@@ -80,6 +87,8 @@ UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char
 {
     assert(source != nullptr && dest != nullptr);
 
+    CurrentMetrics::Increment metric_increment(CurrentMetrics::Compressing);
+
     dest[0] = getMethodByte();
     UInt8 header_size = getHeaderSize();
     /// Write data from header_size
@@ -93,6 +102,8 @@ UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, ch
 {
     assert(source != nullptr && dest != nullptr);
 
+    CurrentMetrics::Increment metric_increment(CurrentMetrics::Decompressing);
+
     UInt8 header_size = getHeaderSize();
     if (source_size < header_size)
         throw Exception(decompression_error_code,

src/Compression/ParallelCompressedWriteBuffer.cpp

@@ -0,0 +1,166 @@
+#include <city.h>
+
+#include <base/types.h>
+#include <base/defines.h>
+
+#include <IO/WriteHelpers.h>
+#include <Common/setThreadName.h>
+#include <Common/scope_guard_safe.h>
+#include <Common/CurrentThread.h>
+#include <Common/CurrentMetrics.h>
+
+#include <Compression/ParallelCompressedWriteBuffer.h>
+
+
+namespace CurrentMetrics
+{
+    extern const Metric ParallelCompressedWriteBufferThreads;
+    extern const Metric ParallelCompressedWriteBufferWait;
+}
+
+namespace DB
+{
+
+ParallelCompressedWriteBuffer::ParallelCompressedWriteBuffer(
+    WriteBuffer & out_,
+    CompressionCodecPtr codec_,
+    size_t buf_size_,
+    size_t num_threads_,
+    ThreadPool & pool_)
+    : WriteBuffer(nullptr, 0), out(out_), codec(codec_), buf_size(buf_size_), num_threads(num_threads_), pool(pool_)
+{
+    buffers.emplace_back(buf_size);
+    current_buffer = buffers.begin();
+    BufferBase::set(current_buffer->uncompressed.data(), buf_size, 0);
+}
+
+void ParallelCompressedWriteBuffer::nextImpl()
+{
+    if (!offset())
+        return;
+
+    std::unique_lock lock(mutex);
+
+    /// The buffer will be compressed and processed in the thread.
+    current_buffer->busy = true;
+    current_buffer->sequence_num = current_sequence_num;
+    ++current_sequence_num;
+    current_buffer->uncompressed_size = offset();
+    pool.scheduleOrThrowOnError([this, my_current_buffer = current_buffer, thread_group = CurrentThread::getGroup()]
+    {
+        SCOPE_EXIT_SAFE(
+            if (thread_group)
+                CurrentThread::detachFromGroupIfNotDetached();
+        );
+
+        if (thread_group)
+            CurrentThread::attachToGroupIfDetached(thread_group);
+        setThreadName("ParallelCompres");
+
+        compress(my_current_buffer);
+    });
+
+    BufferPair * previous_buffer = &*current_buffer;
+    ++current_buffer;
+    if (current_buffer == buffers.end())
+    {
+        if (buffers.size() < num_threads)
+        {
+            /// If we didn't use all num_threads buffers yet, create a new one.
+            current_buffer = buffers.emplace(current_buffer, buf_size);
+        }
+        else
+        {
+            /// Otherwise, wrap around to the first buffer in the list.
+            current_buffer = buffers.begin();
+        }
+    }
+
+    /// Wait while the buffer becomes not busy
+    if (current_buffer->busy)
+    {
+        CurrentMetrics::Increment metric_increment(CurrentMetrics::ParallelCompressedWriteBufferWait);
+        cond.wait(lock, [&]{ return !current_buffer->busy; });
+    }
+
+    /// Now this buffer can be used.
+    current_buffer->previous = previous_buffer;
+    BufferBase::set(current_buffer->uncompressed.data(), buf_size, 0);
+}
+
+void ParallelCompressedWriteBuffer::finalizeImpl()
+{
+    next();
+    pool.wait();
+}
+
+void ParallelCompressedWriteBuffer::compress(Iterator buffer)
+{
+    CurrentMetrics::Increment metric_increment(CurrentMetrics::ParallelCompressedWriteBufferThreads);
+
+    chassert(buffer->uncompressed_size <= INT_MAX);
+    UInt32 uncompressed_size = static_cast<UInt32>(buffer->uncompressed_size);
+    UInt32 compressed_reserve_size = codec->getCompressedReserveSize(uncompressed_size);
+
+    /// If all previous buffers have been written,
+    /// and if the output buffer has the required capacity,
+    /// we can compress data directly into the output buffer.
+    size_t required_out_capacity = compressed_reserve_size + sizeof(CityHash_v1_0_2::uint128);
+    bool can_write_directly = false;
+
+    if (!buffer->previous)
+    {
+        can_write_directly = out.available() >= required_out_capacity;
+    }
+    else
+    {
+        std::unique_lock lock(mutex);
+        can_write_directly = (!buffer->previous->busy || buffer->previous->sequence_num > buffer->sequence_num)
+            && out.available() >= required_out_capacity;
+    }
+
+    if (can_write_directly)
+    {
+        char * out_compressed_ptr = out.position() + sizeof(CityHash_v1_0_2::uint128);
+        UInt32 compressed_size = codec->compress(buffer->uncompressed.data(), uncompressed_size, out_compressed_ptr);
+
+        CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(out_compressed_ptr, compressed_size);
+
+        writeBinaryLittleEndian(checksum.low64, out);
+        writeBinaryLittleEndian(checksum.high64, out);
+
+        out.position() += compressed_size;
+    }
+    else
+    {
+        buffer->compressed.resize(compressed_reserve_size);
+        UInt32 compressed_size = codec->compress(buffer->uncompressed.data(), uncompressed_size, buffer->compressed.data());
+
+        CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(buffer->compressed.data(), compressed_size);
+
+        /// Wait while all previous buffers have been written.
+        if (buffer->previous)
+        {
+            CurrentMetrics::Increment metric_wait_increment(CurrentMetrics::ParallelCompressedWriteBufferWait);
+            std::unique_lock lock(mutex);
+            cond.wait(lock, [&]{ return !buffer->previous->busy || buffer->previous->sequence_num > buffer->sequence_num; });
+        }
+
+        writeBinaryLittleEndian(checksum.low64, out);
+        writeBinaryLittleEndian(checksum.high64, out);
+
+        out.write(buffer->compressed.data(), compressed_size);
+    }
+
+    std::unique_lock lock(mutex);
+    buffer->busy = false;
+    cond.notify_all();
+}
+
+ParallelCompressedWriteBuffer::~ParallelCompressedWriteBuffer()
+{
+    if (!canceled)
+        finalize();
+}
+
+}

src/Compression/ParallelCompressedWriteBuffer.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <list>
+#include <memory>
+
+#include <Common/PODArray.h>
+
+#include <IO/WriteBuffer.h>
+#include <IO/BufferWithOwnMemory.h>
+#include <Compression/ICompressionCodec.h>
+#include <Compression/CompressionFactory.h>
+#include <Common/ThreadPool.h>
+
+
+namespace DB
+{
+
+/** Uses multi-buffering for parallel compression.
+  * When the buffer is filled, it will be compressed in the background,
+  * and a new buffer is created for the next input data.
+  */
+class ParallelCompressedWriteBuffer final : public WriteBuffer
+{
+public:
+    explicit ParallelCompressedWriteBuffer(
+        WriteBuffer & out_,
+        CompressionCodecPtr codec_,
+        size_t buf_size_,
+        size_t num_threads_,
+        ThreadPool & pool_);
+
+    ~ParallelCompressedWriteBuffer() override;
+
+private:
+    void nextImpl() override;
+    void finalizeImpl() override;
+
+    WriteBuffer & out;
+    CompressionCodecPtr codec;
+    size_t buf_size;
+    size_t num_threads;
+    ThreadPool & pool;
+
+    struct BufferPair
+    {
+        explicit BufferPair(size_t input_size)
+            : uncompressed(input_size)
+        {
+        }
+
+        Memory<> uncompressed;
+        size_t uncompressed_size = 0;
+        PODArray<char> compressed;
+        BufferPair * previous = nullptr;
+        size_t sequence_num = 0;
+        bool busy = false;
+    };
+
+    std::mutex mutex;
+    std::condition_variable cond;
+    std::list<BufferPair> buffers;
+
+    using Iterator = std::list<BufferPair>::iterator;
+    Iterator current_buffer;
+    size_t current_sequence_num = 0;
+
+    void compress(Iterator buffer);
+};
+
+}

src/Coordination/tests/gtest_coordination.cpp

@@ -330,7 +330,7 @@ TYPED_TEST(CoordinationTest, TestSummingRaft1)
     this->setLogDirectory("./logs");
     this->setStateFileDirectory(".");
 
-    SummingRaftServer s1(1, "localhost", 44444, this->keeper_context);
+    SummingRaftServer s1(1, "localhost", 0, this->keeper_context);
     SCOPE_EXIT(if (std::filesystem::exists("./state")) std::filesystem::remove("./state"););
 
     /// Single node is leader

src/Core/Settings.cpp

@@ -873,6 +873,12 @@ In CREATE TABLE statement allows specifying Variant type with similar variant ty
 )", 0) \
     DECLARE(Bool, allow_suspicious_primary_key, false, R"(
 Allow suspicious `PRIMARY KEY`/`ORDER BY` for MergeTree (i.e. SimpleAggregateFunction).
+)", 0) \
+    DECLARE(Bool, allow_suspicious_types_in_group_by, false, R"(
+Allows or restricts using [Variant](../../sql-reference/data-types/variant.md) and [Dynamic](../../sql-reference/data-types/dynamic.md) types in GROUP BY keys.
+)", 0) \
+    DECLARE(Bool, allow_suspicious_types_in_order_by, false, R"(
+Allows or restricts using [Variant](../../sql-reference/data-types/variant.md) and [Dynamic](../../sql-reference/data-types/dynamic.md) types in ORDER BY keys.
 )", 0) \
     DECLARE(Bool, compile_expressions, false, R"(
 Compile some scalar functions and operators to native code. Due to a bug in the LLVM compiler infrastructure, on AArch64 machines, it is known to lead to a nullptr dereference and, consequently, server crash. Do not enable this setting.
@@ -2863,7 +2869,7 @@ Limit on size of multipart/form-data content. This setting cannot be parsed from
     DECLARE(Bool, calculate_text_stack_trace, true, R"(
 Calculate text stack trace in case of exceptions during query execution. This is the default. It requires symbol lookups that may slow down fuzzing tests when a huge amount of wrong queries are executed. In normal cases, you should not disable this option.
 )", 0) \
-    DECLARE(Bool, enable_job_stack_trace, false, R"(
+    DECLARE(Bool, enable_job_stack_trace, true, R"(
 Output stack trace of a job creator when job results in exception
 )", 0) \
     DECLARE(Bool, allow_ddl, true, R"(
@@ -5705,6 +5711,8 @@ If enabled, MongoDB tables will return an error when a MongoDB query cannot be b
 )", 0) \
     DECLARE(Bool, implicit_select, false, R"(
 Allow writing simple SELECT queries without the leading SELECT keyword, which makes it simple for calculator-style usage, e.g. `1 + 2` becomes a valid query.
+
+In `clickhouse-local` it is enabled by default and can be explicitly disabled.
 )", 0) \
     \
     \
@@ -5862,7 +5870,7 @@ Experimental data deduplication for SELECT queries based on part UUIDs
 // Please add settings related to formats in Core/FormatFactorySettings.h, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS.
 
 #define OBSOLETE_SETTINGS(M, ALIAS) \
-    /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
+    /** Obsolete settings which are kept around for compatibility reasons. They have no effect anymore. */ \
     MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 0) \
     MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \
     MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \

src/Core/SettingsChangesHistory.cpp

@@ -64,7 +64,9 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
     },
     {"24.11",
         {
-            {"read_in_order_use_virtual_row", false, false, "Use virtual row while reading in order of primary key or its monotonic function fashion. It is useful when searching over multiple parts as only relevant ones are touched."},
+            {"enable_job_stack_trace", false, true, "Enable by default collecting stack traces from job's scheduling."},
+            {"allow_suspicious_types_in_group_by", true, false, "Don't allow Variant/Dynamic types in GROUP BY by default"},
+            {"allow_suspicious_types_in_order_by", true, false, "Don't allow Variant/Dynamic types in ORDER BY by default"},
             {"distributed_cache_discard_connection_if_unread_data", true, true, "New setting"},
             {"filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage", true, true, "New setting"},
             {"filesystem_cache_enable_background_download_during_fetch", true, true, "New setting"},
@@ -75,6 +77,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"backup_restore_keeper_max_retries_while_handling_error", 0, 20, "New setting."},
             {"backup_restore_finish_timeout_after_error_sec", 0, 180, "New setting."},
             {"parallel_replicas_local_plan", false, true, "Use local plan for local replica in a query with parallel replicas"},
+            {"read_in_order_use_virtual_row", false, false, "Use virtual row while reading in order of primary key or its monotonic function fashion. It is useful when searching over multiple parts as only relevant ones are touched."},
         }
     },
     {"24.10",

src/Core/SortCursor.h

@@ -35,6 +35,11 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+extern const int LOGICAL_ERROR;
+}
+
 /** Cursor allows to compare rows in different blocks (and parts).
   * Cursor moves inside single block.
   * It is used in priority queue.
@@ -83,21 +88,27 @@ struct SortCursorImpl
     SortCursorImpl(
         const Block & header,
         const Columns & columns,
+        size_t num_rows,
         const SortDescription & desc_,
         size_t order_ = 0,
         IColumn::Permutation * perm = nullptr)
         : desc(desc_), sort_columns_size(desc.size()), order(order_), need_collation(desc.size())
     {
-        reset(columns, header, perm);
+        reset(columns, header, num_rows, perm);
     }
 
     bool empty() const { return rows == 0; }
 
     /// Set the cursor to the beginning of the new block.
-    void reset(const Block & block, IColumn::Permutation * perm = nullptr) { reset(block.getColumns(), block, perm); }
+    void reset(const Block & block, IColumn::Permutation * perm = nullptr)
+    {
+        if (block.getColumns().empty())
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty column list in block");
+        reset(block.getColumns(), block, block.getColumns()[0]->size(), perm);
+    }
 
     /// Set the cursor to the beginning of the new block.
-    void reset(const Columns & columns, const Block & block, IColumn::Permutation * perm = nullptr)
+    void reset(const Columns & columns, const Block & block, UInt64 num_rows, IColumn::Permutation * perm = nullptr)
     {
         all_columns.clear();
         sort_columns.clear();
@@ -125,7 +136,7 @@ struct SortCursorImpl
         }
 
         pos = 0;
-        rows = all_columns[0]->size();
+        rows = num_rows;
         permutation = perm;
     }
 

src/Core/SortDescription.h

@@ -33,9 +33,12 @@ struct FillColumnDescription
     DataTypePtr fill_to_type;
     Field fill_step;        /// Default = +1 or -1 according to direction
     std::optional<IntervalKind> step_kind;
+    Field fill_staleness;   /// Default = Null - should not be considered
+    std::optional<IntervalKind> staleness_kind;
 
-    using StepFunction = std::function<void(Field &)>;
+    using StepFunction = std::function<void(Field &, Int32 jumps_count)>;
     StepFunction step_func;
+    StepFunction staleness_step_func;
 };
 
 /// Description of the sorting rule by one column.

src/DataTypes/DataTypeObject.cpp

@@ -1,6 +1,9 @@
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeObject.h>
 #include <DataTypes/DataTypeObjectDeprecated.h>
+#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/DataTypeString.h>
 #include <DataTypes/Serializations/SerializationJSON.h>
 #include <DataTypes/Serializations/SerializationObjectTypedPath.h>
 #include <DataTypes/Serializations/SerializationObjectDynamicPath.h>
@@ -230,6 +233,15 @@ MutableColumnPtr DataTypeObject::createColumn() const
     return ColumnObject::create(std::move(typed_path_columns), max_dynamic_paths, max_dynamic_types);
 }
 
+void DataTypeObject::forEachChild(const ChildCallback & callback) const
+{
+    for (const auto & [path, type] : typed_paths)
+    {
+        callback(*type);
+        type->forEachChild(callback);
+    }
+}
+
 namespace
 {
 
@@ -522,6 +534,13 @@ static DataTypePtr createObject(const ASTPtr & arguments, const DataTypeObject::
     return std::make_shared<DataTypeObject>(schema_format, std::move(typed_paths), std::move(paths_to_skip), std::move(path_regexps_to_skip), max_dynamic_paths, max_dynamic_types);
 }
 
+const DataTypePtr & DataTypeObject::getTypeOfSharedData()
+{
+    /// Array(Tuple(String, String))
+    static const DataTypePtr type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeTuple>(DataTypes{std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>()}, Names{"paths", "values"}));
+    return type;
+}
+
 static DataTypePtr createJSON(const ASTPtr & arguments)
 {
     auto context = CurrentThread::getQueryContext();

src/DataTypes/DataTypeObject.h

@@ -50,6 +50,8 @@ public:
 
     bool equals(const IDataType & rhs) const override;
 
+    void forEachChild(const ChildCallback &) const override;
+
     bool hasDynamicSubcolumnsData() const override { return true; }
     std::unique_ptr<SubstreamData> getDynamicSubcolumnData(std::string_view subcolumn_name, const SubstreamData & data, bool throw_if_null) const override;
 
@@ -63,6 +65,9 @@ public:
     size_t getMaxDynamicTypes() const { return max_dynamic_types; }
     size_t getMaxDynamicPaths() const { return max_dynamic_paths; }
 
+    /// Shared data has type Array(Tuple(String, String)).
+    static const DataTypePtr & getTypeOfSharedData();
+
 private:
     SchemaFormat schema_format;
     /// Set of paths with types that were specified in type declaration.

src/DataTypes/Serializations/SerializationDynamic.cpp

@@ -26,8 +26,8 @@ namespace ErrorCodes
 
 struct SerializeBinaryBulkStateDynamic : public ISerialization::SerializeBinaryBulkState
 {
-    SerializationDynamic::DynamicStructureSerializationVersion structure_version;
-    size_t max_dynamic_types;
+    SerializationDynamic::DynamicSerializationVersion structure_version;
+    size_t num_dynamic_types;
     DataTypePtr variant_type;
     Names variant_names;
     SerializationPtr variant_serialization;
@@ -81,15 +81,15 @@ void SerializationDynamic::enumerateStreams(
     settings.path.pop_back();
 }
 
-SerializationDynamic::DynamicStructureSerializationVersion::DynamicStructureSerializationVersion(UInt64 version) : value(static_cast<Value>(version))
+SerializationDynamic::DynamicSerializationVersion::DynamicSerializationVersion(UInt64 version) : value(static_cast<Value>(version))
 {
     checkVersion(version);
 }
 
-void SerializationDynamic::DynamicStructureSerializationVersion::checkVersion(UInt64 version)
+void SerializationDynamic::DynamicSerializationVersion::checkVersion(UInt64 version)
 {
-    if (version != VariantTypeName)
-        throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Dynamic structure serialization.");
+    if (version != V1 && version != V2)
+        throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Dynamic structure serialization: {}", version);
 }
 
 void SerializationDynamic::serializeBinaryBulkStatePrefix(
@@ -108,22 +108,17 @@ void SerializationDynamic::serializeBinaryBulkStatePrefix(
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Missing stream for Dynamic column structure during serialization of binary bulk state prefix");
 
     /// Write structure serialization version.
-    UInt64 structure_version = DynamicStructureSerializationVersion::Value::VariantTypeName;
+    UInt64 structure_version = DynamicSerializationVersion::Value::V2;
     writeBinaryLittleEndian(structure_version, *stream);
     auto dynamic_state = std::make_shared<SerializeBinaryBulkStateDynamic>(structure_version);
 
-    dynamic_state->max_dynamic_types = column_dynamic.getMaxDynamicTypes();
-    /// Write max_dynamic_types parameter, because it can differ from the max_dynamic_types
-    /// that is specified in the Dynamic type (we could decrease it before merge).
-    writeVarUInt(dynamic_state->max_dynamic_types, *stream);
-
     dynamic_state->variant_type = variant_info.variant_type;
     dynamic_state->variant_names = variant_info.variant_names;
     const auto & variant_column = column_dynamic.getVariantColumn();
 
-    /// Write information about variants.
-    size_t num_variants = dynamic_state->variant_names.size() - 1; /// Don't write shared variant, Dynamic column should always have it.
-    writeVarUInt(num_variants, *stream);
+    /// Write information about dynamic types.
+    dynamic_state->num_dynamic_types = dynamic_state->variant_names.size() - 1; ///  -1 for SharedVariant
+    writeVarUInt(dynamic_state->num_dynamic_types, *stream);
     if (settings.data_types_binary_encoding)
     {
         const auto & variants = assert_cast<const DataTypeVariant &>(*dynamic_state->variant_type).getVariants();
@@ -251,22 +246,25 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationDynamic::deserializeD
         UInt64 structure_version;
         readBinaryLittleEndian(structure_version, *structure_stream);
         auto structure_state = std::make_shared<DeserializeBinaryBulkStateDynamicStructure>(structure_version);
-        /// Read max_dynamic_types parameter.
-        readVarUInt(structure_state->max_dynamic_types, *structure_stream);
+        if (structure_state->structure_version.value == DynamicSerializationVersion::Value::V1)
+        {
+            /// Skip max_dynamic_types parameter in V1 serialization version.
+            size_t max_dynamic_types;
+            readVarUInt(max_dynamic_types, *structure_stream);
+        }
         /// Read information about variants.
         DataTypes variants;
-        size_t num_variants;
-        readVarUInt(num_variants, *structure_stream);
-        variants.reserve(num_variants + 1); /// +1 for shared variant.
+        readVarUInt(structure_state->num_dynamic_types, *structure_stream);
+        variants.reserve(structure_state->num_dynamic_types + 1); /// +1 for shared variant.
         if (settings.data_types_binary_encoding)
         {
-            for (size_t i = 0; i != num_variants; ++i)
+            for (size_t i = 0; i != structure_state->num_dynamic_types; ++i)
                 variants.push_back(decodeDataType(*structure_stream));
         }
         else
         {
             String data_type_name;
-            for (size_t i = 0; i != num_variants; ++i)
+            for (size_t i = 0; i != structure_state->num_dynamic_types; ++i)
             {
                 readStringBinary(data_type_name, *structure_stream);
                 variants.push_back(DataTypeFactory::instance().get(data_type_name));
@@ -364,9 +362,6 @@ void SerializationDynamic::serializeBinaryBulkWithMultipleStreamsAndCountTotalSi
     if (!variant_info.variant_type->equals(*dynamic_state->variant_type))
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of internal columns of Dynamic. Expected: {}, Got: {}", dynamic_state->variant_type->getName(), variant_info.variant_type->getName());
 
-    if (column_dynamic.getMaxDynamicTypes() != dynamic_state->max_dynamic_types)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of max_dynamic_types parameter of Dynamic. Expected: {}, Got: {}", dynamic_state->max_dynamic_types, column_dynamic.getMaxDynamicTypes());
-
     settings.path.push_back(Substream::DynamicData);
     assert_cast<const SerializationVariant &>(*dynamic_state->variant_serialization)
         .serializeBinaryBulkWithMultipleStreamsAndUpdateVariantStatistics(
@@ -424,7 +419,7 @@ void SerializationDynamic::deserializeBinaryBulkWithMultipleStreams(
 
     if (mutable_column->empty())
     {
-        column_dynamic.setMaxDynamicPaths(structure_state->max_dynamic_types);
+        column_dynamic.setMaxDynamicPaths(structure_state->num_dynamic_types);
         column_dynamic.setVariantType(structure_state->variant_type);
         column_dynamic.setStatistics(structure_state->statistics);
     }

src/DataTypes/Serializations/SerializationDynamic.h

@@ -16,18 +16,28 @@ public:
     {
     }
 
-    struct DynamicStructureSerializationVersion
+    struct DynamicSerializationVersion
     {
         enum Value
         {
-            VariantTypeName = 1,
+            /// V1 serialization:
+            /// - DynamicStructure stream:
+            ///     <max_dynamic_types parameter>
+            ///     <actual number of dynamic types>
+            ///     <list of dynamic types (list of variants in nested Variant column without SharedVariant)>
+            ///     <statistics with number of values for each dynamic type> (only in MergeTree serialization)
+            ///     <statistics with number of values for some types in SharedVariant> (only in MergeTree serialization)
+            /// - DynamicData stream: contains the data of nested Variant column.
+            V1 = 1,
+            /// V2 serialization: the same as V1 but without max_dynamic_types parameter in DynamicStructure stream.
+            V2 = 2,
         };
 
         Value value;
 
         static void checkVersion(UInt64 version);
 
-        explicit DynamicStructureSerializationVersion(UInt64 version);
+        explicit DynamicSerializationVersion(UInt64 version);
     };
 
     void enumerateStreams(
@@ -113,9 +123,9 @@ private:
 
     struct DeserializeBinaryBulkStateDynamicStructure : public ISerialization::DeserializeBinaryBulkState
     {
-        DynamicStructureSerializationVersion structure_version;
+        DynamicSerializationVersion structure_version;
         DataTypePtr variant_type;
-        size_t max_dynamic_types;
+        size_t num_dynamic_types;
         ColumnDynamic::StatisticsPtr statistics;
 
         explicit DeserializeBinaryBulkStateDynamicStructure(UInt64 structure_version_)

src/DataTypes/Serializations/SerializationObject.cpp

@@ -25,7 +25,7 @@ SerializationObject::SerializationObject(
     : typed_path_serializations(std::move(typed_path_serializations_))
     , paths_to_skip(paths_to_skip_)
     , dynamic_serialization(std::make_shared<SerializationDynamic>())
-    , shared_data_serialization(getTypeOfSharedData()->getDefaultSerialization())
+    , shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization())
 {
     /// We will need sorted order of typed paths to serialize them in order for consistency.
     sorted_typed_paths.reserve(typed_path_serializations.size());
@@ -38,13 +38,6 @@ SerializationObject::SerializationObject(
         path_regexps_to_skip.emplace_back(regexp_str);
 }
 
-const DataTypePtr & SerializationObject::getTypeOfSharedData()
-{
-    /// Array(Tuple(String, String))
-    static const DataTypePtr type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeTuple>(DataTypes{std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>()}, Names{"paths", "values"}));
-    return type;
-}
-
 bool SerializationObject::shouldSkipPath(const String & path) const
 {
     if (paths_to_skip.contains(path))
@@ -70,14 +63,13 @@ SerializationObject::ObjectSerializationVersion::ObjectSerializationVersion(UInt
 
 void SerializationObject::ObjectSerializationVersion::checkVersion(UInt64 version)
 {
-    if (version != V1 && version != STRING)
+    if (version != V1 && version != V2 && version != STRING)
         throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Object structure serialization.");
 }
 
 struct SerializeBinaryBulkStateObject: public ISerialization::SerializeBinaryBulkState
 {
     SerializationObject::ObjectSerializationVersion serialization_version;
-    size_t max_dynamic_paths;
     std::vector<String> sorted_dynamic_paths;
     std::unordered_map<String, ISerialization::SerializeBinaryBulkStatePtr> typed_path_states;
     std::unordered_map<String, ISerialization::SerializeBinaryBulkStatePtr> dynamic_path_states;
@@ -168,7 +160,7 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings,
 
     settings.path.push_back(Substream::ObjectSharedData);
     auto shared_data_substream_data = SubstreamData(shared_data_serialization)
-                                          .withType(getTypeOfSharedData())
+                                          .withType(DataTypeObject::getTypeOfSharedData())
                                           .withColumn(column_object ? column_object->getSharedDataPtr() : nullptr)
                                           .withSerializationInfo(data.serialization_info)
                                           .withDeserializeState(deserialize_state ? deserialize_state->shared_data_state : nullptr);
@@ -195,7 +187,7 @@ void SerializationObject::serializeBinaryBulkStatePrefix(
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Missing stream for Object column structure during serialization of binary bulk state prefix");
 
     /// Write serialization version.
-    UInt64 serialization_version = settings.write_json_as_string ? ObjectSerializationVersion::Value::STRING : ObjectSerializationVersion::Value::V1;
+    UInt64 serialization_version = settings.write_json_as_string ? ObjectSerializationVersion::Value::STRING : ObjectSerializationVersion::Value::V2;
     writeBinaryLittleEndian(serialization_version, *stream);
 
     auto object_state = std::make_shared<SerializeBinaryBulkStateObject>(serialization_version);
@@ -205,9 +197,6 @@ void SerializationObject::serializeBinaryBulkStatePrefix(
         return;
     }
 
-    object_state->max_dynamic_paths = column_object.getMaxDynamicPaths();
-    /// Write max_dynamic_paths parameter.
-    writeVarUInt(object_state->max_dynamic_paths, *stream);
     /// Write all dynamic paths in sorted order.
     object_state->sorted_dynamic_paths.reserve(dynamic_paths.size());
     for (const auto & [path, _] : dynamic_paths)
@@ -367,10 +356,15 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb
         UInt64 serialization_version;
         readBinaryLittleEndian(serialization_version, *structure_stream);
         auto structure_state = std::make_shared<DeserializeBinaryBulkStateObjectStructure>(serialization_version);
-        if (structure_state->serialization_version.value == ObjectSerializationVersion::Value::V1)
+        if (structure_state->serialization_version.value == ObjectSerializationVersion::Value::V1 || structure_state->serialization_version.value == ObjectSerializationVersion::Value::V2)
         {
-            /// Read max_dynamic_paths parameter.
-            readVarUInt(structure_state->max_dynamic_paths, *structure_stream);
+            if (structure_state->serialization_version.value == ObjectSerializationVersion::Value::V1)
+            {
+                /// Skip max_dynamic_paths parameter in V1 serialization version.
+                size_t max_dynamic_paths;
+                readVarUInt(max_dynamic_paths, *structure_stream);
+            }
+
             /// Read the sorted list of dynamic paths.
             size_t dynamic_paths_size;
             readVarUInt(dynamic_paths_size, *structure_stream);
@@ -453,9 +447,6 @@ void SerializationObject::serializeBinaryBulkWithMultipleStreams(
     const auto & dynamic_paths = column_object.getDynamicPaths();
     const auto & shared_data = column_object.getSharedDataPtr();
 
-    if (column_object.getMaxDynamicPaths() != object_state->max_dynamic_paths)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of max_dynamic_paths parameter of Object. Expected: {}, Got: {}", object_state->max_dynamic_paths, column_object.getMaxDynamicPaths());
-
     if (column_object.getDynamicPaths().size() != object_state->sorted_dynamic_paths.size())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of number of dynamic paths in Object. Expected: {}, Got: {}", object_state->sorted_dynamic_paths.size(), column_object.getDynamicPaths().size());
 
@@ -604,7 +595,7 @@ void SerializationObject::deserializeBinaryBulkWithMultipleStreams(
     /// If it's a new object column, set dynamic paths and statistics.
     if (column_object.empty())
     {
-        column_object.setMaxDynamicPaths(structure_state->max_dynamic_paths);
+        column_object.setMaxDynamicPaths(structure_state->sorted_dynamic_paths.size());
         column_object.setDynamicPaths(structure_state->sorted_dynamic_paths);
         column_object.setStatistics(structure_state->statistics);
     }

src/DataTypes/Serializations/SerializationObject.h

@@ -31,6 +31,8 @@ public:
             ///   - ObjectDynamicPath stream for each column in dynamic paths
             ///   - ObjectSharedData stream shared data column.
             V1 = 0,
+            /// V2 serialization: the same as V1 but without max_dynamic_paths parameter in ObjectStructure stream.
+            V2 = 2,
             /// String serialization:
             ///  - ObjectData stream with single String column containing serialized JSON.
             STRING = 1,
@@ -98,7 +100,6 @@ private:
     struct DeserializeBinaryBulkStateObjectStructure : public ISerialization::DeserializeBinaryBulkState
     {
         ObjectSerializationVersion serialization_version;
-        size_t max_dynamic_paths;
         std::vector<String> sorted_dynamic_paths;
         std::unordered_set<String> dynamic_paths;
         /// Paths statistics. Map (dynamic path) -> (number of non-null values in this path).
@@ -111,9 +112,6 @@ private:
         DeserializeBinaryBulkSettings & settings,
         SubstreamsDeserializeStatesCache * cache);
 
-    /// Shared data has type Array(Tuple(String, String)).
-    static const DataTypePtr & getTypeOfSharedData();
-
     struct TypedPathSubcolumnCreator : public ISubcolumnCreator
     {
         String path;

src/DataTypes/Serializations/SerializationObjectDynamicPath.cpp

@@ -18,7 +18,7 @@ SerializationObjectDynamicPath::SerializationObjectDynamicPath(
     , path(path_)
     , path_subcolumn(path_subcolumn_)
     , dynamic_serialization(std::make_shared<SerializationDynamic>())
-    , shared_data_serialization(SerializationObject::getTypeOfSharedData()->getDefaultSerialization())
+    , shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization())
     , max_dynamic_types(max_dynamic_types_)
 {
 }
@@ -67,8 +67,8 @@ void SerializationObjectDynamicPath::enumerateStreams(
     {
         settings.path.push_back(Substream::ObjectSharedData);
         auto shared_data_substream_data = SubstreamData(shared_data_serialization)
-                                              .withType(data.type ? SerializationObject::getTypeOfSharedData() : nullptr)
-                                              .withColumn(data.column ? SerializationObject::getTypeOfSharedData()->createColumn() : nullptr)
+                                              .withType(data.type ? DataTypeObject::getTypeOfSharedData() : nullptr)
+                                              .withColumn(data.column ? DataTypeObject::getTypeOfSharedData()->createColumn() : nullptr)
                                               .withSerializationInfo(data.serialization_info)
                                               .withDeserializeState(deserialize_state->nested_state);
         settings.path.back().data = shared_data_substream_data;
@@ -164,7 +164,7 @@ void SerializationObjectDynamicPath::deserializeBinaryBulkWithMultipleStreams(
         settings.path.push_back(Substream::ObjectSharedData);
         /// Initialize shared_data column if needed.
         if (result_column->empty())
-            dynamic_path_state->shared_data = SerializationObject::getTypeOfSharedData()->createColumn();
+            dynamic_path_state->shared_data = DataTypeObject::getTypeOfSharedData()->createColumn();
         size_t prev_size = result_column->size();
         shared_data_serialization->deserializeBinaryBulkWithMultipleStreams(dynamic_path_state->shared_data, limit, settings, dynamic_path_state->nested_state, cache);
         /// If we need to read a subcolumn from Dynamic column, create an empty Dynamic column, fill it and extract subcolumn.

src/DataTypes/Serializations/SerializationSubObject.cpp

@@ -17,7 +17,7 @@ SerializationSubObject::SerializationSubObject(
     : path_prefix(path_prefix_)
     , typed_paths_serializations(typed_paths_serializations_)
     , dynamic_serialization(std::make_shared<SerializationDynamic>())
-    , shared_data_serialization(SerializationObject::getTypeOfSharedData()->getDefaultSerialization())
+    , shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization())
 {
 }
 
@@ -64,8 +64,8 @@ void SerializationSubObject::enumerateStreams(
     /// We will need to read shared data to find all paths with requested prefix.
     settings.path.push_back(Substream::ObjectSharedData);
     auto shared_data_substream_data = SubstreamData(shared_data_serialization)
-                                          .withType(data.type ? SerializationObject::getTypeOfSharedData() : nullptr)
-                                          .withColumn(data.column ? SerializationObject::getTypeOfSharedData()->createColumn() : nullptr)
+                                          .withType(data.type ? DataTypeObject::getTypeOfSharedData() : nullptr)
+                                          .withColumn(data.column ? DataTypeObject::getTypeOfSharedData()->createColumn() : nullptr)
                                           .withSerializationInfo(data.serialization_info)
                                           .withDeserializeState(deserialize_state ? deserialize_state->shared_data_state : nullptr);
     settings.path.back().data = shared_data_substream_data;
@@ -208,7 +208,7 @@ void SerializationSubObject::deserializeBinaryBulkWithMultipleStreams(
     settings.path.push_back(Substream::ObjectSharedData);
     /// If it's a new object column, reinitialize column for shared data.
     if (result_column->empty())
-        sub_object_state->shared_data = SerializationObject::getTypeOfSharedData()->createColumn();
+        sub_object_state->shared_data = DataTypeObject::getTypeOfSharedData()->createColumn();
     size_t prev_size = column_object.size();
     shared_data_serialization->deserializeBinaryBulkWithMultipleStreams(sub_object_state->shared_data, limit, settings, sub_object_state->shared_data_state, cache);
     settings.path.pop_back();

src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp

@@ -307,6 +307,13 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure(
     if (!columns.empty())
         columns_part = fmt::format(" AND attname IN ('{}')", boost::algorithm::join(columns, "','"));
 
+    /// Bypassing the error of the missing column `attgenerated` in the system table `pg_attribute` for PostgreSQL versions below 12.
+    /// This trick involves executing a special query to the DBMS in advance to obtain the correct line with comment /// if column has GENERATED.
+    /// The result of the query will be the name of the column `attgenerated` or an empty string declaration for PostgreSQL version 11 and below.
+    /// This change does not degrade the function's performance but restores support for older versions and fix ERROR: column "attgenerated" does not exist.
+    pqxx::result gen_result{tx.exec("select case when current_setting('server_version_num')::int < 120000 then '''''' else 'attgenerated' end as generated")};
+    std::string generated = gen_result[0][0].as<std::string>();
+
     std::string query = fmt::format(
            "SELECT attname AS name, " /// column name
            "format_type(atttypid, atttypmod) AS type, " /// data type
@@ -315,11 +322,11 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure(
            "atttypid as type_id, "
            "atttypmod as type_modifier, "
            "attnum as att_num, "
-           "attgenerated as generated " /// if column has GENERATED
+           "{} as generated " /// if column has GENERATED
            "FROM pg_attribute "
            "WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) {}"
            "AND NOT attisdropped AND attnum > 0 "
-           "ORDER BY attnum ASC", where, columns_part);
+           "ORDER BY attnum ASC", generated, where, columns_part); /// Now we use variable `generated` to form query string. End of trick.
 
     auto postgres_table_with_schema = postgres_schema.empty() ? postgres_table : doubleQuoteString(postgres_schema) + '.' + doubleQuoteString(postgres_table);
     table.physical_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, false);

src/Databases/enableAllExperimentalSettings.cpp

@@ -32,6 +32,8 @@ void enableAllExperimentalSettings(ContextMutablePtr context)
 
     context->setSetting("allow_suspicious_low_cardinality_types", 1);
     context->setSetting("allow_suspicious_fixed_string_types", 1);
+    context->setSetting("allow_suspicious_types_in_group_by", 1);
+    context->setSetting("allow_suspicious_types_in_order_by", 1);
     context->setSetting("allow_suspicious_indices", 1);
     context->setSetting("allow_suspicious_codecs", 1);
     context->setSetting("allow_hyperscan", 1);

src/Dictionaries/HashedDictionary.h

@@ -334,22 +334,26 @@ HashedDictionary<dictionary_key_type, sparse, sharded>::~HashedDictionary()
         if (container.empty())
             return;
 
-        pool.trySchedule([&container, thread_group = CurrentThread::getGroup()]
-        {
-            SCOPE_EXIT_SAFE(
+        if (!pool.trySchedule([&container, thread_group = CurrentThread::getGroup()]
+            {
+                SCOPE_EXIT_SAFE(
+                    if (thread_group)
+                        CurrentThread::detachFromGroupIfNotDetached();
+                );
+
+                /// Do not account memory that was occupied by the dictionaries for the query/user context.
+                MemoryTrackerBlockerInThread memory_blocker;
+
                 if (thread_group)
-                    CurrentThread::detachFromGroupIfNotDetached();
-            );
+                    CurrentThread::attachToGroupIfDetached(thread_group);
+                setThreadName("HashedDictDtor");
 
-            /// Do not account memory that was occupied by the dictionaries for the query/user context.
+                clearContainer(container);
+            }))
+        {
             MemoryTrackerBlockerInThread memory_blocker;
-
-            if (thread_group)
-                CurrentThread::attachToGroupIfDetached(thread_group);
-            setThreadName("HashedDictDtor");
-
             clearContainer(container);
-        });
+        }
 
         ++hash_tables_count;
     };

src/Formats/PrettyFormatHelpers.cpp

@@ -0,0 +1,107 @@
+#include <Formats/PrettyFormatHelpers.h>
+#include <IO/WriteBuffer.h>
+#include <IO/WriteHelpers.h>
+#include <Processors/Chunk.h>
+#include <Common/formatReadable.h>
+
+
+static constexpr const char * GRAY_COLOR = "\033[90m";
+static constexpr const char * UNDERSCORE = "\033[4m";
+static constexpr const char * RESET_COLOR = "\033[0m";
+
+
+namespace DB
+{
+
+void writeReadableNumberTipIfSingleValue(WriteBuffer & out, const Chunk & chunk, const FormatSettings & settings, bool color)
+{
+    if (chunk.getNumRows() == 1 && chunk.getNumColumns() == 1)
+        writeReadableNumberTip(out, *chunk.getColumns()[0], 0, settings, color);
+}
+
+void writeReadableNumberTip(WriteBuffer & out, const IColumn & column, size_t row, const FormatSettings & settings, bool color)
+{
+    if (column.isNullAt(row))
+        return;
+
+    auto value = column.getFloat64(row);
+    auto threshold = settings.pretty.output_format_pretty_single_large_number_tip_threshold;
+
+    if (threshold && isFinite(value) && abs(value) > threshold)
+    {
+        if (color)
+            writeCString(GRAY_COLOR, out);
+        writeCString(" -- ", out);
+        formatReadableQuantity(value, out, 2);
+        if (color)
+            writeCString(RESET_COLOR, out);
+    }
+}
+
+
+String highlightDigitGroups(String source)
+{
+    if (source.size() <= 4)
+        return source;
+
+    bool is_regular_number = true;
+    size_t num_digits_before_decimal = 0;
+    for (auto c : source)
+    {
+        if (c == '-' || c == ' ')
+            continue;
+        if (c == '.')
+            break;
+        if (c >= '0' && c <= '9')
+        {
+            ++num_digits_before_decimal;
+        }
+        else
+        {
+            is_regular_number = false;
+            break;
+        }
+    }
+
+    if (!is_regular_number || num_digits_before_decimal <= 4)
+        return source;
+
+    String result;
+    size_t size = source.size();
+    result.reserve(2 * size);
+
+    bool before_decimal = true;
+    size_t digit_num = 0;
+    for (size_t i = 0; i < size; ++i)
+    {
+        auto c = source[i];
+        if (before_decimal && c >= '0' && c <= '9')
+        {
+            ++digit_num;
+            size_t offset = num_digits_before_decimal - digit_num;
+            if (offset && offset % 3 == 0)
+            {
+                result += UNDERSCORE;
+                result += c;
+                result += RESET_COLOR;
+            }
+            else
+            {
+                result += c;
+            }
+        }
+        else if (c == '.')
+        {
+            before_decimal = false;
+            result += c;
+        }
+        else
+        {
+            result += c;
+        }
+    }
+
+    return result;
+}
+
+}

src/Formats/PrettyFormatHelpers.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <base/types.h>
+
+
+namespace DB
+{
+
+class Chunk;
+class IColumn;
+class WriteBuffer;
+struct FormatSettings;
+
+/// Prints text describing the number in the form of: -- 12.34 million
+void writeReadableNumberTip(WriteBuffer & out, const IColumn & column, size_t row, const FormatSettings & settings, bool color);
+void writeReadableNumberTipIfSingleValue(WriteBuffer & out, const Chunk & chunk, const FormatSettings & settings, bool color);
+
+/// Underscores digit groups related to thousands using terminal ANSI escape sequences.
+String highlightDigitGroups(String source);
+
+}

src/Functions/FunctionsConversion.cpp

@@ -3921,7 +3921,7 @@ private:
         }
     }
 
-    WrapperType createTupleToObjectWrapper(const DataTypeTuple & from_tuple, bool has_nullable_subcolumns) const
+    WrapperType createTupleToObjectDeprecatedWrapper(const DataTypeTuple & from_tuple, bool has_nullable_subcolumns) const
     {
         if (!from_tuple.haveExplicitNames())
             throw Exception(ErrorCodes::TYPE_MISMATCH,
@@ -3968,7 +3968,7 @@ private:
         };
     }
 
-    WrapperType createMapToObjectWrapper(const DataTypeMap & from_map, bool has_nullable_subcolumns) const
+    WrapperType createMapToObjectDeprecatedWrapper(const DataTypeMap & from_map, bool has_nullable_subcolumns) const
     {
         auto key_value_types = from_map.getKeyValueTypes();
 
@@ -4048,11 +4048,11 @@ private:
     {
         if (const auto * from_tuple = checkAndGetDataType<DataTypeTuple>(from_type.get()))
         {
-            return createTupleToObjectWrapper(*from_tuple, to_type->hasNullableSubcolumns());
+            return createTupleToObjectDeprecatedWrapper(*from_tuple, to_type->hasNullableSubcolumns());
         }
         else if (const auto * from_map = checkAndGetDataType<DataTypeMap>(from_type.get()))
         {
-            return createMapToObjectWrapper(*from_map, to_type->hasNullableSubcolumns());
+            return createMapToObjectDeprecatedWrapper(*from_map, to_type->hasNullableSubcolumns());
         }
         else if (checkAndGetDataType<DataTypeString>(from_type.get()))
         {
@@ -4081,23 +4081,43 @@ private:
             "Cast to Object can be performed only from flatten named Tuple, Map or String. Got: {}", from_type->getName());
     }
 
+
     WrapperType createObjectWrapper(const DataTypePtr & from_type, const DataTypeObject * to_object) const
     {
         if (checkAndGetDataType<DataTypeString>(from_type.get()))
         {
             return [this](ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * nullable_source, size_t input_rows_count)
             {
-                auto res = ConvertImplGenericFromString<true>::execute(arguments, result_type, nullable_source, input_rows_count, context)->assumeMutable();
-                res->finalize();
-                return res;
+                return ConvertImplGenericFromString<true>::execute(arguments, result_type, nullable_source, input_rows_count, context);
+            };
+        }
+
+        /// Cast Tuple/Object/Map to JSON type through serializing into JSON string and parsing back into JSON column.
+        /// Potentially we can do smarter conversion Tuple -> JSON with type preservation, but it's questionable how exactly Tuple should be
+        /// converted to JSON (for example, should we recursively convert nested Array(Tuple) to Array(JSON) or not, should we infer types from String fields, etc).
+        if (checkAndGetDataType<DataTypeObjectDeprecated>(from_type.get()) || checkAndGetDataType<DataTypeTuple>(from_type.get()) || checkAndGetDataType<DataTypeMap>(from_type.get()))
+        {
+            return [this](ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * nullable_source, size_t input_rows_count)
+            {
+                auto json_string = ColumnString::create();
+                ColumnStringHelpers::WriteHelper write_helper(assert_cast<ColumnString &>(*json_string), input_rows_count);
+                auto & write_buffer = write_helper.getWriteBuffer();
+                FormatSettings format_settings = context ? getFormatSettings(context) : FormatSettings{};
+                auto serialization = arguments[0].type->getDefaultSerialization();
+                for (size_t i = 0; i < input_rows_count; ++i)
+                {
+                    serialization->serializeTextJSON(*arguments[0].column, i, write_buffer, format_settings);
+                    write_helper.rowWritten();
+                }
+                write_helper.finalize();
+
+                ColumnsWithTypeAndName args_with_json_string = {ColumnWithTypeAndName(json_string->getPtr(), std::make_shared<DataTypeString>(), "")};
+                return ConvertImplGenericFromString<true>::execute(args_with_json_string, result_type, nullable_source, input_rows_count, context);
             };
         }
 
         /// TODO: support CAST between JSON types with different parameters
-        ///       support CAST from Map to JSON
-        ///       support CAST from Tuple to JSON
-        ///       support CAST from Object('json') to JSON
-        throw Exception(ErrorCodes::TYPE_MISMATCH, "Cast to {} can be performed only from String. Got: {}", magic_enum::enum_name(to_object->getSchemaFormat()), from_type->getName());
+        throw Exception(ErrorCodes::TYPE_MISMATCH, "Cast to {} can be performed only from String/Map/Object/Tuple. Got: {}", magic_enum::enum_name(to_object->getSchemaFormat()), from_type->getName());
     }
 
     WrapperType createVariantToVariantWrapper(const DataTypeVariant & from_variant, const DataTypeVariant & to_variant) const

src/Functions/FunctionsMiscellaneous.h

@@ -6,6 +6,7 @@
 #include <IO/WriteBufferFromString.h>
 #include <IO/Operators.h>
 #include <Columns/ColumnFunction.h>
+#include <Columns/ColumnConst.h>
 #include <DataTypes/DataTypesNumber.h>
 
 
@@ -112,6 +113,7 @@ public:
         NamesAndTypesList lambda_arguments;
         String return_name;
         DataTypePtr return_type;
+        bool allow_constant_folding;
     };
 
     using CapturePtr = std::shared_ptr<Capture>;
@@ -122,6 +124,7 @@ public:
     String getName() const override { return "FunctionCapture"; }
 
     bool useDefaultImplementationForNulls() const override { return false; }
+
     /// It's possible if expression_actions contains function that don't use
     /// default implementation for Nothing and one of captured columns can be Nothing
     /// Example: SELECT arrayMap(x -> [x, arrayElement(y, 0)], []), [] as y
@@ -148,7 +151,26 @@ public:
         auto function = std::make_unique<FunctionExpression>(expression_actions, types, names,
                                                              capture->return_type, capture->return_name);
 
-        return ColumnFunction::create(input_rows_count, std::move(function), arguments);
+        /// If all the captured arguments are constant, let's also return ColumnConst (with ColumnFunction inside it).
+        /// Consequently, it allows to treat higher order functions with constant arrays and constant captured columns
+        /// as constant expressions.
+        /// Consequently, it allows its usage in contexts requiring constants, such as the right hand side of IN.
+        bool constant_folding = capture->allow_constant_folding
+            && std::all_of(arguments.begin(), arguments.end(),
+            [](const auto & arg) { return arg.column->isConst(); });
+
+        if (constant_folding)
+        {
+            ColumnsWithTypeAndName arguments_resized = arguments;
+            for (auto & elem : arguments_resized)
+                elem.column = elem.column->cloneResized(1);
+
+            return ColumnConst::create(ColumnFunction::create(1, std::move(function), arguments_resized), input_rows_count);
+        }
+        else
+        {
+            return ColumnFunction::create(input_rows_count, std::move(function), arguments);
+        }
     }
 
 private:
@@ -203,7 +225,8 @@ public:
             const Names & captured_names,
             const NamesAndTypesList & lambda_arguments,
             const DataTypePtr & function_return_type,
-            const String & expression_return_name)
+            const String & expression_return_name,
+            bool allow_constant_folding)
         : expression_actions(std::move(expression_actions_))
     {
         /// Check that expression does not contain unusual actions that will break columns structure.
@@ -246,6 +269,7 @@ public:
                 .lambda_arguments = lambda_arguments,
                 .return_name = expression_return_name,
                 .return_type = function_return_type,
+                .allow_constant_folding = allow_constant_folding,
         });
     }
 

src/Functions/IFunction.h

@@ -184,7 +184,7 @@ public:
 
     /** If function isSuitableForConstantFolding then, this method will be called during query analysis
       * if some arguments are constants. For example logical functions (AndFunction, OrFunction) can
-      * return they result based on some constant arguments.
+      * return the result based on some constant arguments.
       * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForNothing,
       * useDefaultImplementationForConstants, useDefaultImplementationForLowCardinality are not applied.
       */

src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp

@@ -24,92 +24,7 @@ namespace ErrorCodes
 
 void UserDefinedSQLFunctionVisitor::visit(ASTPtr & ast)
 {
-    if (!ast)
-    {
-        chassert(false);
-        return;
-    }
-
-    /// FIXME: this helper should use updatePointerToChild(), but
-    /// forEachPointerToChild() is not implemented for ASTColumnDeclaration
-    /// (and also some members should be adjusted for this).
-    const auto visit_child_with_shared_ptr = [&](ASTPtr & child)
-    {
-        if (!child)
-            return;
-
-        auto * old_value = child.get();
-        visit(child);
-
-        // child did not change
-        if (old_value == child.get())
-            return;
-
-        // child changed, we need to modify it in the list of children of the parent also
-        for (auto & current_child : ast->children)
-        {
-            if (current_child.get() == old_value)
-                current_child = child;
-        }
-    };
-
-    if (auto * col_decl = ast->as<ASTColumnDeclaration>())
-    {
-        visit_child_with_shared_ptr(col_decl->default_expression);
-        visit_child_with_shared_ptr(col_decl->ttl);
-        return;
-    }
-
-    if (auto * storage = ast->as<ASTStorage>())
-    {
-        const auto visit_child = [&](IAST * & child)
-        {
-            if (!child)
-                return;
-
-            if (const auto * function = child->template as<ASTFunction>())
-            {
-                std::unordered_set<std::string> udf_in_replace_process;
-                auto replace_result = tryToReplaceFunction(*function, udf_in_replace_process);
-                if (replace_result)
-                    ast->setOrReplace(child, replace_result);
-            }
-
-            visit(child);
-        };
-
-        visit_child(storage->partition_by);
-        visit_child(storage->primary_key);
-        visit_child(storage->order_by);
-        visit_child(storage->sample_by);
-        visit_child(storage->ttl_table);
-
-        return;
-    }
-
-    if (auto * alter = ast->as<ASTAlterCommand>())
-    {
-        /// It is OK to use updatePointerToChild() because ASTAlterCommand implements forEachPointerToChild()
-        const auto visit_child_update_parent = [&](ASTPtr & child)
-        {
-            if (!child)
-                return;
-
-            auto * old_ptr = child.get();
-            visit(child);
-            auto * new_ptr = child.get();
-
-            /// Some AST classes have naked pointers to children elements as members.
-            /// We have to replace them if the child was replaced.
-            if (new_ptr != old_ptr)
-                ast->updatePointerToChild(old_ptr, new_ptr);
-        };
-
-        for (auto & children : alter->children)
-            visit_child_update_parent(children);
-
-        return;
-    }
+    chassert(ast);
 
     if (const auto * function = ast->template as<ASTFunction>())
     {
@@ -120,7 +35,19 @@ void UserDefinedSQLFunctionVisitor::visit(ASTPtr & ast)
     }
 
     for (auto & child : ast->children)
+    {
+        if (!child)
+            return;
+
+        auto * old_ptr = child.get();
         visit(child);
+        auto * new_ptr = child.get();
+
+        /// Some AST classes have naked pointers to children elements as members.
+        /// We have to replace them if the child was replaced.
+        if (new_ptr != old_ptr)
+            ast->updatePointerToChild(old_ptr, new_ptr);
+    }
 }
 
 void UserDefinedSQLFunctionVisitor::visit(IAST * ast)

src/Functions/array/FunctionArrayMapped.h

@@ -282,7 +282,9 @@ public:
             if (!column_with_type_and_name.column)
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function.", getName());
 
-            const auto * column_function = typeid_cast<const ColumnFunction *>(column_with_type_and_name.column.get());
+            auto column_function_materialized = column_with_type_and_name.column->convertToFullColumnIfConst();
+
+            const auto * column_function = typeid_cast<const ColumnFunction *>(column_function_materialized.get());
             if (!column_function)
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function.", getName());