Merge branch 'ClickHouse:master' into hive_partitioning_default

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,7 +12,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py
 - Backward Incompatible Change
 - Build/Testing/Packaging Improvement
 - Documentation (changelog entry is not required)
-- Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+- Critical Bug Fix (crash, data loss, RBAC)
 - Bug Fix (user-visible misbehavior in an official stable release)
 - CI Fix or Improvement (changelog entry is not required)
 - Not for changelog (changelog entry is not required)

README.md

@@ -47,6 +47,7 @@ Upcoming meetups
 * [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21
 * [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26
 * [Amsterdam Meetup](https://www.meetup.com/clickhouse-netherlands-user-group/events/303638814) - December 3
+* [Stockholm Meetup](https://www.meetup.com/clickhouse-stockholm-user-group/events/304382411) - December 9
 * [New York Meetup](https://www.meetup.com/clickhouse-new-york-user-group/events/304268174) - December 9
 * [San Francisco Meetup](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/304286951/) - December 12
 

contrib/SimSIMD

@@ -1 +1 @@
-Subproject commit ee3c9c9c00b51645f62a1a9e99611b78c0052a21
+Subproject commit fa60f1b8e3582c50978f0ae86c2ebb6c9af957f3

docker/keeper/entrypoint.sh

@@ -1,21 +1,31 @@
 #!/bin/bash
 
-set +x
 set -eo pipefail
 shopt -s nullglob
 
 DO_CHOWN=1
-if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then
+if [[ "${CLICKHOUSE_RUN_AS_ROOT:=0}" = "1" || "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]]; then
     DO_CHOWN=0
 fi
 
-CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
-CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+# CLICKHOUSE_UID and CLICKHOUSE_GID are kept for backward compatibility, but deprecated
+# One must use either "docker run --user" or CLICKHOUSE_RUN_AS_ROOT=1 to run the process as
+# FIXME: Remove ALL CLICKHOUSE_UID CLICKHOUSE_GID before 25.3
+if [[ "${CLICKHOUSE_UID:-}" || "${CLICKHOUSE_GID:-}" ]]; then
+    echo 'WARNING: Support for CLICKHOUSE_UID/CLICKHOUSE_GID will be removed in a couple of releases.' >&2
+    echo 'WARNING: Either use a proper "docker run --user=xxx:xxxx" argument instead of CLICKHOUSE_UID/CLICKHOUSE_GID' >&2
+    echo 'WARNING: or set "CLICKHOUSE_RUN_AS_ROOT=1" ENV to run the clickhouse-server as root:root' >&2
+fi
 
-# support --user
-if [ "$(id -u)" = "0" ]; then
-    USER=$CLICKHOUSE_UID
-    GROUP=$CLICKHOUSE_GID
+# support `docker run --user=xxx:xxxx`
+if [[ "$(id -u)" = "0" ]]; then
+    if [[ "$CLICKHOUSE_RUN_AS_ROOT" = 1 ]]; then
+        USER=0
+        GROUP=0
+    else
+        USER="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
+        GROUP="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+    fi
     if command -v gosu &> /dev/null; then
         gosu="gosu $USER:$GROUP"
     elif command -v su-exec &> /dev/null; then
@@ -82,11 +92,11 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
 
     # There is a config file. It is already tested with gosu (if it is readably by keeper user)
     if [ -f "$KEEPER_CONFIG" ]; then
-        exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@"
+        exec $gosu clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@"
     fi
 
     # There is no config file. Will use embedded one
-    exec $gosu /usr/bin/clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
+    exec $gosu clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
 fi
 
 # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image

docker/server/Dockerfile.ubuntu

@@ -88,34 +88,32 @@ RUN if [ -n "${single_binary_location_url}" ]; then \
 #docker-official-library:on
 
 # A fallback to installation from ClickHouse repository
-RUN if ! clickhouse local -q "SELECT ''" > /dev/null 2>&1; then \
-        apt-get update \
-        && apt-get install --yes --no-install-recommends \
-            apt-transport-https \
-            dirmngr \
-            gnupg2 \
-        && mkdir -p /etc/apt/sources.list.d \
-        && GNUPGHOME=$(mktemp -d) \
-        && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
-            --keyring /usr/share/keyrings/clickhouse-keyring.gpg \
-            --keyserver hkp://keyserver.ubuntu.com:80 \
-            --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
-        && rm -rf "$GNUPGHOME" \
-        && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
-        && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
-        && echo "installing from repository: ${REPOSITORY}" \
-        && apt-get update \
-        && for package in ${PACKAGES}; do \
-            packages="${packages} ${package}=${VERSION}" \
-        ; done \
-        && apt-get install --allow-unauthenticated --yes --no-install-recommends ${packages} || exit 1 \
-        && rm -rf \
-            /var/lib/apt/lists/* \
-            /var/cache/debconf \
-            /tmp/* \
-        && apt-get autoremove --purge -yq libksba8 \
-        && apt-get autoremove -yq \
-    ; fi
+# It works unless the clickhouse binary already exists
+RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \
+    ; apt-get update \
+    && apt-get install --yes --no-install-recommends \
+        dirmngr \
+        gnupg2 \
+    && mkdir -p /etc/apt/sources.list.d \
+    && GNUPGHOME=$(mktemp -d) \
+    && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
+        --keyring /usr/share/keyrings/clickhouse-keyring.gpg \
+        --keyserver hkp://keyserver.ubuntu.com:80 \
+        --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
+    && rm -rf "$GNUPGHOME" \
+    && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
+    && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
+    && echo "installing from repository: ${REPOSITORY}" \
+    && apt-get update \
+    && for package in ${PACKAGES}; do \
+        packages="${packages} ${package}=${VERSION}" \
+    ; done \
+    && apt-get install --yes --no-install-recommends ${packages} || exit 1 \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /var/cache/debconf \
+        /tmp/* \
+    && apt-get autoremove --purge -yq dirmngr gnupg2
 
 # post install
 # we need to allow "others" access to clickhouse folder, because docker container
@@ -126,8 +124,6 @@ RUN clickhouse-local -q 'SELECT * FROM system.build_options' \
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
 ENV TZ UTC
 
 RUN mkdir /docker-entrypoint-initdb.d

docker/server/README.md

@@ -1,3 +1,11 @@
+<!---
+The README.md is generated by README.sh from the following sources:
+- README.src/content.md
+- README.src/license.md
+
+If you want to change it, edit these files
+-->
+
 # ClickHouse Server Docker Image
 
 ## What is ClickHouse?
@@ -8,6 +16,7 @@ ClickHouse works 100-1000x faster than traditional database management systems,
 
 For more information and documentation see https://clickhouse.com/.
 
+<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
 ## Versions
 
 -	The `latest` tag points to the latest release of the latest stable branch.
@@ -16,11 +25,12 @@ For more information and documentation see https://clickhouse.com/.
 -	The tag `head` is built from the latest commit to the default branch.
 -	Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
 
+<!-- REMOVE UNTIL HERE -->
 ### Compatibility
 
 -	The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
 -	The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
--	Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run [--privileged | --security-opt seccomp=unconfined]` instead, however that has security implications.
+-	Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run --security-opt seccomp=unconfined` instead, however that has security implications.
 
 ## How to use this image
 
@@ -30,7 +40,7 @@ For more information and documentation see https://clickhouse.com/.
 docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```
 
-By default, ClickHouse will be accessible only via the Docker network. See the [networking section below](#networking).
+By default, ClickHouse will be accessible only via the Docker network. See the **networking** section below.
 
 By default, starting above server instance will be run as the `default` user without password.
 
@@ -47,7 +57,7 @@ More information about the [ClickHouse client](https://clickhouse.com/docs/en/in
 ### connect to it using curl
 
 ```bash
-echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server curlimages/curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
+echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server buildpack-deps:curl curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
 ```
 
 More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/).
@@ -70,7 +80,7 @@ echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @-
 
 `22.6.3.35`
 
-or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
+Or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
 
 ```bash
 docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
@@ -88,8 +98,8 @@ Typically you may want to mount the following folders inside your container to a
 
 ```bash
 docker run -d \
-    -v $(realpath ./ch_data):/var/lib/clickhouse/ \
-    -v $(realpath ./ch_logs):/var/log/clickhouse-server/ \
+    -v "$PWD/ch_data:/var/lib/clickhouse/" \
+    -v "$PWD/ch_logs:/var/log/clickhouse-server/" \
     --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```
 
@@ -111,6 +121,8 @@ docker run -d \
     --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```
 
+Read more in [knowledge base](https://clickhouse.com/docs/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker).
+
 ## Configuration
 
 The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/).
@@ -126,8 +138,8 @@ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /pa
 ### Start server as custom user
 
 ```bash
-# $(pwd)/data/clickhouse should exist and be owned by current user
-docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
+# $PWD/data/clickhouse should exist and be owned by current user
+docker run --rm --user "${UID}:${GID}" --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
 ```
 
 When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start.
@@ -135,7 +147,7 @@ When you use the image with local directories mounted, you probably want to spec
 ### Start server from root (useful in case of enabled user namespace)
 
 ```bash
-docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
+docker run --rm -e CLICKHOUSE_RUN_AS_ROOT=1 --name clickhouse-server-userns -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server
 ```
 
 ### How to create default database and user on starting

docker/server/README.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -ueo pipefail
+
+# A script to generate README.sh close to as it done in https://github.com/docker-library/docs
+
+WORKDIR=$(dirname "$0")
+SCRIPT_NAME=$(basename "$0")
+CONTENT=README.src/content.md
+LICENSE=README.src/license.md
+cd "$WORKDIR"
+
+R=README.md
+
+cat > "$R" <<EOD
+<!---
+The $R is generated by $SCRIPT_NAME from the following sources:
+- $CONTENT
+- $LICENSE
+
+If you want to change it, edit these files
+-->
+
+EOD
+
+cat "$CONTENT" >> "$R"
+
+cat >> "$R" <<EOD
+
+## License
+
+$(cat $LICENSE)
+EOD
+
+# Remove %%LOGO%% from the file with one line below
+sed -i '/^%%LOGO%%/,+1d' "$R"
+
+# Replace each %%IMAGE%% with our `clickhouse/clickhouse-server`
+sed -i '/%%IMAGE%%/s:%%IMAGE%%:clickhouse/clickhouse-server:g' $R

docker/server/README.src/README-short.txt

@@ -0,0 +1 @@
+ClickHouse is the fastest and most resource efficient OSS database for real-time apps and analytics.

docker/server/README.src/content.md

@@ -0,0 +1,170 @@
+# ClickHouse Server Docker Image
+
+## What is ClickHouse?
+
+%%LOGO%%
+
+ClickHouse is an open-source column-oriented DBMS (columnar database management system) for online analytical processing (OLAP) that allows users to generate analytical reports using SQL queries in real-time.
+
+ClickHouse works 100-1000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second. With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance.
+
+For more information and documentation see https://clickhouse.com/.
+
+<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
+## Versions
+
+-	The `latest` tag points to the latest release of the latest stable branch.
+-	Branch tags like `22.2` point to the latest release of the corresponding branch.
+-	Full version tags like `22.2.3.5` point to the corresponding release.
+-	The tag `head` is built from the latest commit to the default branch.
+-	Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
+
+<!-- REMOVE UNTIL HERE -->
+### Compatibility
+
+-	The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
+-	The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
+-	Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run --security-opt seccomp=unconfined` instead, however that has security implications.
+
+## How to use this image
+
+### start server instance
+
+```bash
+docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+```
+
+By default, ClickHouse will be accessible only via the Docker network. See the **networking** section below.
+
+By default, starting above server instance will be run as the `default` user without password.
+
+### connect to it from a native client
+
+```bash
+docker run -it --rm --link some-clickhouse-server:clickhouse-server --entrypoint clickhouse-client %%IMAGE%% --host clickhouse-server
+# OR
+docker exec -it some-clickhouse-server clickhouse-client
+```
+
+More information about the [ClickHouse client](https://clickhouse.com/docs/en/interfaces/cli/).
+
+### connect to it using curl
+
+```bash
+echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server buildpack-deps:curl curl 'http://clickhouse-server:8123/?query=' -s --data-binary @-
+```
+
+More information about the [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/).
+
+### stopping / removing the container
+
+```bash
+docker stop some-clickhouse-server
+docker rm some-clickhouse-server
+```
+
+### networking
+
+You can expose your ClickHouse running in docker by [mapping a particular port](https://docs.docker.com/config/containers/container-networking/) from inside the container using host ports:
+
+```bash
+docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @-
+```
+
+`22.6.3.35`
+
+Or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
+
+```bash
+docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @-
+```
+
+`22.6.3.35`
+
+### Volumes
+
+Typically you may want to mount the following folders inside your container to achieve persistency:
+
+-	`/var/lib/clickhouse/` - main folder where ClickHouse stores the data
+-	`/var/log/clickhouse-server/` - logs
+
+```bash
+docker run -d \
+    -v "$PWD/ch_data:/var/lib/clickhouse/" \
+    -v "$PWD/ch_logs:/var/log/clickhouse-server/" \
+    --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+```
+
+You may also want to mount:
+
+-	`/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments
+-	`/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments
+-	`/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below).
+
+### Linux capabilities
+
+ClickHouse has some advanced functionality, which requires enabling several [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html).
+
+They are optional and can be enabled using the following [docker command-line arguments](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities):
+
+```bash
+docker run -d \
+    --cap-add=SYS_NICE --cap-add=NET_ADMIN --cap-add=IPC_LOCK \
+    --name some-clickhouse-server --ulimit nofile=262144:262144 %%IMAGE%%
+```
+
+Read more in [knowledge base](https://clickhouse.com/docs/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker).
+
+## Configuration
+
+The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/).
+
+ClickHouse configuration is represented with a file "config.xml" ([documentation](https://clickhouse.com/docs/en/operations/configuration_files/))
+
+### Start server instance with custom configuration
+
+```bash
+docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml %%IMAGE%%
+```
+
+### Start server as custom user
+
+```bash
+# $PWD/data/clickhouse should exist and be owned by current user
+docker run --rm --user "${UID}:${GID}" --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" %%IMAGE%%
+```
+
+When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start.
+
+### Start server from root (useful in case of enabled user namespace)
+
+```bash
+docker run --rm -e CLICKHOUSE_RUN_AS_ROOT=1 --name clickhouse-server-userns -v "$PWD/logs/clickhouse:/var/log/clickhouse-server" -v "$PWD/data/clickhouse:/var/lib/clickhouse" %%IMAGE%%
+```
+
+### How to create default database and user on starting
+
+Sometimes you may want to create a user (user named `default` is used by default) and database on a container start. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`:
+
+```bash
+docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp %%IMAGE%%
+```
+
+## How to extend this image
+
+To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service.  
+Also, you can provide environment variables `CLICKHOUSE_USER` & `CLICKHOUSE_PASSWORD` that will be used for clickhouse-client during initialization.
+
+For example, to add an additional user and database, add the following to `/docker-entrypoint-initdb.d/init-db.sh`:
+
+```bash
+#!/bin/bash
+set -e
+
+clickhouse client -n <<-EOSQL
+    CREATE DATABASE docker;
+    CREATE TABLE docker.docker (x Int32) ENGINE = Log;
+EOSQL
+```

docker/server/README.src/github-repo

@@ -0,0 +1 @@
+https://github.com/ClickHouse/ClickHouse

docker/server/README.src/license.md

@@ -0,0 +1 @@
+View [license information](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) for the software contained in this image.

docker/server/README.src/logo.svg

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 616 616">
+  <defs>
+    <style>
+      .cls-1 {
+        clip-path: url(#clippath);
+      }
+
+      .cls-2 {
+        fill: none;
+      }
+
+      .cls-2, .cls-3, .cls-4 {
+        stroke-width: 0px;
+      }
+
+      .cls-3 {
+        fill: #1e1e1e;
+      }
+
+      .cls-4 {
+        fill: #faff69;
+      }
+    </style>
+    <clipPath id="clippath">
+      <rect class="cls-2" x="83.23" y="71.73" width="472.55" height="472.55"/>
+    </clipPath>
+  </defs>
+  <g id="Layer_2" data-name="Layer 2">
+    <rect class="cls-4" width="616" height="616"/>
+  </g>
+  <g id="Layer_1" data-name="Layer 1">
+    <g class="cls-1">
+      <g>
+        <path class="cls-3" d="m120.14,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
+        <path class="cls-3" d="m208.75,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
+        <path class="cls-3" d="m297.35,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
+        <path class="cls-3" d="m385.94,113.3c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.66,2.09,4.66,4.66v389.38c0,2.57-2.09,4.66-4.66,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66V113.3Z"/>
+        <path class="cls-3" d="m474.56,268.36c0-2.57,2.09-4.66,4.66-4.66h34.98c2.57,0,4.65,2.09,4.65,4.66v79.28c0,2.57-2.09,4.66-4.65,4.66h-34.98c-2.57,0-4.66-2.09-4.66-4.66v-79.28Z"/>
+      </g>
+    </g>
+  </g>
+</svg>

docker/server/README.src/maintainer.md

@@ -0,0 +1 @@
+[ClickHouse Inc.](%%GITHUB-REPO%%)

docker/server/README.src/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

docker/server/entrypoint.sh

@@ -4,17 +4,28 @@ set -eo pipefail
 shopt -s nullglob
 
 DO_CHOWN=1
-if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then
+if [[ "${CLICKHOUSE_RUN_AS_ROOT:=0}" = "1" || "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]]; then
     DO_CHOWN=0
 fi
 
-CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
-CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+# CLICKHOUSE_UID and CLICKHOUSE_GID are kept for backward compatibility, but deprecated
+# One must use either "docker run --user" or CLICKHOUSE_RUN_AS_ROOT=1 to run the process as
+# FIXME: Remove ALL CLICKHOUSE_UID CLICKHOUSE_GID before 25.3
+if [[ "${CLICKHOUSE_UID:-}" || "${CLICKHOUSE_GID:-}" ]]; then
+    echo 'WARNING: Support for CLICKHOUSE_UID/CLICKHOUSE_GID will be removed in a couple of releases.' >&2
+    echo 'WARNING: Either use a proper "docker run --user=xxx:xxxx" argument instead of CLICKHOUSE_UID/CLICKHOUSE_GID' >&2
+    echo 'WARNING: or set "CLICKHOUSE_RUN_AS_ROOT=1" ENV to run the clickhouse-server as root:root' >&2
+fi
 
-# support --user
-if [ "$(id -u)" = "0" ]; then
-    USER=$CLICKHOUSE_UID
-    GROUP=$CLICKHOUSE_GID
+# support `docker run --user=xxx:xxxx`
+if [[ "$(id -u)" = "0" ]]; then
+    if [[ "$CLICKHOUSE_RUN_AS_ROOT" = 1 ]]; then
+        USER=0
+        GROUP=0
+    else
+        USER="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
+        GROUP="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+    fi
 else
     USER="$(id -u)"
     GROUP="$(id -g)"
@@ -55,14 +66,14 @@ function create_directory_and_do_chown() {
     [ -z "$dir" ] && return
     # ensure directories exist
     if [ "$DO_CHOWN" = "1" ]; then
-        mkdir="mkdir"
+        mkdir=( mkdir )
     else
         # if DO_CHOWN=0 it means that the system does not map root user to "admin" permissions
         # it mainly happens on NFS mounts where root==nobody for security reasons
         # thus mkdir MUST run with user id/gid and not from nobody that has zero permissions
-        mkdir="/usr/bin/clickhouse su "${USER}:${GROUP}" mkdir"
+        mkdir=( clickhouse su "${USER}:${GROUP}" mkdir )
     fi
-    if ! $mkdir -p "$dir"; then
+    if ! "${mkdir[@]}" -p "$dir"; then
         echo "Couldn't create necessary directory: $dir"
         exit 1
     fi
@@ -143,7 +154,7 @@ if [ -n "${RUN_INITDB_SCRIPTS}" ]; then
         fi
 
         # Listen only on localhost until the initialization is done
-        /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
+        clickhouse su "${USER}:${GROUP}" clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
         pid="$!"
 
         # check if clickhouse is ready to accept connections
@@ -203,18 +214,8 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
     CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0}
     export CLICKHOUSE_WATCHDOG_ENABLE
 
-    # An option for easy restarting and replacing clickhouse-server in a container, especially in Kubernetes.
-    # For example, you can replace the clickhouse-server binary to another and restart it while keeping the container running.
-    if [[ "${CLICKHOUSE_DOCKER_RESTART_ON_EXIT:-0}" -eq "1" ]]; then
-        while true; do
-            # This runs the server as a child process of the shell script:
-            /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@" ||:
-            echo >&2 'ClickHouse Server exited, and the environment variable CLICKHOUSE_DOCKER_RESTART_ON_EXIT is set to 1. Restarting the server.'
-        done
-    else
-        # This replaces the shell script with the server:
-        exec /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@"
-    fi
+    # This replaces the shell script with the server:
+    exec clickhouse su "${USER}:${GROUP}" clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@"
 fi
 
 # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image

docs/en/engines/table-engines/mergetree-family/annindexes.md

@@ -54,7 +54,7 @@ Parameters:
 - `distance_function`: either `L2Distance` (the [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance) - the length of a
   line between two points in Euclidean space), or `cosineDistance` (the [cosine
   distance](https://en.wikipedia.org/wiki/Cosine_similarity#Cosine_distance)- the angle between two non-zero vectors).
-- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing the vector with reduced precision (optional, default: `bf16`)
+- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing vectors with reduced precision (optional, default: `bf16`)
 - `hnsw_max_connections_per_layer`: the number of neighbors per HNSW graph node, also known as `M` in the [HNSW
   paper](https://doi.org/10.1109/TPAMI.2018.2889473) (optional, default: 32)
 - `hnsw_candidate_list_size_for_construction`: the size of the dynamic candidate list when constructing the HNSW graph, also known as
@@ -92,8 +92,8 @@ Vector similarity indexes currently support two distance functions:
 - `cosineDistance`, also called cosine similarity, is the cosine of the angle between two (non-zero) vectors
   ([Wikipedia](https://en.wikipedia.org/wiki/Cosine_similarity)).
 
-Vector similarity indexes allows storing the vectors in reduced precision formats. Supported scalar kinds are `f64`, `f32`, `f16` or `i8`.
-If no scalar kind was specified during index creation, `f16` is used as default.
+Vector similarity indexes allows storing the vectors in reduced precision formats. Supported scalar kinds are `f64`, `f32`, `f16`, `bf16`,
+and `i8`. If no scalar kind was specified during index creation, `bf16` is used as default.
 
 For normalized data, `L2Distance` is usually a better choice, otherwise `cosineDistance` is recommended to compensate for scale. If no
 distance function was specified during index creation, `L2Distance` is used as default.

docs/en/getting-started/example-datasets/tpch.md

@@ -33,6 +33,21 @@ Then, generate the data. Parameter `-s` specifies the scale factor. For example,
 ./dbgen -s 100
 ```
 
+Detailed table sizes with scale factor 100:
+
+| Table    | size (in rows) | size (compressed in ClickHouse) |
+|----------|----------------|---------------------------------|
+| nation   | 25             | 2 kB                            |
+| region   | 5              | 1 kB                            |
+| part     | 20.000.000     | 895 MB                          |
+| supplier | 1.000.000      | 75 MB                           |
+| partsupp | 80.000.000     | 4.37 GB                         |
+| customer | 15.000.000     | 1.19 GB                         |
+| orders   | 150.000.000    | 6.15 GB                         |
+| lineitem | 600.00.00      | 26.69 GB                        |
+
+(Compressed sizes in ClickHouse are taken from `system.tables.total_bytes` and based on below table definitions.)
+
 Now create tables in ClickHouse.
 
 We stick as closely as possible to the rules of the TPC-H specification:
@@ -151,10 +166,37 @@ clickhouse-client --format_csv_delimiter '|' --query "INSERT INTO orders FORMAT
 clickhouse-client --format_csv_delimiter '|' --query "INSERT INTO lineitem FORMAT CSV" < lineitem.tbl
 ```
 
-The queries are generated by `./qgen -s <scaling_factor>`. Example queries for `s = 100`:
+:::note
+Instead of using tpch-kit and generating the tables by yourself, you can alternatively import the data from a public S3 bucket. Make sure
+to create empty tables first using above `CREATE` statements.
+
+```sql
+-- Scaling factor 1
+INSERT INTO nation SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/nation.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO region SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/region.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO part SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/part.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO supplier SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/supplier.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO partsupp SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/partsupp.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO customer SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/customer.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO orders SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/orders.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO lineitem SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/1/lineitem.tbl', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+
+-- Scaling factor 100
+INSERT INTO nation SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/nation.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO region SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/region.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO part SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/part.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO supplier SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/supplier.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO partsupp SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/partsupp.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO customer SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/customer.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO orders SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/orders.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+INSERT INTO lineitem SELECT * FROM s3('https://clickhouse-datasets.s3.amazonaws.com/h/100/lineitem.tbl.gz', NOSIGN, CSV) SETTINGS format_csv_delimiter = '|', input_format_defaults_for_omitted_fields = 1, input_format_csv_empty_as_default = 1;
+````
+:::
 
 ## Queries
 
+The queries are generated by `./qgen -s <scaling_factor>`. Example queries for `s = 100`:
+
 **Correctness**
 
 The result of the queries agrees with the official results unless mentioned otherwise. To verify, generate a TPC-H database with scale

docs/en/operations/server-configuration-parameters/settings.md

@@ -131,16 +131,6 @@ Type: UInt64
 
 Default: 8
 
-## background_pool_size
-
-Sets the number of threads performing background merges and mutations for tables with MergeTree engines. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance.
-
-Before changing it, please also take a look at related MergeTree settings, such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation`.
-
-Type: UInt64
-
-Default: 16
-
 ## background_schedule_pool_size
 
 The maximum number of threads that will be used for constantly executing some lightweight periodic operations for replicated tables, Kafka streaming, and DNS cache updates.

docs/en/sql-reference/aggregate-functions/reference/index.md

@@ -7,119 +7,4 @@ toc_hidden: true
 
 # List of Aggregate Functions
 
-Standard aggregate functions:
-
-- [count](../reference/count.md)
-- [min](../reference/min.md)
-- [max](../reference/max.md)
-- [sum](../reference/sum.md)
-- [avg](../reference/avg.md)
-- [any](../reference/any.md)
-- [stddevPop](../reference/stddevpop.md)
-- [stddevPopStable](../reference/stddevpopstable.md)
-- [stddevSamp](../reference/stddevsamp.md)
-- [stddevSampStable](../reference/stddevsampstable.md)
-- [varPop](../reference/varpop.md)
-- [varSamp](../reference/varsamp.md)
-- [corr](../reference/corr.md)
-- [corr](../reference/corrstable.md)
-- [corrMatrix](../reference/corrmatrix.md)
-- [covarPop](../reference/covarpop.md)
-- [covarStable](../reference/covarpopstable.md)
-- [covarPopMatrix](../reference/covarpopmatrix.md)
-- [covarSamp](../reference/covarsamp.md)
-- [covarSampStable](../reference/covarsampstable.md)
-- [covarSampMatrix](../reference/covarsampmatrix.md)
-- [entropy](../reference/entropy.md)
-- [exponentialMovingAverage](../reference/exponentialmovingaverage.md)
-- [intervalLengthSum](../reference/intervalLengthSum.md)
-- [kolmogorovSmirnovTest](../reference/kolmogorovsmirnovtest.md)
-- [mannwhitneyutest](../reference/mannwhitneyutest.md)
-- [median](../reference/median.md)
-- [rankCorr](../reference/rankCorr.md)
-- [sumKahan](../reference/sumkahan.md)
-- [studentTTest](../reference/studentttest.md)
-- [welchTTest](../reference/welchttest.md)
-
-ClickHouse-specific aggregate functions:
-
-- [aggThrow](../reference/aggthrow.md)
-- [analysisOfVariance](../reference/analysis_of_variance.md)
-- [any](../reference/any.md)
-- [anyHeavy](../reference/anyheavy.md)
-- [anyLast](../reference/anylast.md)
-- [boundingRatio](../reference/boundrat.md)
-- [first_value](../reference/first_value.md)
-- [last_value](../reference/last_value.md)
-- [argMin](../reference/argmin.md)
-- [argMax](../reference/argmax.md)
-- [avgWeighted](../reference/avgweighted.md)
-- [topK](../reference/topk.md)
-- [topKWeighted](../reference/topkweighted.md)
-- [deltaSum](../reference/deltasum.md)
-- [deltaSumTimestamp](../reference/deltasumtimestamp.md)
-- [flameGraph](../reference/flame_graph.md)
-- [groupArray](../reference/grouparray.md)
-- [groupArrayLast](../reference/grouparraylast.md)
-- [groupUniqArray](../reference/groupuniqarray.md)
-- [groupArrayInsertAt](../reference/grouparrayinsertat.md)
-- [groupArrayMovingAvg](../reference/grouparraymovingavg.md)
-- [groupArrayMovingSum](../reference/grouparraymovingsum.md)
-- [groupArraySample](../reference/grouparraysample.md)
-- [groupArraySorted](../reference/grouparraysorted.md)
-- [groupArrayIntersect](../reference/grouparrayintersect.md)
-- [groupBitAnd](../reference/groupbitand.md)
-- [groupBitOr](../reference/groupbitor.md)
-- [groupBitXor](../reference/groupbitxor.md)
-- [groupBitmap](../reference/groupbitmap.md)
-- [groupBitmapAnd](../reference/groupbitmapand.md)
-- [groupBitmapOr](../reference/groupbitmapor.md)
-- [groupBitmapXor](../reference/groupbitmapxor.md)
-- [sumWithOverflow](../reference/sumwithoverflow.md)
-- [sumMap](../reference/summap.md)
-- [sumMapWithOverflow](../reference/summapwithoverflow.md)
-- [sumMapFiltered](../parametric-functions.md/#summapfiltered)
-- [sumMapFilteredWithOverflow](../parametric-functions.md/#summapfilteredwithoverflow)
-- [minMap](../reference/minmap.md)
-- [maxMap](../reference/maxmap.md)
-- [skewSamp](../reference/skewsamp.md)
-- [skewPop](../reference/skewpop.md)
-- [kurtSamp](../reference/kurtsamp.md)
-- [kurtPop](../reference/kurtpop.md)
-- [uniq](../reference/uniq.md)
-- [uniqExact](../reference/uniqexact.md)
-- [uniqCombined](../reference/uniqcombined.md)
-- [uniqCombined64](../reference/uniqcombined64.md)
-- [uniqHLL12](../reference/uniqhll12.md)
-- [uniqTheta](../reference/uniqthetasketch.md)
-- [quantile](../reference/quantile.md)
-- [quantiles](../reference/quantiles.md)
-- [quantileExact](../reference/quantileexact.md)
-- [quantileExactLow](../reference/quantileexact.md#quantileexactlow)
-- [quantileExactHigh](../reference/quantileexact.md#quantileexacthigh)
-- [quantileExactWeighted](../reference/quantileexactweighted.md)
-- [quantileTiming](../reference/quantiletiming.md)
-- [quantileTimingWeighted](../reference/quantiletimingweighted.md)
-- [quantileDeterministic](../reference/quantiledeterministic.md)
-- [quantileTDigest](../reference/quantiletdigest.md)
-- [quantileTDigestWeighted](../reference/quantiletdigestweighted.md)
-- [quantileBFloat16](../reference/quantilebfloat16.md#quantilebfloat16)
-- [quantileBFloat16Weighted](../reference/quantilebfloat16.md#quantilebfloat16weighted)
-- [quantileDD](../reference/quantileddsketch.md#quantileddsketch)
-- [simpleLinearRegression](../reference/simplelinearregression.md)
-- [singleValueOrNull](../reference/singlevalueornull.md)
-- [stochasticLinearRegression](../reference/stochasticlinearregression.md)
-- [stochasticLogisticRegression](../reference/stochasticlogisticregression.md)
-- [categoricalInformationValue](../reference/categoricalinformationvalue.md)
-- [contingency](../reference/contingency.md)
-- [cramersV](../reference/cramersv.md)
-- [cramersVBiasCorrected](../reference/cramersvbiascorrected.md)
-- [theilsU](../reference/theilsu.md)
-- [maxIntersections](../reference/maxintersections.md)
-- [maxIntersectionsPosition](../reference/maxintersectionsposition.md)
-- [meanZTest](../reference/meanztest.md)
-- [quantileGK](../reference/quantileGK.md)
-- [quantileInterpolatedWeighted](../reference/quantileinterpolatedweighted.md)
-- [sparkBar](../reference/sparkbar.md)
-- [sumCount](../reference/sumcount.md)
-- [largestTriangleThreeBuckets](../reference/largestTriangleThreeBuckets.md)
+ClickHouse supports all standard SQL aggregate functions ([sum](../reference/sum.md), [avg](../reference/avg.md), [min](../reference/min.md), [max](../reference/max.md), [count](../reference/count.md)), as well as a wide range of other aggregate functions.

docs/en/sql-reference/data-types/aggregatefunction.md

@@ -6,7 +6,9 @@ sidebar_label: AggregateFunction
 
 # AggregateFunction
 
-Aggregate functions can have an implementation-defined intermediate state that can be serialized to an `AggregateFunction(...)` data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/create/view.md). The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix. To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix.
+Aggregate functions have an implementation-defined intermediate state that can be serialized to an `AggregateFunction(...)` data type and stored in a table, usually, by means of [a materialized view](../../sql-reference/statements/create/view.md).
+The common way to produce an aggregate function state is by calling the aggregate function with the `-State` suffix.
+To get the final result of aggregation in the future, you must use the same aggregate function with the `-Merge`suffix.
 
 `AggregateFunction(name, types_of_arguments...)` — parametric data type.
 

docs/en/sql-reference/data-types/index.md

@@ -6,29 +6,8 @@ sidebar_position: 1
 
 # Data Types in ClickHouse
 
-ClickHouse can store various kinds of data in table cells. This section describes the supported data types and special considerations for using and/or implementing them if any.
+This section describes the data types supported by ClickHouse, for example [integers](int-uint.md), [floats](float.md) and [strings](string.md).
 
-:::note
-You can check whether a data type name is case-sensitive in the [system.data_type_families](../../operations/system-tables/data_type_families.md#system_tables-data_type_families) table.
-:::
-
-ClickHouse data types include:
-
-- **Integer types**: [signed and unsigned integers](./int-uint.md) (`UInt8`, `UInt16`, `UInt32`, `UInt64`, `UInt128`, `UInt256`, `Int8`, `Int16`, `Int32`, `Int64`, `Int128`, `Int256`)
-- **Floating-point numbers**: [floats](./float.md)(`Float32` and `Float64`) and [`Decimal` values](./decimal.md)
-- **Boolean**: ClickHouse has a [`Boolean` type](./boolean.md)
-- **Strings**: [`String`](./string.md) and [`FixedString`](./fixedstring.md)
-- **Dates**: use [`Date`](./date.md) and [`Date32`](./date32.md) for days, and [`DateTime`](./datetime.md) and [`DateTime64`](./datetime64.md) for instances in time
-- **Object**: the [`Object`](./json.md) stores a JSON document in a single column (deprecated)
-- **JSON**: the [`JSON` object](./newjson.md) stores a JSON document in a single column
-- **UUID**: a performant option for storing [`UUID` values](./uuid.md)
-- **Low cardinality types**: use an [`Enum`](./enum.md) when you have a handful of unique values, or use [`LowCardinality`](./lowcardinality.md) when you have up to 10,000 unique values of a column
-- **Arrays**: any column can be defined as an [`Array` of values](./array.md)
-- **Maps**: use [`Map`](./map.md) for storing key/value pairs
-- **Aggregation function types**: use [`SimpleAggregateFunction`](./simpleaggregatefunction.md) and [`AggregateFunction`](./aggregatefunction.md) for storing the intermediate status of aggregate function results
-- **Nested data structures**: A [`Nested` data structure](./nested-data-structures/index.md) is like a table inside a cell
-- **Tuples**: A [`Tuple` of elements](./tuple.md), each having an individual type.
-- **Nullable**: [`Nullable`](./nullable.md) allows you to store a value as `NULL` when a value is "missing" (instead of the column settings its default value for the data type)
-- **IP addresses**: use [`IPv4`](./ipv4.md) and [`IPv6`](./ipv6.md) to efficiently store IP addresses
-- **Geo types**: for [geographical data](./geo.md), including `Point`, `Ring`, `Polygon` and `MultiPolygon`
-- **Special data types**: including [`Expression`](./special-data-types/expression.md), [`Set`](./special-data-types/set.md), [`Nothing`](./special-data-types/nothing.md) and [`Interval`](./special-data-types/interval.md)
+System table [system.data_type_families](../../operations/system-tables/data_type_families.md#system_tables-data_type_families) provides an
+overview of all available data types.
+It also shows whether a data type is an alias to another data type and its name is case-sensitive (e.g. `bool` vs. `BOOL`).

docs/en/sql-reference/data-types/json.md

@@ -7,7 +7,7 @@ keywords: [object, data type]
 
 # Object Data Type (deprecated)
 
-**This feature is not production-ready and is now deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-formats/json/overview) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864).
+**This feature is not production-ready and deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-formats/json/overview) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864).
 
 <hr />
 

docs/en/sql-reference/data-types/simpleaggregatefunction.md

@@ -5,7 +5,9 @@ sidebar_label: SimpleAggregateFunction
 ---
 # SimpleAggregateFunction
 
-`SimpleAggregateFunction(name, types_of_arguments...)` data type stores current value of the aggregate function, and does not store its full state as [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) does. This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. This property guarantees that partial aggregation results are enough to compute the combined one, so we do not have to store and process any extra data.
+`SimpleAggregateFunction(name, types_of_arguments...)` data type stores current value (intermediate state) of the aggregate function, but not its full state as [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) does.
+This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`.
+This property guarantees that partial aggregation results are enough to compute the combined one, so we do not have to store and process any extra data.
 
 The common way to produce an aggregate function value is by calling the aggregate function with the [-SimpleState](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-simplestate) suffix.
 

docs/en/sql-reference/functions/geo/index.md

@@ -5,70 +5,4 @@ sidebar_position: 62
 title: "Geo Functions"
 ---
 
-
-## Geographical Coordinates Functions
-
-- [greatCircleDistance](./coordinates.md#greatcircledistance)
-- [geoDistance](./coordinates.md#geodistance)
-- [greatCircleAngle](./coordinates.md#greatcircleangle)
-- [pointInEllipses](./coordinates.md#pointinellipses)
-- [pointInPolygon](./coordinates.md#pointinpolygon)
-
-## Geohash Functions
-- [geohashEncode](./geohash.md#geohashencode)
-- [geohashDecode](./geohash.md#geohashdecode)
-- [geohashesInBox](./geohash.md#geohashesinbox)
-
-## H3 Indexes Functions
-
-- [h3IsValid](./h3.md#h3isvalid)
-- [h3GetResolution](./h3.md#h3getresolution)
-- [h3EdgeAngle](./h3.md#h3edgeangle)
-- [h3EdgeLengthM](./h3.md#h3edgelengthm)
-- [h3EdgeLengthKm](./h3.md#h3edgelengthkm)
-- [geoToH3](./h3.md#geotoh3)
-- [h3ToGeo](./h3.md#h3togeo)
-- [h3ToGeoBoundary](./h3.md#h3togeoboundary)
-- [h3kRing](./h3.md#h3kring)
-- [h3GetBaseCell](./h3.md#h3getbasecell)
-- [h3HexAreaM2](./h3.md#h3hexaream2)
-- [h3HexAreaKm2](./h3.md#h3hexareakm2)
-- [h3IndexesAreNeighbors](./h3.md#h3indexesareneighbors)
-- [h3ToChildren](./h3.md#h3tochildren)
-- [h3ToParent](./h3.md#h3toparent)
-- [h3ToString](./h3.md#h3tostring)
-- [stringToH3](./h3.md#stringtoh3)
-- [h3GetResolution](./h3.md#h3getresolution)
-- [h3IsResClassIII](./h3.md#h3isresclassiii)
-- [h3IsPentagon](./h3.md#h3ispentagon)
-- [h3GetFaces](./h3.md#h3getfaces)
-- [h3CellAreaM2](./h3.md#h3cellaream2)
-- [h3CellAreaRads2](./h3.md#h3cellarearads2)
-- [h3ToCenterChild](./h3.md#h3tocenterchild)
-- [h3ExactEdgeLengthM](./h3.md#h3exactedgelengthm)
-- [h3ExactEdgeLengthKm](./h3.md#h3exactedgelengthkm)
-- [h3ExactEdgeLengthRads](./h3.md#h3exactedgelengthrads)
-- [h3NumHexagons](./h3.md#h3numhexagons)
-- [h3Line](./h3.md#h3line)
-- [h3Distance](./h3.md#h3distance)
-- [h3HexRing](./h3.md#h3hexring)
-- [h3GetUnidirectionalEdge](./h3.md#h3getunidirectionaledge)
-- [h3UnidirectionalEdgeIsValid](./h3.md#h3unidirectionaledgeisvalid)
-- [h3GetOriginIndexFromUnidirectionalEdge](./h3.md#h3getoriginindexfromunidirectionaledge)
-- [h3GetDestinationIndexFromUnidirectionalEdge](./h3.md#h3getdestinationindexfromunidirectionaledge)
-- [h3GetIndexesFromUnidirectionalEdge](./h3.md#h3getindexesfromunidirectionaledge)
-- [h3GetUnidirectionalEdgesFromHexagon](./h3.md#h3getunidirectionaledgesfromhexagon)
-- [h3GetUnidirectionalEdgeBoundary](./h3.md#h3getunidirectionaledgeboundary)
-
-## S2 Index Functions
-
-- [geoToS2](./s2.md#geotos2)
-- [s2ToGeo](./s2.md#s2togeo)
-- [s2GetNeighbors](./s2.md#s2getneighbors)
-- [s2CellsIntersect](./s2.md#s2cellsintersect)
-- [s2CapContains](./s2.md#s2capcontains)
-- [s2CapUnion](./s2.md#s2capunion)
-- [s2RectAdd](./s2.md#s2rectadd)
-- [s2RectContains](./s2.md#s2rectcontains)
-- [s2RectUnion](./s2.md#s2rectunion)
-- [s2RectIntersection](./s2.md#s2rectintersection)
+Functions for working with geometric objects, for example [to calculate distances between points on a sphere](./coordinates.md), [compute geohashes](./geohash.md), and work with [h3 indexes](./h3.md).

docs/en/sql-reference/functions/index.md

@@ -24,7 +24,7 @@ All expressions in a query that have the same AST (the same record or same resul
 
 ## Types of Results
 
-All functions return a single return as the result (not several values, and not zero values). The type of result is usually defined only by the types of arguments, not by the values. Exceptions are the tupleElement function (the a.N operator), and the toFixedString function.
+All functions return a single value as the result (not several values, and not zero values). The type of result is usually defined only by the types of arguments, not by the values. Exceptions are the tupleElement function (the a.N operator), and the toFixedString function.
 
 ## Constants
 

docs/en/sql-reference/statements/create/index.md

@@ -6,16 +6,4 @@ sidebar_label: CREATE
 
 # CREATE Queries
 
-Create queries make a new entity of one of the following kinds:
-
-- [DATABASE](/docs/en/sql-reference/statements/create/database.md)
-- [TABLE](/docs/en/sql-reference/statements/create/table.md)
-- [VIEW](/docs/en/sql-reference/statements/create/view.md)
-- [DICTIONARY](/docs/en/sql-reference/statements/create/dictionary.md)
-- [FUNCTION](/docs/en/sql-reference/statements/create/function.md)
-- [USER](/docs/en/sql-reference/statements/create/user.md)
-- [ROLE](/docs/en/sql-reference/statements/create/role.md)
-- [ROW POLICY](/docs/en/sql-reference/statements/create/row-policy.md)
-- [QUOTA](/docs/en/sql-reference/statements/create/quota.md)
-- [SETTINGS PROFILE](/docs/en/sql-reference/statements/create/settings-profile.md)
-- [NAMED COLLECTION](/docs/en/sql-reference/statements/create/named-collection.md)
+CREATE queries create (for example) new [databases](/docs/en/sql-reference/statements/create/database.md), [tables](/docs/en/sql-reference/statements/create/table.md) and [views](/docs/en/sql-reference/statements/create/view.md).

docs/en/sql-reference/statements/index.md

@@ -6,27 +6,4 @@ sidebar_label: List of statements
 
 # ClickHouse SQL Statements
 
-Statements represent various kinds of action you can perform using SQL queries. Each kind of statement has it’s own syntax and usage details that are described separately:
-
-- [SELECT](/docs/en/sql-reference/statements/select/index.md)
-- [INSERT INTO](/docs/en/sql-reference/statements/insert-into.md)
-- [CREATE](/docs/en/sql-reference/statements/create/index.md)
-- [ALTER](/docs/en/sql-reference/statements/alter/index.md)
-- [SYSTEM](/docs/en/sql-reference/statements/system.md)
-- [SHOW](/docs/en/sql-reference/statements/show.md)
-- [GRANT](/docs/en/sql-reference/statements/grant.md)
-- [REVOKE](/docs/en/sql-reference/statements/revoke.md)
-- [ATTACH](/docs/en/sql-reference/statements/attach.md)
-- [CHECK TABLE](/docs/en/sql-reference/statements/check-table.md)
-- [DESCRIBE TABLE](/docs/en/sql-reference/statements/describe-table.md)
-- [DETACH](/docs/en/sql-reference/statements/detach.md)
-- [DROP](/docs/en/sql-reference/statements/drop.md)
-- [EXISTS](/docs/en/sql-reference/statements/exists.md)
-- [KILL](/docs/en/sql-reference/statements/kill.md)
-- [OPTIMIZE](/docs/en/sql-reference/statements/optimize.md)
-- [RENAME](/docs/en/sql-reference/statements/rename.md)
-- [SET](/docs/en/sql-reference/statements/set.md)
-- [SET ROLE](/docs/en/sql-reference/statements/set-role.md)
-- [TRUNCATE](/docs/en/sql-reference/statements/truncate.md)
-- [USE](/docs/en/sql-reference/statements/use.md)
-- [EXPLAIN](/docs/en/sql-reference/statements/explain.md)
+Users interact with ClickHouse using SQL statements. ClickHouse supports common SQL statements like [SELECT](select/index.md) and [CREATE](create/index.md), but it also provides specialized statements like [KILL](kill.md) and [OPTIMIZE](optimize.md).

programs/client/Client.cpp

@@ -431,7 +431,7 @@ catch (const Exception & e)
     bool need_print_stack_trace = config().getBool("stacktrace", false) && e.code() != ErrorCodes::NETWORK_ERROR;
     std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl << std::endl;
     /// If exception code isn't zero, we should return non-zero return code anyway.
-    return e.code() ? e.code() : -1;
+    return static_cast<UInt8>(e.code()) ? e.code() : -1;
 }
 catch (...)
 {
@@ -1390,7 +1390,8 @@ int mainEntryClickHouseClient(int argc, char ** argv)
     catch (const DB::Exception & e)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
-        return 1;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
     catch (const boost::program_options::error & e)
     {
@@ -1399,7 +1400,8 @@ int mainEntryClickHouseClient(int argc, char ** argv)
     }
     catch (...)
     {
-        std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
-        return 1;
+        std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }

programs/compressor/Compressor.cpp

@@ -9,6 +9,7 @@
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadBufferFromFile.h>
 #include <Compression/CompressedWriteBuffer.h>
+#include <Compression/ParallelCompressedWriteBuffer.h>
 #include <Compression/CompressedReadBuffer.h>
 #include <Compression/CompressedReadBufferFromFile.h>
 #include <IO/WriteHelpers.h>
@@ -17,6 +18,8 @@
 #include <Parsers/ExpressionElementParsers.h>
 #include <Compression/CompressionFactory.h>
 #include <Common/TerminalSize.h>
+#include <Common/ThreadPool.h>
+#include <Common/CurrentMetrics.h>
 #include <Core/Defines.h>
 
 
@@ -29,6 +32,13 @@ namespace DB
     }
 }
 
+namespace CurrentMetrics
+{
+    extern const Metric LocalThread;
+    extern const Metric LocalThreadActive;
+    extern const Metric LocalThreadScheduled;
+}
+
 
 namespace
 {
@@ -77,11 +87,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
             ("decompress,d", "decompress")
             ("offset-in-compressed-file", po::value<size_t>()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)")
             ("offset-in-decompressed-block", po::value<size_t>()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)")
-            ("block-size,b", po::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
+            ("block-size,b", po::value<size_t>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
             ("hc", "use LZ4HC instead of LZ4")
             ("zstd", "use ZSTD instead of LZ4")
             ("codec", po::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
             ("level", po::value<int>(), "compression level for codecs specified via flags")
+            ("threads", po::value<size_t>()->default_value(1), "number of threads for parallel compression")
             ("none", "use no compression instead of LZ4")
             ("stat", "print block statistics of compressed data")
             ("stacktrace", "print stacktrace of exception")
@@ -109,7 +120,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
         bool stat_mode = options.count("stat");
         bool use_none = options.count("none");
         print_stacktrace = options.count("stacktrace");
-        unsigned block_size = options["block-size"].as<unsigned>();
+        size_t block_size = options["block-size"].as<size_t>();
+        size_t num_threads = options["threads"].as<size_t>();
         std::vector<std::string> codecs;
         if (options.count("codec"))
             codecs = options["codec"].as<std::vector<std::string>>();
@@ -117,6 +129,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
         if ((use_lz4hc || use_zstd || use_none) && !codecs.empty())
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong options, codec flags like --zstd and --codec options are mutually exclusive");
 
+        if (num_threads < 1)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid value of `threads` parameter");
+
+        if (num_threads > 1 && decompress)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Parallel mode is only implemented for compression (not for decompression)");
+
         if (!codecs.empty() && options.count("level"))
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong options, --level is not compatible with --codec list");
 
@@ -145,7 +163,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
         else
             codec = CompressionCodecFactory::instance().get(method_family, level);
 
-
         std::unique_ptr<ReadBufferFromFileBase> rb;
         std::unique_ptr<WriteBufferFromFileBase> wb;
 
@@ -186,9 +203,20 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
         else
         {
             /// Compression
-            CompressedWriteBuffer to(*wb, codec, block_size);
-            copyData(*rb, to);
-            to.finalize();
+
+            if (num_threads == 1)
+            {
+                CompressedWriteBuffer to(*wb, codec, block_size);
+                copyData(*rb, to);
+                to.finalize();
+            }
+            else
+            {
+                ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads);
+                ParallelCompressedWriteBuffer to(*wb, codec, block_size, num_threads, pool);
+                copyData(*rb, to);
+                to.finalize();
+            }
         }
     }
     catch (...)

programs/disks/DisksApp.cpp

@@ -546,16 +546,18 @@ int mainEntryClickHouseDisks(int argc, char ** argv)
     catch (const DB::Exception & e)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
-        return 0;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
     catch (const boost::program_options::error & e)
     {
         std::cerr << "Bad arguments: " << e.what() << std::endl;
-        return 0;
+        return DB::ErrorCodes::BAD_ARGUMENTS;
     }
     catch (...)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
-        return 0;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }

programs/keeper-client/KeeperClient.cpp

@@ -448,7 +448,8 @@ int mainEntryClickHouseKeeperClient(int argc, char ** argv)
     catch (const DB::Exception & e)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
-        return 1;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
     catch (const boost::program_options::error & e)
     {
@@ -458,6 +459,7 @@ int mainEntryClickHouseKeeperClient(int argc, char ** argv)
     catch (...)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
-        return 1;
+        auto code = DB::getCurrentExceptionCode();
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }

programs/keeper/Keeper.cpp

@@ -81,7 +81,7 @@ int mainEntryClickHouseKeeper(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }
 
@@ -672,7 +672,7 @@ catch (...)
     /// Poco does not provide stacktrace.
     tryLogCurrentException("Application");
     auto code = getCurrentExceptionCode();
-    return code ? code : -1;
+    return static_cast<UInt8>(code) ? code : -1;
 }
 
 

programs/library-bridge/LibraryBridge.cpp

@@ -13,7 +13,7 @@ int mainEntryClickHouseLibraryBridge(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }
 

programs/local/LocalServer.cpp

@@ -22,7 +22,6 @@
 #include <Interpreters/ProcessList.h>
 #include <Interpreters/loadMetadata.h>
 #include <Interpreters/registerInterpreters.h>
-#include <base/getFQDNOrHostName.h>
 #include <Access/AccessControl.h>
 #include <Common/PoolId.h>
 #include <Common/Exception.h>
@@ -31,7 +30,6 @@
 #include <Common/ThreadStatus.h>
 #include <Common/TLDListsHolder.h>
 #include <Common/quoteString.h>
-#include <Common/randomSeed.h>
 #include <Common/ThreadPool.h>
 #include <Common/CurrentMetrics.h>
 #include <Loggers/OwnFormattingChannel.h>
@@ -50,7 +48,6 @@
 #include <Dictionaries/registerDictionaries.h>
 #include <Disks/registerDisks.h>
 #include <Formats/registerFormats.h>
-#include <boost/algorithm/string/replace.hpp>
 #include <boost/program_options/options_description.hpp>
 #include <base/argsToConfig.h>
 #include <filesystem>
@@ -71,9 +68,11 @@ namespace CurrentMetrics
 
 namespace DB
 {
+
 namespace Setting
 {
     extern const SettingsBool allow_introspection_functions;
+    extern const SettingsBool implicit_select;
     extern const SettingsLocalFSReadMethod storage_file_read_method;
 }
 
@@ -126,6 +125,7 @@ void applySettingsOverridesForLocal(ContextMutablePtr context)
 
     settings[Setting::allow_introspection_functions] = true;
     settings[Setting::storage_file_read_method] = LocalFSReadMethod::mmap;
+    settings[Setting::implicit_select] = true;
 
     context->setSettings(settings);
 }
@@ -615,12 +615,14 @@ catch (const DB::Exception & e)
 {
     bool need_print_stack_trace = getClientConfiguration().getBool("stacktrace", false);
     std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl;
-    return e.code() ? e.code() : -1;
+    auto code = DB::getCurrentExceptionCode();
+    return static_cast<UInt8>(code) ? code : 1;
 }
 catch (...)
 {
-    std::cerr << getCurrentExceptionMessage(false) << std::endl;
-    return getCurrentExceptionCode();
+    std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
+    auto code = DB::getCurrentExceptionCode();
+    return static_cast<UInt8>(code) ? code : 1;
 }
 
 void LocalServer::updateLoggerLevel(const String & logs_level)
@@ -1029,7 +1031,7 @@ int mainEntryClickHouseLocal(int argc, char ** argv)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
     catch (const boost::program_options::error & e)
     {
@@ -1040,6 +1042,6 @@ int mainEntryClickHouseLocal(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }

programs/obfuscator/Obfuscator.cpp

@@ -1480,5 +1480,5 @@ catch (...)
 {
     std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
     auto code = DB::getCurrentExceptionCode();
-    return code ? code : 1;
+    return static_cast<UInt8>(code) ? code : 1;
 }

programs/odbc-bridge/ODBCBridge.cpp

@@ -13,7 +13,7 @@ int mainEntryClickHouseODBCBridge(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }
 

programs/server/Server.cpp

@@ -343,7 +343,7 @@ int mainEntryClickHouseServer(int argc, char ** argv)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
         auto code = DB::getCurrentExceptionCode();
-        return code ? code : 1;
+        return static_cast<UInt8>(code) ? code : 1;
     }
 }
 
@@ -2537,7 +2537,7 @@ catch (...)
     /// Poco does not provide stacktrace.
     tryLogCurrentException("Application");
     auto code = getCurrentExceptionCode();
-    return code ? code : -1;
+    return static_cast<UInt8>(code) ? code : -1;
 }
 
 std::unique_ptr<TCPProtocolStackFactory> Server::buildProtocolStackFromConfig(

programs/su/su.cpp

@@ -59,7 +59,13 @@ void setUserAndGroup(std::string arg_uid, std::string arg_gid)
                 throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid);
 
             if (!result)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid);
+            {
+                if (0 != getgrgid_r(gid, &entry, buf.get(), buf_size, &result))
+                    throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid);
+
+                if (!result)
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid);
+            }
 
             gid = entry.gr_gid;
         }
@@ -84,7 +90,13 @@ void setUserAndGroup(std::string arg_uid, std::string arg_gid)
                 throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid);
 
             if (!result)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid);
+            {
+                if (0 != getpwuid_r(uid, &entry, buf.get(), buf_size, &result))
+                    throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot do 'getpwuid_r' to obtain uid from user name ({})", uid);
+
+                if (!result)
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid);
+            }
 
             uid = entry.pw_uid;
         }

src/Analyzer/Resolve/QueryAnalyzer.cpp

@@ -51,7 +51,6 @@
 #include <Analyzer/ArrayJoinNode.h>
 #include <Analyzer/JoinNode.h>
 #include <Analyzer/UnionNode.h>
-#include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/QueryTreeBuilder.h>
 #include <Analyzer/IQueryTreeNode.h>
 #include <Analyzer/Identifier.h>
@@ -3023,9 +3022,10 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
         argument_column.name = arguments_projection_names[function_argument_index];
 
         /** If function argument is lambda, save lambda argument index and initialize argument type as DataTypeFunction
-          * where function argument types are initialized with empty array of lambda arguments size.
+          * where function argument types are initialized with empty arrays of lambda arguments size.
           */
-        if (const auto * lambda_node = function_argument->as<const LambdaNode>())
+        const auto * lambda_node = function_argument->as<const LambdaNode>();
+        if (lambda_node)
         {
             size_t lambda_arguments_size = lambda_node->getArguments().getNodes().size();
             argument_column.type = std::make_shared<DataTypeFunction>(DataTypes(lambda_arguments_size, nullptr), nullptr);
@@ -3497,15 +3497,11 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
         else
             function_base = function->build(argument_columns);
 
-        /// Do not constant fold get scalar functions
-        // bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" ||
-        //     function_name == "shardCount" || function_name == "hostName" || function_name == "tcpPort";
-
         /** If function is suitable for constant folding try to convert it to constant.
           * Example: SELECT plus(1, 1);
           * Result: SELECT 2;
           */
-        if (function_base->isSuitableForConstantFolding()) // && !disable_constant_folding)
+        if (function_base->isSuitableForConstantFolding())
         {
             auto result_type = function_base->getResultType();
             auto executable_function = function_base->prepare(argument_columns);
@@ -3514,7 +3510,9 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
 
             if (all_arguments_constants)
             {
-                size_t num_rows = function_arguments.empty() ? 0 : argument_columns.front().column->size();
+                size_t num_rows = 0;
+                if (!argument_columns.empty())
+                    num_rows = argument_columns.front().column->size();
                 column = executable_function->execute(argument_columns, result_type, num_rows, true);
             }
             else

src/Client/ClientBase.cpp

@@ -1650,6 +1650,11 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
     if (!parsed_insert_query)
         return;
 
+    /// If it's clickhouse-local, and the input data reading is already baked into the query pipeline,
+    /// don't read the data again here. This happens in some cases (e.g. input() table function) but not others (e.g. INFILE).
+    if (!connection->isSendDataNeeded())
+        return;
+
     bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && isStdinNotEmptyAndValid(std_in);
 
     if (need_render_progress)
@@ -2674,7 +2679,10 @@ void ClientBase::runInteractive()
 #if USE_REPLXX
     replxx::Replxx::highlighter_callback_t highlight_callback{};
     if (getClientConfiguration().getBool("highlight", true))
-        highlight_callback = highlight;
+        highlight_callback = [this](const String & query, std::vector<replxx::Replxx::Color> & colors)
+        {
+            highlight(query, colors, *client_context);
+        };
 
     ReplxxLineReader lr(
         *suggest,

src/Client/ClientBaseHelpers.cpp

@@ -5,6 +5,8 @@
 #include <Parsers/ParserQuery.h>
 #include <Parsers/parseQuery.h>
 #include <Common/UTF8Helpers.h>
+#include <Core/Settings.h>
+#include <Interpreters/Context.h>
 
 #include <iostream>
 
@@ -12,6 +14,11 @@
 namespace DB
 {
 
+namespace Setting
+{
+    extern const SettingsBool implicit_select;
+}
+
 /// Should we celebrate a bit?
 bool isNewYearMode()
 {
@@ -95,7 +102,7 @@ bool isChineseNewYearMode(const String & local_tz)
 }
 
 #if USE_REPLXX
-void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors)
+void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors, const Context & context)
 {
     using namespace replxx;
 
@@ -135,13 +142,27 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
 
     /// Currently we highlight only the first query in the multi-query mode.
 
-    ParserQuery parser(end);
+    ParserQuery parser(end, false, context.getSettingsRef()[Setting::implicit_select]);
     ASTPtr ast;
     bool parse_res = false;
 
     try
     {
-        parse_res = parser.parse(token_iterator, ast, expected);
+        while (!token_iterator->isEnd())
+        {
+            parse_res = parser.parse(token_iterator, ast, expected);
+            if (!parse_res)
+                break;
+
+            if (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon)
+            {
+                parse_res = false;
+                break;
+            }
+
+            while (token_iterator->type == TokenType::Semicolon)
+                ++token_iterator;
+        }
     }
     catch (...)
     {
@@ -175,7 +196,7 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
 
     /// Highlight the last error in red. If the parser failed or the lexer found an invalid token,
     /// or if it didn't parse all the data (except, the data for INSERT query, which is legitimately unparsed)
-    if ((!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon))
+    if ((!parse_res || last_token.isError())
         && !(insert_data && expected.max_parsed_pos >= insert_data)
         && expected.max_parsed_pos >= prev)
     {

src/Client/ClientBaseHelpers.h

@@ -11,13 +11,15 @@
 namespace DB
 {
 
+class Context;
+
 /// Should we celebrate a bit?
 bool isNewYearMode();
 
 bool isChineseNewYearMode(const String & local_tz);
 
 #if USE_REPLXX
-void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors);
+void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors, const Context & context);
 #endif
 
 }

src/Client/IServerConnection.h

@@ -109,6 +109,10 @@ public:
     /// Send block of data; if name is specified, server will write it to external (temporary) table of that name.
     virtual void sendData(const Block & block, const String & name, bool scalar) = 0;
 
+    /// Whether the client needs to read and send the data for the INSERT.
+    /// False if the server will read the data through other means (in particular if clickhouse-local added input reading step directly into the query pipeline).
+    virtual bool isSendDataNeeded() const { return true; }
+
     /// Send all contents of external (temporary) tables.
     virtual void sendExternalTablesData(ExternalTablesData & data) = 0;
 

src/Client/LocalConnection.cpp

@@ -328,6 +328,11 @@ void LocalConnection::sendData(const Block & block, const String &, bool)
         sendProfileEvents();
 }
 
+bool LocalConnection::isSendDataNeeded() const
+{
+    return !state || state->input_pipeline == nullptr;
+}
+
 void LocalConnection::sendCancel()
 {
     state->is_cancelled = true;

src/Client/LocalConnection.h

@@ -120,6 +120,8 @@ public:
 
     void sendData(const Block & block, const String & name/* = "" */, bool scalar/* = false */) override;
 
+    bool isSendDataNeeded() const override;
+
     void sendExternalTablesData(ExternalTablesData &) override;
 
     void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) override;

src/Columns/ColumnFunction.cpp

@@ -72,6 +72,26 @@ ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
     return ColumnFunction::create(length, function, capture, is_short_circuit_argument, is_function_compiled);
 }
 
+Field ColumnFunction::operator[](size_t n) const
+{
+    Field res;
+    get(n, res);
+    return res;
+}
+
+void ColumnFunction::get(size_t n, Field & res) const
+{
+    const size_t tuple_size = captured_columns.size();
+
+    res = Tuple();
+    Tuple & res_tuple = res.safeGet<Tuple &>();
+    res_tuple.reserve(tuple_size);
+
+    for (size_t i = 0; i < tuple_size; ++i)
+        res_tuple.push_back((*captured_columns[i].column)[n]);
+}
+
+
 #if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnFunction::insertFrom(const IColumn & src, size_t n)
 #else

src/Columns/ColumnFunction.h

@@ -60,15 +60,9 @@ public:
     void appendArguments(const ColumnsWithTypeAndName & columns);
     ColumnWithTypeAndName reduce() const;
 
-    Field operator[](size_t) const override
-    {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName());
-    }
+    Field operator[](size_t n) const override;
 
-    void get(size_t, Field &) const override
-    {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName());
-    }
+    void get(size_t n, Field & res) const override;
 
     StringRef getDataAt(size_t) const override
     {

src/Common/CurrentMetrics.cpp

@@ -41,6 +41,10 @@
     M(PostgreSQLConnection, "Number of client connections using PostgreSQL protocol") \
     M(OpenFileForRead, "Number of files open for reading") \
     M(OpenFileForWrite, "Number of files open for writing") \
+    M(Compressing, "Number of compress operations using internal compression codecs") \
+    M(Decompressing, "Number of decompress operations using internal compression codecs") \
+    M(ParallelCompressedWriteBufferThreads, "Number of threads in all instances of ParallelCompressedWriteBuffer - these threads are doing parallel compression and writing") \
+    M(ParallelCompressedWriteBufferWait, "Number of threads in all instances of ParallelCompressedWriteBuffer that are currently waiting for buffer to become available for writing") \
     M(TotalTemporaryFiles, "Number of temporary files created") \
     M(TemporaryFilesForSort, "Number of temporary files created for external sorting") \
     M(TemporaryFilesForAggregation, "Number of temporary files created for external aggregation") \
@@ -99,6 +103,9 @@
     M(IOThreads, "Number of threads in the IO thread pool.") \
     M(IOThreadsActive, "Number of threads in the IO thread pool running a task.") \
     M(IOThreadsScheduled, "Number of queued or active jobs in the IO thread pool.") \
+    M(CompressionThread, "Number of threads in compression thread pools.") \
+    M(CompressionThreadActive, "Number of threads in compression thread pools running a task.") \
+    M(CompressionThreadScheduled, "Number of queued or active jobs in compression thread pools.") \
     M(ThreadPoolRemoteFSReaderThreads, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool.") \
     M(ThreadPoolRemoteFSReaderThreadsActive, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool running a task.") \
     M(ThreadPoolRemoteFSReaderThreadsScheduled, "Number of queued or active jobs in the thread pool for remote_filesystem_read_method=threadpool.") \

src/Common/ThreadPool.h

@@ -122,7 +122,7 @@ public:
     void scheduleOrThrowOnError(Job job, Priority priority = {});
 
     /// Similar to scheduleOrThrowOnError(...). Wait for specified amount of time and schedule a job or return false.
-    bool trySchedule(Job job, Priority priority = {}, uint64_t wait_microseconds = 0) noexcept;
+    [[nodiscard]] bool trySchedule(Job job, Priority priority = {}, uint64_t wait_microseconds = 0) noexcept;
 
     /// Similar to scheduleOrThrowOnError(...). Wait for specified amount of time and schedule a job or throw an exception.
     void scheduleOrThrow(Job job, Priority priority = {}, uint64_t wait_microseconds = 0, bool propagate_opentelemetry_tracing_context = true);
@@ -142,7 +142,7 @@ public:
 
     /// Returns true if the pool already terminated
     /// (and any further scheduling will produce CANNOT_SCHEDULE_TASK exception)
-    bool finished() const;
+    [[nodiscard]] bool finished() const;
 
     void setMaxThreads(size_t value);
     void setMaxFreeThreads(size_t value);

src/Compression/CompressedWriteBuffer.cpp

@@ -2,7 +2,6 @@
 #include <cstring>
 
 #include <base/types.h>
-#include <base/unaligned.h>
 #include <base/defines.h>
 
 #include <IO/WriteHelpers.h>

src/Compression/ICompressionCodec.cpp

@@ -5,11 +5,18 @@
 #include <Parsers/ASTFunction.h>
 #include <base/unaligned.h>
 #include <Common/Exception.h>
+#include <Common/CurrentMetrics.h>
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Compression/CompressionCodecMultiple.h>
 
 
+namespace CurrentMetrics
+{
+    extern const Metric Compressing;
+    extern const Metric Decompressing;
+}
+
 namespace DB
 {
 
@@ -80,6 +87,8 @@ UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char
 {
     assert(source != nullptr && dest != nullptr);
 
+    CurrentMetrics::Increment metric_increment(CurrentMetrics::Compressing);
+
     dest[0] = getMethodByte();
     UInt8 header_size = getHeaderSize();
     /// Write data from header_size
@@ -93,6 +102,8 @@ UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, ch
 {
     assert(source != nullptr && dest != nullptr);
 
+    CurrentMetrics::Increment metric_increment(CurrentMetrics::Decompressing);
+
     UInt8 header_size = getHeaderSize();
     if (source_size < header_size)
         throw Exception(decompression_error_code,

src/Compression/ParallelCompressedWriteBuffer.cpp

@@ -0,0 +1,166 @@
+#include <city.h>
+
+#include <base/types.h>
+#include <base/defines.h>
+
+#include <IO/WriteHelpers.h>
+#include <Common/setThreadName.h>
+#include <Common/scope_guard_safe.h>
+#include <Common/CurrentThread.h>
+#include <Common/CurrentMetrics.h>
+
+#include <Compression/ParallelCompressedWriteBuffer.h>
+
+
+namespace CurrentMetrics
+{
+    extern const Metric ParallelCompressedWriteBufferThreads;
+    extern const Metric ParallelCompressedWriteBufferWait;
+}
+
+namespace DB
+{
+
+ParallelCompressedWriteBuffer::ParallelCompressedWriteBuffer(
+    WriteBuffer & out_,
+    CompressionCodecPtr codec_,
+    size_t buf_size_,
+    size_t num_threads_,
+    ThreadPool & pool_)
+    : WriteBuffer(nullptr, 0), out(out_), codec(codec_), buf_size(buf_size_), num_threads(num_threads_), pool(pool_)
+{
+    buffers.emplace_back(buf_size);
+    current_buffer = buffers.begin();
+    BufferBase::set(current_buffer->uncompressed.data(), buf_size, 0);
+}
+
+void ParallelCompressedWriteBuffer::nextImpl()
+{
+    if (!offset())
+        return;
+
+    std::unique_lock lock(mutex);
+
+    /// The buffer will be compressed and processed in the thread.
+    current_buffer->busy = true;
+    current_buffer->sequence_num = current_sequence_num;
+    ++current_sequence_num;
+    current_buffer->uncompressed_size = offset();
+    pool.scheduleOrThrowOnError([this, my_current_buffer = current_buffer, thread_group = CurrentThread::getGroup()]
+    {
+        SCOPE_EXIT_SAFE(
+            if (thread_group)
+                CurrentThread::detachFromGroupIfNotDetached();
+        );
+
+        if (thread_group)
+            CurrentThread::attachToGroupIfDetached(thread_group);
+        setThreadName("ParallelCompres");
+
+        compress(my_current_buffer);
+    });
+
+    BufferPair * previous_buffer = &*current_buffer;
+    ++current_buffer;
+    if (current_buffer == buffers.end())
+    {
+        if (buffers.size() < num_threads)
+        {
+            /// If we didn't use all num_threads buffers yet, create a new one.
+            current_buffer = buffers.emplace(current_buffer, buf_size);
+        }
+        else
+        {
+            /// Otherwise, wrap around to the first buffer in the list.
+            current_buffer = buffers.begin();
+        }
+    }
+
+    /// Wait while the buffer becomes not busy
+    if (current_buffer->busy)
+    {
+        CurrentMetrics::Increment metric_increment(CurrentMetrics::ParallelCompressedWriteBufferWait);
+        cond.wait(lock, [&]{ return !current_buffer->busy; });
+    }
+
+    /// Now this buffer can be used.
+    current_buffer->previous = previous_buffer;
+    BufferBase::set(current_buffer->uncompressed.data(), buf_size, 0);
+}
+
+void ParallelCompressedWriteBuffer::finalizeImpl()
+{
+    next();
+    pool.wait();
+}
+
+void ParallelCompressedWriteBuffer::compress(Iterator buffer)
+{
+    CurrentMetrics::Increment metric_increment(CurrentMetrics::ParallelCompressedWriteBufferThreads);
+
+    chassert(buffer->uncompressed_size <= INT_MAX);
+    UInt32 uncompressed_size = static_cast<UInt32>(buffer->uncompressed_size);
+    UInt32 compressed_reserve_size = codec->getCompressedReserveSize(uncompressed_size);
+
+    /// If all previous buffers have been written,
+    /// and if the output buffer has the required capacity,
+    /// we can compress data directly into the output buffer.
+    size_t required_out_capacity = compressed_reserve_size + sizeof(CityHash_v1_0_2::uint128);
+    bool can_write_directly = false;
+
+    if (!buffer->previous)
+    {
+        can_write_directly = out.available() >= required_out_capacity;
+    }
+    else
+    {
+        std::unique_lock lock(mutex);
+        can_write_directly = (!buffer->previous->busy || buffer->previous->sequence_num > buffer->sequence_num)
+            && out.available() >= required_out_capacity;
+    }
+
+    if (can_write_directly)
+    {
+        char * out_compressed_ptr = out.position() + sizeof(CityHash_v1_0_2::uint128);
+        UInt32 compressed_size = codec->compress(buffer->uncompressed.data(), uncompressed_size, out_compressed_ptr);
+
+        CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(out_compressed_ptr, compressed_size);
+
+        writeBinaryLittleEndian(checksum.low64, out);
+        writeBinaryLittleEndian(checksum.high64, out);
+
+        out.position() += compressed_size;
+    }
+    else
+    {
+        buffer->compressed.resize(compressed_reserve_size);
+        UInt32 compressed_size = codec->compress(buffer->uncompressed.data(), uncompressed_size, buffer->compressed.data());
+
+        CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(buffer->compressed.data(), compressed_size);
+
+        /// Wait while all previous buffers have been written.
+        if (buffer->previous)
+        {
+            CurrentMetrics::Increment metric_wait_increment(CurrentMetrics::ParallelCompressedWriteBufferWait);
+            std::unique_lock lock(mutex);
+            cond.wait(lock, [&]{ return !buffer->previous->busy || buffer->previous->sequence_num > buffer->sequence_num; });
+        }
+
+        writeBinaryLittleEndian(checksum.low64, out);
+        writeBinaryLittleEndian(checksum.high64, out);
+
+        out.write(buffer->compressed.data(), compressed_size);
+    }
+
+    std::unique_lock lock(mutex);
+    buffer->busy = false;
+    cond.notify_all();
+}
+
+ParallelCompressedWriteBuffer::~ParallelCompressedWriteBuffer()
+{
+    if (!canceled)
+        finalize();
+}
+
+}

src/Compression/ParallelCompressedWriteBuffer.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <list>
+#include <memory>
+
+#include <Common/PODArray.h>
+
+#include <IO/WriteBuffer.h>
+#include <IO/BufferWithOwnMemory.h>
+#include <Compression/ICompressionCodec.h>
+#include <Compression/CompressionFactory.h>
+#include <Common/ThreadPool.h>
+
+
+namespace DB
+{
+
+/** Uses multi-buffering for parallel compression.
+  * When the buffer is filled, it will be compressed in the background,
+  * and a new buffer is created for the next input data.
+  */
+class ParallelCompressedWriteBuffer final : public WriteBuffer
+{
+public:
+    explicit ParallelCompressedWriteBuffer(
+        WriteBuffer & out_,
+        CompressionCodecPtr codec_,
+        size_t buf_size_,
+        size_t num_threads_,
+        ThreadPool & pool_);
+
+    ~ParallelCompressedWriteBuffer() override;
+
+private:
+    void nextImpl() override;
+    void finalizeImpl() override;
+
+    WriteBuffer & out;
+    CompressionCodecPtr codec;
+    size_t buf_size;
+    size_t num_threads;
+    ThreadPool & pool;
+
+    struct BufferPair
+    {
+        explicit BufferPair(size_t input_size)
+            : uncompressed(input_size)
+        {
+        }
+
+        Memory<> uncompressed;
+        size_t uncompressed_size = 0;
+        PODArray<char> compressed;
+        BufferPair * previous = nullptr;
+        size_t sequence_num = 0;
+        bool busy = false;
+    };
+
+    std::mutex mutex;
+    std::condition_variable cond;
+    std::list<BufferPair> buffers;
+
+    using Iterator = std::list<BufferPair>::iterator;
+    Iterator current_buffer;
+    size_t current_sequence_num = 0;
+
+    void compress(Iterator buffer);
+};
+
+}

src/Coordination/tests/gtest_coordination.cpp

@@ -330,7 +330,7 @@ TYPED_TEST(CoordinationTest, TestSummingRaft1)
     this->setLogDirectory("./logs");
     this->setStateFileDirectory(".");
 
-    SummingRaftServer s1(1, "localhost", 44444, this->keeper_context);
+    SummingRaftServer s1(1, "localhost", 0, this->keeper_context);
     SCOPE_EXIT(if (std::filesystem::exists("./state")) std::filesystem::remove("./state"););
 
     /// Single node is leader

src/Core/Settings.cpp

@@ -2869,7 +2869,7 @@ Limit on size of multipart/form-data content. This setting cannot be parsed from
     DECLARE(Bool, calculate_text_stack_trace, true, R"(
 Calculate text stack trace in case of exceptions during query execution. This is the default. It requires symbol lookups that may slow down fuzzing tests when a huge amount of wrong queries are executed. In normal cases, you should not disable this option.
 )", 0) \
-    DECLARE(Bool, enable_job_stack_trace, false, R"(
+    DECLARE(Bool, enable_job_stack_trace, true, R"(
 Output stack trace of a job creator when job results in exception
 )", 0) \
     DECLARE(Bool, allow_ddl, true, R"(
@@ -5708,6 +5708,8 @@ If enabled, MongoDB tables will return an error when a MongoDB query cannot be b
 )", 0) \
     DECLARE(Bool, implicit_select, false, R"(
 Allow writing simple SELECT queries without the leading SELECT keyword, which makes it simple for calculator-style usage, e.g. `1 + 2` becomes a valid query.
+
+In `clickhouse-local` it is enabled by default and can be explicitly disabled.
 )", 0) \
     \
     \

src/Core/SettingsChangesHistory.cpp

@@ -64,6 +64,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
     },
     {"24.11",
         {
+            {"enable_job_stack_trace", false, true, "Enable by default collecting stack traces from job's scheduling."},
             {"allow_suspicious_types_in_group_by", true, false, "Don't allow Variant/Dynamic types in GROUP BY by default"},
             {"allow_suspicious_types_in_order_by", true, false, "Don't allow Variant/Dynamic types in ORDER BY by default"},
             {"distributed_cache_discard_connection_if_unread_data", true, true, "New setting"},

src/Dictionaries/HashedDictionary.h

@@ -334,22 +334,26 @@ HashedDictionary<dictionary_key_type, sparse, sharded>::~HashedDictionary()
         if (container.empty())
             return;
 
-        pool.trySchedule([&container, thread_group = CurrentThread::getGroup()]
-        {
-            SCOPE_EXIT_SAFE(
+        if (!pool.trySchedule([&container, thread_group = CurrentThread::getGroup()]
+            {
+                SCOPE_EXIT_SAFE(
+                    if (thread_group)
+                        CurrentThread::detachFromGroupIfNotDetached();
+                );
+
+                /// Do not account memory that was occupied by the dictionaries for the query/user context.
+                MemoryTrackerBlockerInThread memory_blocker;
+
                 if (thread_group)
-                    CurrentThread::detachFromGroupIfNotDetached();
-            );
+                    CurrentThread::attachToGroupIfDetached(thread_group);
+                setThreadName("HashedDictDtor");
 
-            /// Do not account memory that was occupied by the dictionaries for the query/user context.
+                clearContainer(container);
+            }))
+        {
             MemoryTrackerBlockerInThread memory_blocker;
-
-            if (thread_group)
-                CurrentThread::attachToGroupIfDetached(thread_group);
-            setThreadName("HashedDictDtor");
-
             clearContainer(container);
-        });
+        }
 
         ++hash_tables_count;
     };

src/Formats/PrettyFormatHelpers.cpp

@@ -0,0 +1,107 @@
+#include <Formats/PrettyFormatHelpers.h>
+#include <IO/WriteBuffer.h>
+#include <IO/WriteHelpers.h>
+#include <Processors/Chunk.h>
+#include <Common/formatReadable.h>
+
+
+static constexpr const char * GRAY_COLOR = "\033[90m";
+static constexpr const char * UNDERSCORE = "\033[4m";
+static constexpr const char * RESET_COLOR = "\033[0m";
+
+
+namespace DB
+{
+
+void writeReadableNumberTipIfSingleValue(WriteBuffer & out, const Chunk & chunk, const FormatSettings & settings, bool color)
+{
+    if (chunk.getNumRows() == 1 && chunk.getNumColumns() == 1)
+        writeReadableNumberTip(out, *chunk.getColumns()[0], 0, settings, color);
+}
+
+void writeReadableNumberTip(WriteBuffer & out, const IColumn & column, size_t row, const FormatSettings & settings, bool color)
+{
+    if (column.isNullAt(row))
+        return;
+
+    auto value = column.getFloat64(row);
+    auto threshold = settings.pretty.output_format_pretty_single_large_number_tip_threshold;
+
+    if (threshold && isFinite(value) && abs(value) > threshold)
+    {
+        if (color)
+            writeCString(GRAY_COLOR, out);
+        writeCString(" -- ", out);
+        formatReadableQuantity(value, out, 2);
+        if (color)
+            writeCString(RESET_COLOR, out);
+    }
+}
+
+
+String highlightDigitGroups(String source)
+{
+    if (source.size() <= 4)
+        return source;
+
+    bool is_regular_number = true;
+    size_t num_digits_before_decimal = 0;
+    for (auto c : source)
+    {
+        if (c == '-' || c == ' ')
+            continue;
+        if (c == '.')
+            break;
+        if (c >= '0' && c <= '9')
+        {
+            ++num_digits_before_decimal;
+        }
+        else
+        {
+            is_regular_number = false;
+            break;
+        }
+    }
+
+    if (!is_regular_number || num_digits_before_decimal <= 4)
+        return source;
+
+    String result;
+    size_t size = source.size();
+    result.reserve(2 * size);
+
+    bool before_decimal = true;
+    size_t digit_num = 0;
+    for (size_t i = 0; i < size; ++i)
+    {
+        auto c = source[i];
+        if (before_decimal && c >= '0' && c <= '9')
+        {
+            ++digit_num;
+            size_t offset = num_digits_before_decimal - digit_num;
+            if (offset && offset % 3 == 0)
+            {
+                result += UNDERSCORE;
+                result += c;
+                result += RESET_COLOR;
+            }
+            else
+            {
+                result += c;
+            }
+        }
+        else if (c == '.')
+        {
+            before_decimal = false;
+            result += c;
+        }
+        else
+        {
+            result += c;
+        }
+    }
+
+    return result;
+}
+
+}

src/Formats/PrettyFormatHelpers.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <base/types.h>
+
+
+namespace DB
+{
+
+class Chunk;
+class IColumn;
+class WriteBuffer;
+struct FormatSettings;
+
+/// Prints text describing the number in the form of: -- 12.34 million
+void writeReadableNumberTip(WriteBuffer & out, const IColumn & column, size_t row, const FormatSettings & settings, bool color);
+void writeReadableNumberTipIfSingleValue(WriteBuffer & out, const Chunk & chunk, const FormatSettings & settings, bool color);
+
+/// Underscores digit groups related to thousands using terminal ANSI escape sequences.
+String highlightDigitGroups(String source);
+
+}

src/Functions/FunctionsMiscellaneous.h

@@ -6,6 +6,7 @@
 #include <IO/WriteBufferFromString.h>
 #include <IO/Operators.h>
 #include <Columns/ColumnFunction.h>
+#include <Columns/ColumnConst.h>
 #include <DataTypes/DataTypesNumber.h>
 
 
@@ -112,6 +113,7 @@ public:
         NamesAndTypesList lambda_arguments;
         String return_name;
         DataTypePtr return_type;
+        bool allow_constant_folding;
     };
 
     using CapturePtr = std::shared_ptr<Capture>;
@@ -122,6 +124,7 @@ public:
     String getName() const override { return "FunctionCapture"; }
 
     bool useDefaultImplementationForNulls() const override { return false; }
+
     /// It's possible if expression_actions contains function that don't use
     /// default implementation for Nothing and one of captured columns can be Nothing
     /// Example: SELECT arrayMap(x -> [x, arrayElement(y, 0)], []), [] as y
@@ -148,7 +151,26 @@ public:
         auto function = std::make_unique<FunctionExpression>(expression_actions, types, names,
                                                              capture->return_type, capture->return_name);
 
-        return ColumnFunction::create(input_rows_count, std::move(function), arguments);
+        /// If all the captured arguments are constant, let's also return ColumnConst (with ColumnFunction inside it).
+        /// Consequently, it allows to treat higher order functions with constant arrays and constant captured columns
+        /// as constant expressions.
+        /// Consequently, it allows its usage in contexts requiring constants, such as the right hand side of IN.
+        bool constant_folding = capture->allow_constant_folding
+            && std::all_of(arguments.begin(), arguments.end(),
+            [](const auto & arg) { return arg.column->isConst(); });
+
+        if (constant_folding)
+        {
+            ColumnsWithTypeAndName arguments_resized = arguments;
+            for (auto & elem : arguments_resized)
+                elem.column = elem.column->cloneResized(1);
+
+            return ColumnConst::create(ColumnFunction::create(1, std::move(function), arguments_resized), input_rows_count);
+        }
+        else
+        {
+            return ColumnFunction::create(input_rows_count, std::move(function), arguments);
+        }
     }
 
 private:
@@ -203,7 +225,8 @@ public:
             const Names & captured_names,
             const NamesAndTypesList & lambda_arguments,
             const DataTypePtr & function_return_type,
-            const String & expression_return_name)
+            const String & expression_return_name,
+            bool allow_constant_folding)
         : expression_actions(std::move(expression_actions_))
     {
         /// Check that expression does not contain unusual actions that will break columns structure.
@@ -246,6 +269,7 @@ public:
                 .lambda_arguments = lambda_arguments,
                 .return_name = expression_return_name,
                 .return_type = function_return_type,
+                .allow_constant_folding = allow_constant_folding,
         });
     }
 

src/Functions/IFunction.h

@@ -184,7 +184,7 @@ public:
 
     /** If function isSuitableForConstantFolding then, this method will be called during query analysis
       * if some arguments are constants. For example logical functions (AndFunction, OrFunction) can
-      * return they result based on some constant arguments.
+      * return the result based on some constant arguments.
       * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForNothing,
       * useDefaultImplementationForConstants, useDefaultImplementationForLowCardinality are not applied.
       */

src/Functions/array/FunctionArrayMapped.h

@@ -282,7 +282,9 @@ public:
             if (!column_with_type_and_name.column)
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function.", getName());
 
-            const auto * column_function = typeid_cast<const ColumnFunction *>(column_with_type_and_name.column.get());
+            auto column_function_materialized = column_with_type_and_name.column->convertToFullColumnIfConst();
+
+            const auto * column_function = typeid_cast<const ColumnFunction *>(column_function_materialized.get());
             if (!column_function)
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function.", getName());
 

src/Functions/array/arrayFold.cpp

@@ -87,7 +87,9 @@ public:
         if (!lambda_function_with_type_and_name.column)
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function", getName());
 
-        const auto * lambda_function = typeid_cast<const ColumnFunction *>(lambda_function_with_type_and_name.column.get());
+        auto lambda_function_materialized = lambda_function_with_type_and_name.column->convertToFullColumnIfConst();
+
+        const auto * lambda_function = typeid_cast<const ColumnFunction *>(lambda_function_materialized.get());
         if (!lambda_function)
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function", getName());
 

src/Functions/bitShiftLeft.cpp

@@ -25,8 +25,10 @@ struct BitShiftLeftImpl
     {
         if constexpr (is_big_int_v<B>)
             throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftLeft is not implemented for big integers as second argument");
-        else if (b < 0 || static_cast<UInt256>(b) > 8 * sizeof(A))
-            throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
+        else if (b < 0)
+            throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value");
+        else if (static_cast<UInt256>(b) > 8 * sizeof(A))
+            return static_cast<Result>(0);
         else if constexpr (is_big_int_v<A>)
             return static_cast<Result>(a) << static_cast<UInt32>(b);
         else
@@ -43,9 +45,10 @@ struct BitShiftLeftImpl
             const UInt8 word_size = 8 * sizeof(*pos);
             size_t n = end - pos;
             const UInt128 bit_limit = static_cast<UInt128>(word_size) * n;
-            if (b < 0 || static_cast<decltype(bit_limit)>(b) > bit_limit)
-                throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
-            if (b == bit_limit)
+            if (b < 0)
+                throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value");
+
+            if (b == bit_limit || static_cast<decltype(bit_limit)>(b) > bit_limit)
             {
                 // insert default value
                 out_vec.push_back(0);
@@ -111,9 +114,10 @@ struct BitShiftLeftImpl
             const UInt8 word_size = 8;
             size_t n = end - pos;
             const UInt128 bit_limit = static_cast<UInt128>(word_size) * n;
-            if (b < 0 || static_cast<decltype(bit_limit)>(b) > bit_limit)
-                throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
-            if (b == bit_limit)
+            if (b < 0)
+                throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value");
+
+            if (b == bit_limit || static_cast<decltype(bit_limit)>(b) > bit_limit)
             {
                 // insert default value
                 out_vec.resize_fill(out_vec.size() + n);

src/Functions/bitShiftRight.cpp

@@ -26,8 +26,10 @@ struct BitShiftRightImpl
     {
         if constexpr (is_big_int_v<B>)
             throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftRight is not implemented for big integers as second argument");
-        else if (b < 0 || static_cast<UInt256>(b) > 8 * sizeof(A))
-            throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
+        else if (b < 0)
+            throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value");
+        else if (static_cast<UInt256>(b) > 8 * sizeof(A))
+            return static_cast<Result>(0);
         else if constexpr (is_big_int_v<A>)
             return static_cast<Result>(a) >> static_cast<UInt32>(b);
         else
@@ -59,9 +61,10 @@ struct BitShiftRightImpl
             const UInt8 word_size = 8;
             size_t n = end - pos;
             const UInt128 bit_limit = static_cast<UInt128>(word_size) * n;
-            if (b < 0 || static_cast<decltype(bit_limit)>(b) > bit_limit)
-                throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
-            if (b == bit_limit)
+            if (b < 0)
+                throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value");
+
+            if (b == bit_limit || static_cast<decltype(bit_limit)>(b) > bit_limit)
             {
                 /// insert default value
                 out_vec.push_back(0);
@@ -99,9 +102,10 @@ struct BitShiftRightImpl
             const UInt8 word_size = 8;
             size_t n = end - pos;
             const UInt128 bit_limit = static_cast<UInt128>(word_size) * n;
-            if (b < 0 || static_cast<decltype(bit_limit)>(b) > bit_limit)
-                throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
-            if (b == bit_limit)
+            if (b < 0)
+                throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value");
+
+            if (b == bit_limit || static_cast<decltype(bit_limit)>(b) > bit_limit)
             {
                 // insert default value
                 out_vec.resize_fill(out_vec.size() + n);

src/Functions/nested.cpp

@@ -108,27 +108,29 @@ public:
     {
         size_t arguments_size = arguments.size();
 
-        const auto * lhs_array = assert_cast<const ColumnArray *>(arguments.at(1).column.get());
+        ColumnPtr first_array_materialized = arguments[1].column->convertToFullColumnIfConst();
+        const ColumnArray & first_array = assert_cast<const ColumnArray &>(*first_array_materialized);
 
         Columns data_columns;
         data_columns.reserve(arguments_size);
-        data_columns.push_back(lhs_array->getDataPtr());
+        data_columns.push_back(first_array.getDataPtr());
 
         for (size_t i = 2; i < arguments_size; ++i)
         {
-            const auto * rhs_array = assert_cast<const ColumnArray *>(arguments[i].column.get());
+            ColumnPtr other_array_materialized = arguments[i].column->convertToFullColumnIfConst();
+            const ColumnArray & other_array = assert_cast<const ColumnArray &>(*other_array_materialized);
 
-            if (!lhs_array->hasEqualOffsets(*rhs_array))
+            if (!first_array.hasEqualOffsets(other_array))
                 throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH,
                     "The argument 2 and argument {} of function {} have different array offsets",
                     i + 1,
                     getName());
 
-            data_columns.push_back(rhs_array->getDataPtr());
+            data_columns.push_back(other_array.getDataPtr());
         }
 
         auto tuple_column = ColumnTuple::create(std::move(data_columns));
-        auto array_column = ColumnArray::create(std::move(tuple_column), lhs_array->getOffsetsPtr());
+        auto array_column = ColumnArray::create(std::move(tuple_column), first_array.getOffsetsPtr());
 
         return array_column;
     }
@@ -168,7 +170,12 @@ REGISTER_FUNCTION(Nested)
 {
     factory.registerFunction<FunctionNested>(FunctionDocumentation{
         .description=R"(
+This is a function used internally by the ClickHouse engine and not meant to be used directly.
+
 Returns the array of tuples from multiple arrays.
+
+The first argument must be a constant array of Strings determining the names of the resulting Tuple.
+The other arguments must be arrays of the same size.
 )",
         .examples{{"nested", "SELECT nested(['keys', 'values'], ['key_1', 'key_2'], ['value_1','value_2'])", ""}},
         .categories{"OtherFunctions"}

src/IO/S3/Client.cpp

@@ -645,7 +645,7 @@ Client::doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request
             try
             {
                 /// S3 does retries network errors actually.
-                /// But it is matter when errors occur.
+                /// But it does matter when errors occur.
                 /// This code retries a specific case when
                 /// network error happens when XML document is being read from the response body.
                 /// Hence, the response body is a stream, network errors are possible at reading.
@@ -656,8 +656,9 @@ Client::doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request
                 /// Requests that expose the response stream as an answer are not retried with that code. E.g. GetObject.
                 return request_fn_(request_);
             }
-            catch (Poco::Net::ConnectionResetException &)
+            catch (Poco::Net::NetException &)
             {
+                /// This includes "connection reset", "malformed message", and possibly other exceptions.
 
                 if constexpr (IsReadMethod)
                 {

src/Interpreters/ActionsVisitor.cpp

@@ -1308,7 +1308,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
                     String lambda_name = data.getUniqueName("__lambda");
 
                     auto function_capture = std::make_shared<FunctionCaptureOverloadResolver>(
-                            lambda_actions, captured, lambda_arguments, result_type, result_name);
+                            lambda_actions, captured, lambda_arguments, result_type, result_name, false);
                     data.addFunction(function_capture, captured, lambda_name);
 
                     argument_types[i] = std::make_shared<DataTypeFunction>(lambda_type->getArgumentTypes(), result_type);

src/Interpreters/InterpreterCreateQuery.cpp

@@ -1467,7 +1467,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     bool is_secondary_query = getContext()->getZooKeeperMetadataTransaction() && !getContext()->getZooKeeperMetadataTransaction()->isInitialQuery();
     auto mode = getLoadingStrictnessLevel(create.attach, /*force_attach*/ false, /*has_force_restore_data_flag*/ false, is_secondary_query || is_restore_from_backup);
 
-    if (!create.sql_security && create.supportSQLSecurity() && !getContext()->getServerSettings()[ServerSetting::ignore_empty_sql_security_in_create_view_query])
+    if (!create.sql_security && create.supportSQLSecurity() && (create.refresh_strategy || !getContext()->getServerSettings()[ServerSetting::ignore_empty_sql_security_in_create_view_query]))
         create.sql_security = std::make_shared<ASTSQLSecurity>();
 
     if (create.sql_security)

src/Interpreters/InterpreterInsertQuery.cpp

@@ -121,8 +121,7 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query)
 
             if (current_context->getSettingsRef()[Setting::allow_experimental_analyzer])
             {
-                InterpreterSelectQueryAnalyzer interpreter_select(query.select, current_context, select_query_options);
-                header_block = interpreter_select.getSampleBlock();
+                header_block = InterpreterSelectQueryAnalyzer::getSampleBlock(query.select, current_context, select_query_options);
             }
             else
             {

src/Interpreters/InterpreterSystemQuery.cpp

@@ -795,9 +795,9 @@ BlockIO InterpreterSystemQuery::execute()
         case Type::WAIT_FAILPOINT:
         {
             getContext()->checkAccess(AccessType::SYSTEM_FAILPOINT);
-            LOG_TRACE(log, "waiting for failpoint {}", query.fail_point_name);
+            LOG_TRACE(log, "Waiting for failpoint {}", query.fail_point_name);
             FailPointInjection::pauseFailPoint(query.fail_point_name);
-            LOG_TRACE(log, "finished failpoint {}", query.fail_point_name);
+            LOG_TRACE(log, "Finished waiting for failpoint {}", query.fail_point_name);
             break;
         }
         case Type::RESET_COVERAGE:

src/Planner/PlannerActionsVisitor.cpp

@@ -804,7 +804,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
 
     auto lambda_node_name = calculateActionNodeName(node, *planner_context);
     auto function_capture = std::make_shared<FunctionCaptureOverloadResolver>(
-        lambda_actions, captured_column_names, lambda_arguments_names_and_types, lambda_node.getExpression()->getResultType(), lambda_expression_node_name);
+        lambda_actions, captured_column_names, lambda_arguments_names_and_types, lambda_node.getExpression()->getResultType(), lambda_expression_node_name, true);
 
     // TODO: Pass IFunctionBase here not FunctionCaptureOverloadResolver.
     const auto * actions_node = actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), function_capture);

src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp

@@ -1,5 +1,6 @@
 #include <Processors/Formats/Impl/PrettyBlockOutputFormat.h>
 #include <Formats/FormatFactory.h>
+#include <Formats/PrettyFormatHelpers.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
 #include <IO/WriteBufferFromString.h>
@@ -352,7 +353,8 @@ void PrettyBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind
         }
 
         writeCString(grid_symbols.bar, out);
-        writeReadableNumberTip(chunk);
+        if (readable_number_tip)
+            writeReadableNumberTipIfSingleValue(out, chunk, format_settings, color);
         writeCString("\n", out);
     }
 
@@ -392,72 +394,6 @@ void PrettyBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind
 }
 
 
-static String highlightDigitGroups(String source)
-{
-    if (source.size() <= 4)
-        return source;
-
-    bool is_regular_number = true;
-    size_t num_digits_before_decimal = 0;
-    for (auto c : source)
-    {
-        if (c == '-' || c == ' ')
-            continue;
-        if (c == '.')
-            break;
-        if (c >= '0' && c <= '9')
-        {
-            ++num_digits_before_decimal;
-        }
-        else
-        {
-            is_regular_number = false;
-            break;
-        }
-    }
-
-    if (!is_regular_number || num_digits_before_decimal <= 4)
-        return source;
-
-    String result;
-    size_t size = source.size();
-    result.reserve(2 * size);
-
-    bool before_decimal = true;
-    size_t digit_num = 0;
-    for (size_t i = 0; i < size; ++i)
-    {
-        auto c = source[i];
-        if (before_decimal && c >= '0' && c <= '9')
-        {
-            ++digit_num;
-            size_t offset = num_digits_before_decimal - digit_num;
-            if (offset && offset % 3 == 0)
-            {
-                result += "\033[4m";
-                result += c;
-                result += "\033[0m";
-            }
-            else
-            {
-                result += c;
-            }
-        }
-        else if (c == '.')
-        {
-            before_decimal = false;
-            result += c;
-        }
-        else
-        {
-            result += c;
-        }
-    }
-
-    return result;
-}
-
-
 void PrettyBlockOutputFormat::writeValueWithPadding(
     const IColumn & column, const ISerialization & serialization, size_t row_num,
     size_t value_width, size_t pad_to_width, size_t cut_to_width, bool align_right, bool is_number)
@@ -553,30 +489,6 @@ void PrettyBlockOutputFormat::writeSuffix()
     }
 }
 
-void PrettyBlockOutputFormat::writeReadableNumberTip(const Chunk & chunk)
-{
-    const auto & columns = chunk.getColumns();
-    auto is_single_number = readable_number_tip && chunk.getNumRows() == 1 && chunk.getNumColumns() == 1;
-    if (!is_single_number)
-        return;
-
-    if (columns[0]->isNullAt(0))
-        return;
-
-    auto value = columns[0]->getFloat64(0);
-    auto threshold = format_settings.pretty.output_format_pretty_single_large_number_tip_threshold;
-
-    if (threshold && isFinite(value) && abs(value) > threshold)
-    {
-        if (color)
-            writeCString("\033[90m", out);
-        writeCString(" -- ", out);
-        formatReadableQuantity(value, out, 2);
-        if (color)
-            writeCString("\033[0m", out);
-    }
-}
-
 void registerOutputFormatPretty(FormatFactory & factory)
 {
     registerPrettyFormatWithNoEscapesAndMonoBlock<PrettyBlockOutputFormat>(factory, "Pretty");

src/Processors/Formats/Impl/PrettyBlockOutputFormat.h

@@ -38,7 +38,6 @@ protected:
     virtual void writeChunk(const Chunk & chunk, PortKind port_kind);
     void writeMonoChunkIfNeeded();
     void writeSuffix() override;
-    void writeReadableNumberTip(const Chunk & chunk);
 
     void onRowsReadBeforeUpdate() override { total_rows = getRowsReadBefore(); }
 
@@ -56,9 +55,9 @@ protected:
     }
 
     bool color;
+    bool readable_number_tip = false;
 
 private:
-    bool readable_number_tip = false;
     bool mono_block;
     /// For mono_block == true only
     Chunk mono_chunk;

src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp

@@ -4,6 +4,7 @@
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>
 #include <Formats/FormatFactory.h>
+#include <Formats/PrettyFormatHelpers.h>
 #include <Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h>
 
 
@@ -189,7 +190,8 @@ void PrettyCompactBlockOutputFormat::writeRow(
     }
 
     writeCString(grid_symbols.bar, out);
-    writeReadableNumberTip(chunk);
+    if (readable_number_tip)
+        writeReadableNumberTipIfSingleValue(out, chunk, format_settings, color);
     writeCString("\n", out);
 }
 

src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp

@@ -1,4 +1,5 @@
 #include <Formats/FormatFactory.h>
+#include <Formats/PrettyFormatHelpers.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
 #include <Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h>
@@ -102,7 +103,8 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port
             writeValueWithPadding(
                 *columns[column], *serializations[column], row, cur_width, max_widths[column], cut_to_width, type.shouldAlignRightInPrettyFormats(), isNumber(type));
         }
-        writeReadableNumberTip(chunk);
+        if (readable_number_tip)
+            writeReadableNumberTipIfSingleValue(out, chunk, format_settings, color);
         writeChar('\n', out);
     }
 

src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp

@@ -4,7 +4,10 @@
 #include <IO/WriteBufferFromString.h>
 #include <Processors/Formats/Impl/VerticalRowOutputFormat.h>
 #include <Formats/FormatFactory.h>
+#include <Formats/PrettyFormatHelpers.h>
 #include <Common/UTF8Helpers.h>
+#include <DataTypes/DataTypeLowCardinality.h>
+#include <DataTypes/DataTypeNullable.h>
 
 
 namespace DB
@@ -14,6 +17,8 @@ VerticalRowOutputFormat::VerticalRowOutputFormat(
     WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_)
     : IRowOutputFormat(header_, out_), format_settings(format_settings_)
 {
+    color = format_settings.pretty.color == 1 || (format_settings.pretty.color == 2 && format_settings.is_writing_to_terminal);
+
     const auto & sample = getPort(PortKind::Main).getHeader();
     size_t columns = sample.columns();
 
@@ -31,6 +36,7 @@ VerticalRowOutputFormat::VerticalRowOutputFormat(
     }
 
     names_and_paddings.resize(columns);
+    is_number.resize(columns);
     for (size_t i = 0; i < columns; ++i)
     {
         WriteBufferFromString buf(names_and_paddings[i]);
@@ -42,6 +48,7 @@ VerticalRowOutputFormat::VerticalRowOutputFormat(
     {
         size_t new_size = max_name_width - name_widths[i] + names_and_paddings[i].size();
         names_and_paddings[i].resize(new_size, ' ');
+        is_number[i] = isNumber(removeNullable(recursiveRemoveLowCardinality(sample.getByPosition(i).type)));
     }
 }
 
@@ -61,7 +68,26 @@ void VerticalRowOutputFormat::writeField(const IColumn & column, const ISerializ
 
 void VerticalRowOutputFormat::writeValue(const IColumn & column, const ISerialization & serialization, size_t row_num) const
 {
-    serialization.serializeText(column, row_num, out, format_settings);
+    if (color && format_settings.pretty.highlight_digit_groups && is_number[field_number])
+    {
+        String serialized_value;
+        {
+            WriteBufferFromString buf(serialized_value);
+            serialization.serializeText(column, row_num, buf, format_settings);
+        }
+
+        /// Highlight groups of thousands.
+        serialized_value = highlightDigitGroups(serialized_value);
+        out.write(serialized_value.data(), serialized_value.size());
+    }
+    else
+    {
+        serialization.serializeText(column, row_num, out, format_settings);
+    }
+
+    /// Write a tip.
+    if (is_number[field_number])
+        writeReadableNumberTip(out, column, row_num, format_settings, color);
 }
 
 

src/Processors/Formats/Impl/VerticalRowOutputFormat.h

@@ -56,6 +56,9 @@ private:
 
     using NamesAndPaddings = std::vector<String>;
     NamesAndPaddings names_and_paddings;
+
+    std::vector<UInt8> is_number;
+    bool color;
 };
 
 }

src/Processors/Sinks/SinkToStorage.h

@@ -5,6 +5,8 @@
 namespace DB
 {
 
+class Context;
+
 /// Sink which is returned from Storage::write.
 class SinkToStorage : public ExceptionKeepingTransform
 {
@@ -16,12 +18,14 @@ public:
 
     const Block & getHeader() const { return inputs.front().getHeader(); }
     void addTableLock(const TableLockHolder & lock) { table_locks.push_back(lock); }
+    void addInterpreterContext(std::shared_ptr<const Context> context) { interpreter_context.emplace_back(std::move(context)); }
 
 protected:
     virtual void consume(Chunk & chunk) = 0;
 
 private:
     std::vector<TableLockHolder> table_locks;
+    std::vector<std::shared_ptr<const Context>> interpreter_context;
 
     void onConsume(Chunk chunk) override;
     GenerateResult onGenerate() override;

src/Processors/Transforms/AggregatingTransform.h

@@ -107,7 +107,7 @@ struct ManyAggregatedData
                 if (variant->aggregator)
                 {
                     // variant is moved here and will be destroyed in the destructor of the lambda function.
-                    pool->trySchedule(
+                    pool->scheduleOrThrowOnError(
                         [my_variant = std::move(variant), thread_group = CurrentThread::getGroup()]()
                         {
                             SCOPE_EXIT_SAFE(

src/Storages/MergeTree/IMergeTreeDataPart.cpp

@@ -624,6 +624,15 @@ UInt64 IMergeTreeDataPart::getIndexSizeInAllocatedBytes() const
     return res;
 }
 
+UInt64 IMergeTreeDataPart::getIndexGranularityBytes() const
+{
+    return index_granularity.getBytesSize();
+}
+UInt64 IMergeTreeDataPart::getIndexGranularityAllocatedBytes() const
+{
+    return index_granularity.getBytesAllocated();
+}
+
 void IMergeTreeDataPart::assertState(const std::initializer_list<MergeTreeDataPartState> & affordable_states) const
 {
     if (!checkState(affordable_states))
@@ -735,7 +744,9 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks
             loadUUID();
         loadColumns(require_columns_checksums);
         loadChecksums(require_columns_checksums);
+
         loadIndexGranularity();
+        index_granularity.shrinkToFitInMemory();
 
         if (!(*storage.getSettings())[MergeTreeSetting::primary_key_lazy_load])
             getIndex();

src/Storages/MergeTree/IMergeTreeDataPart.h

@@ -380,6 +380,8 @@ public:
     /// For data in RAM ('index')
     UInt64 getIndexSizeInBytes() const;
     UInt64 getIndexSizeInAllocatedBytes() const;
+    UInt64 getIndexGranularityBytes() const;
+    UInt64 getIndexGranularityAllocatedBytes() const;
     UInt64 getMarksCount() const;
     UInt64 getIndexSizeFromFile() const;
 

src/Storages/MergeTree/KeyCondition.cpp

@@ -597,12 +597,15 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
         case (ActionsDAG::ActionType::COLUMN):
         {
             String name;
-            if (const auto * column_const = typeid_cast<const ColumnConst *>(node.column.get()))
+            if (const auto * column_const = typeid_cast<const ColumnConst *>(node.column.get());
+                column_const && column_const->getDataType() != TypeIndex::Function)
+            {
                 /// Re-generate column name for constant.
-                /// DAG form query (with enabled analyzer) uses suffixes for constants, like 1_UInt8.
-                /// DAG from PK does not use it. This breaks matching by column name sometimes.
+                /// DAG from the query (with enabled analyzer) uses suffixes for constants, like 1_UInt8.
+                /// DAG from the PK does not use it. This breaks matching by column name sometimes.
                 /// Ideally, we should not compare names, but DAG subtrees instead.
-                name = ASTLiteral(column_const->getDataColumn()[0]).getColumnName();
+                name = ASTLiteral(column_const->getField()).getColumnName();
+            }
             else
                 name = node.result_name;
 

src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.cpp

@@ -116,7 +116,7 @@ bool allow(
     double sum_size,
     double max_size,
     double min_age,
-    double range_size,
+    size_t range_size,
     double partition_size,
     double min_size_to_lower_base_log,
     double max_size_to_lower_base_log,
@@ -125,6 +125,9 @@ bool allow(
     if (settings.min_age_to_force_merge && min_age >= settings.min_age_to_force_merge)
         return true;
 
+    if (settings.min_parts_to_merge_at_once && range_size < settings.min_parts_to_merge_at_once)
+        return false;
+
     /// Map size to 0..1 using logarithmic scale
     /// Use log(1 + x) instead of log1p(x) because our sum_size is always integer.
     /// Also log1p seems to be slow and significantly affect performance of merges assignment.

src/Storages/MergeTree/MergeSelectors/SimpleMergeSelector.h

@@ -90,6 +90,8 @@ public:
     {
         /// Zero means unlimited. Can be overridden by the same merge tree setting.
         size_t max_parts_to_merge_at_once = 100;
+        /// Zero means no minimum. Can be overridden by the same merge tree setting.
+        size_t min_parts_to_merge_at_once = 0;
 
         /// Some sort of a maximum number of parts in partition. Can be overridden by the same merge tree setting.
         size_t parts_to_throw_insert = 3000;

src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp

@@ -82,6 +82,7 @@ namespace MergeTreeSetting
     extern const MergeTreeSettingsMergeSelectorAlgorithm merge_selector_algorithm;
     extern const MergeTreeSettingsBool merge_selector_enable_heuristic_to_remove_small_parts_at_right;
     extern const MergeTreeSettingsFloat merge_selector_base;
+    extern const MergeTreeSettingsUInt64 min_parts_to_merge_at_once;
 }
 
 namespace ErrorCodes
@@ -267,7 +268,8 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
         if (status == SelectPartsDecision::SELECTED)
             res.insert(all_partition_ids[i]);
         else
-            LOG_TEST(log, "Nothing to merge in partition {}: {}", all_partition_ids[i], out_disable_reason.text);
+            LOG_TEST(log, "Nothing to merge in partition {} with max_total_size_to_merge = {} (looked up {} ranges): {}",
+                all_partition_ids[i], ReadableSize(max_total_size_to_merge), ranges_per_partition[i].size(), out_disable_reason.text);
     }
 
     String best_partition_id_to_optimize = getBestPartitionToOptimizeEntire(info.partitions_info);
@@ -565,6 +567,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
             simple_merge_settings.max_parts_to_merge_at_once = (*data_settings)[MergeTreeSetting::max_parts_to_merge_at_once];
             simple_merge_settings.enable_heuristic_to_remove_small_parts_at_right = (*data_settings)[MergeTreeSetting::merge_selector_enable_heuristic_to_remove_small_parts_at_right];
             simple_merge_settings.base = (*data_settings)[MergeTreeSetting::merge_selector_base];
+            simple_merge_settings.min_parts_to_merge_at_once = (*data_settings)[MergeTreeSetting::min_parts_to_merge_at_once];
 
             if (!(*data_settings)[MergeTreeSetting::min_age_to_force_merge_on_partition_only])
                 simple_merge_settings.min_age_to_force_merge = (*data_settings)[MergeTreeSetting::min_age_to_force_merge_seconds];

src/Storages/MergeTree/MergeTreeIndexGranularity.cpp

@@ -122,4 +122,20 @@ std::string MergeTreeIndexGranularity::describe() const
 {
     return fmt::format("initialized: {}, marks_rows_partial_sums: [{}]", initialized, fmt::join(marks_rows_partial_sums, ", "));
 }
+
+void MergeTreeIndexGranularity::shrinkToFitInMemory()
+{
+    marks_rows_partial_sums.shrink_to_fit();
+}
+
+uint64_t MergeTreeIndexGranularity::getBytesSize() const
+{
+    return marks_rows_partial_sums.size() * sizeof(size_t);
+}
+uint64_t MergeTreeIndexGranularity::getBytesAllocated() const
+{
+    return marks_rows_partial_sums.capacity() * sizeof(size_t);
+}
+
+
 }

src/Storages/MergeTree/MergeTreeIndexGranularity.h

@@ -100,6 +100,11 @@ public:
     void resizeWithFixedGranularity(size_t size, size_t fixed_granularity);
 
     std::string describe() const;
+
+    void shrinkToFitInMemory();
+
+    uint64_t getBytesSize() const;
+    uint64_t getBytesAllocated() const;
 };
 
 }

src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp

@@ -178,23 +178,20 @@ String USearchIndexWithSerialization::Statistics::toString() const
 }
 MergeTreeIndexGranuleVectorSimilarity::MergeTreeIndexGranuleVectorSimilarity(
     const String & index_name_,
-    const Block & index_sample_block_,
     unum::usearch::metric_kind_t metric_kind_,
     unum::usearch::scalar_kind_t scalar_kind_,
     UsearchHnswParams usearch_hnsw_params_)
-    : MergeTreeIndexGranuleVectorSimilarity(index_name_, index_sample_block_, metric_kind_, scalar_kind_, usearch_hnsw_params_, nullptr)
+    : MergeTreeIndexGranuleVectorSimilarity(index_name_, metric_kind_, scalar_kind_, usearch_hnsw_params_, nullptr)
 {
 }
 
 MergeTreeIndexGranuleVectorSimilarity::MergeTreeIndexGranuleVectorSimilarity(
     const String & index_name_,
-    const Block & index_sample_block_,
     unum::usearch::metric_kind_t metric_kind_,
     unum::usearch::scalar_kind_t scalar_kind_,
     UsearchHnswParams usearch_hnsw_params_,
     USearchIndexWithSerializationPtr index_)
     : index_name(index_name_)
-    , index_sample_block(index_sample_block_)
     , metric_kind(metric_kind_)
     , scalar_kind(scalar_kind_)
     , usearch_hnsw_params(usearch_hnsw_params_)
@@ -261,7 +258,7 @@ MergeTreeIndexAggregatorVectorSimilarity::MergeTreeIndexAggregatorVectorSimilari
 
 MergeTreeIndexGranulePtr MergeTreeIndexAggregatorVectorSimilarity::getGranuleAndReset()
 {
-    auto granule = std::make_shared<MergeTreeIndexGranuleVectorSimilarity>(index_name, index_sample_block, metric_kind, scalar_kind, usearch_hnsw_params, index);
+    auto granule = std::make_shared<MergeTreeIndexGranuleVectorSimilarity>(index_name, metric_kind, scalar_kind, usearch_hnsw_params, index);
     index = nullptr;
     return granule;
 }
@@ -490,7 +487,7 @@ MergeTreeIndexVectorSimilarity::MergeTreeIndexVectorSimilarity(
 
 MergeTreeIndexGranulePtr MergeTreeIndexVectorSimilarity::createIndexGranule() const
 {
-    return std::make_shared<MergeTreeIndexGranuleVectorSimilarity>(index.name, index.sample_block, metric_kind, scalar_kind, usearch_hnsw_params);
+    return std::make_shared<MergeTreeIndexGranuleVectorSimilarity>(index.name, metric_kind, scalar_kind, usearch_hnsw_params);
 }
 
 MergeTreeIndexAggregatorPtr MergeTreeIndexVectorSimilarity::createIndexAggregator(const MergeTreeWriterSettings & /*settings*/) const
@@ -531,15 +528,17 @@ void vectorSimilarityIndexValidator(const IndexDescription & index, bool /* atta
 {
     const bool has_two_args = (index.arguments.size() == 2);
     const bool has_five_args = (index.arguments.size() == 5);
+    const bool has_six_args = (index.arguments.size() == 6); /// Legacy index creation syntax before #70616. Supported only to be able to load old tables, can be removed mid-2025.
+                                                             /// The 6th argument (ef_search) is ignored.
 
     /// Check number and type of arguments
-    if (!has_two_args && !has_five_args)
+    if (!has_two_args && !has_five_args && !has_six_args)
         throw Exception(ErrorCodes::INCORRECT_QUERY, "Vector similarity index must have two or five arguments");
     if (index.arguments[0].getType() != Field::Types::String)
         throw Exception(ErrorCodes::INCORRECT_QUERY, "First argument of vector similarity index (method) must be of type String");
     if (index.arguments[1].getType() != Field::Types::String)
         throw Exception(ErrorCodes::INCORRECT_QUERY, "Second argument of vector similarity index (metric) must be of type String");
-    if (has_five_args)
+    if (has_five_args || has_six_args)
     {
         if (index.arguments[2].getType() != Field::Types::String)
             throw Exception(ErrorCodes::INCORRECT_QUERY, "Third argument of vector similarity index (quantization) must be of type String");

src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.h

@@ -69,14 +69,12 @@ struct MergeTreeIndexGranuleVectorSimilarity final : public IMergeTreeIndexGranu
 {
     MergeTreeIndexGranuleVectorSimilarity(
         const String & index_name_,
-        const Block & index_sample_block_,
         unum::usearch::metric_kind_t metric_kind_,
         unum::usearch::scalar_kind_t scalar_kind_,
         UsearchHnswParams usearch_hnsw_params_);
 
     MergeTreeIndexGranuleVectorSimilarity(
         const String & index_name_,
-        const Block & index_sample_block_,
         unum::usearch::metric_kind_t metric_kind_,
         unum::usearch::scalar_kind_t scalar_kind_,
         UsearchHnswParams usearch_hnsw_params_,
@@ -90,7 +88,6 @@ struct MergeTreeIndexGranuleVectorSimilarity final : public IMergeTreeIndexGranu
     bool empty() const override { return !index || index->size() == 0; }
 
     const String index_name;
-    const Block index_sample_block;
     const unum::usearch::metric_kind_t metric_kind;
     const unum::usearch::scalar_kind_t scalar_kind;
     const UsearchHnswParams usearch_hnsw_params;

src/Storages/MergeTree/MergeTreeSettings.cpp

@@ -102,6 +102,7 @@ namespace ErrorCodes
     DECLARE(MergeSelectorAlgorithm, merge_selector_algorithm, MergeSelectorAlgorithm::SIMPLE, "The algorithm to select parts for merges assignment", EXPERIMENTAL) \
     DECLARE(Bool, merge_selector_enable_heuristic_to_remove_small_parts_at_right, true, "Enable heuristic for selecting parts for merge which removes parts from right side of range, if their size is less than specified ratio (0.01) of sum_size. Works for Simple and StochasticSimple merge selectors", 0) \
     DECLARE(Float, merge_selector_base, 5.0, "Affects write amplification of assigned merges (expert level setting, don't change if you don't understand what it is doing). Works for Simple and StochasticSimple merge selectors", 0) \
+    DECLARE(UInt64, min_parts_to_merge_at_once, 0, "Minimal amount of data parts which merge selector can pick to merge at once (expert level setting, don't change if you don't understand what it is doing). 0 - disabled. Works for Simple and StochasticSimple merge selectors.", 0) \
     \
     /** Inserts settings. */ \
     DECLARE(UInt64, parts_to_delay_insert, 1000, "If table contains at least that many active parts in single partition, artificially slow down insert into table. Disabled if set to 0", 0) \

src/Storages/MergeTree/MergedBlockOutputStream.cpp

@@ -207,6 +207,8 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync(
     new_part->setBytesOnDisk(checksums.getTotalSizeOnDisk());
     new_part->setBytesUncompressedOnDisk(checksums.getTotalSizeUncompressedOnDisk());
     new_part->index_granularity = writer->getIndexGranularity();
+    /// Just in case
+    new_part->index_granularity.shrinkToFitInMemory();
     new_part->calculateColumnsAndSecondaryIndicesSizesOnDisk();
 
     /// In mutation, existing_rows_count is already calculated in PartMergerWriter

src/Storages/MergeTree/MutateTask.cpp

@@ -984,6 +984,8 @@ void finalizeMutatedPart(
 
     new_data_part->rows_count = source_part->rows_count;
     new_data_part->index_granularity = source_part->index_granularity;
+    /// Just in case
+    new_data_part->index_granularity.shrinkToFitInMemory();
     new_data_part->setIndex(*source_part->getIndex());
     new_data_part->minmax_idx = source_part->minmax_idx;
     new_data_part->modification_time = time(nullptr);

src/Storages/StorageMaterializedView.cpp

@@ -392,6 +392,7 @@ void StorageMaterializedView::read(
         }
 
         query_plan.addStorageHolder(storage);
+        query_plan.addInterpreterContext(context);
         query_plan.addTableLock(std::move(lock));
     }
 }
@@ -415,6 +416,7 @@ SinkToStoragePtr StorageMaterializedView::write(const ASTPtr & query, const Stor
 
     auto sink = storage->write(query, metadata_snapshot, context, async_insert);
 
+    sink->addInterpreterContext(context);
     sink->addTableLock(lock);
     return sink;
 }

src/Storages/System/StorageSystemParts.cpp

@@ -75,6 +75,8 @@ StorageSystemParts::StorageSystemParts(const StorageID & table_id_)
         {"data_version",                                std::make_shared<DataTypeUInt64>(),    "Number that is used to determine which mutations should be applied to the data part (mutations with a version higher than data_version)."},
         {"primary_key_bytes_in_memory",                 std::make_shared<DataTypeUInt64>(),    "The amount of memory (in bytes) used by primary key values."},
         {"primary_key_bytes_in_memory_allocated",       std::make_shared<DataTypeUInt64>(),    "The amount of memory (in bytes) reserved for primary key values."},
+        {"index_granularity_bytes_in_memory",           std::make_shared<DataTypeUInt64>(),    "The amount of memory (in bytes) used by index granularity values."},
+        {"index_granularity_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>(),    "The amount of memory (in bytes) reserved for index granularity values."},
         {"is_frozen",                                   std::make_shared<DataTypeUInt8>(),     "Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup does not exist. "},
 
         {"database",                                    std::make_shared<DataTypeString>(),    "Name of the database."},
@@ -216,6 +218,10 @@ void StorageSystemParts::processNextStorage(
             columns[res_index++]->insert(part->getIndexSizeInBytes());
         if (columns_mask[src_index++])
             columns[res_index++]->insert(part->getIndexSizeInAllocatedBytes());
+        if (columns_mask[src_index++])
+            columns[res_index++]->insert(part->getIndexGranularityBytes());
+        if (columns_mask[src_index++])
+            columns[res_index++]->insert(part->getIndexGranularityAllocatedBytes());
         if (columns_mask[src_index++])
             columns[res_index++]->insert(part->is_frozen.load(std::memory_order_relaxed));
 

tests/docker_scripts/upgrade_runner.sh

@@ -63,7 +63,7 @@ install_packages previous_release_package_folder
 function save_settings_clean()
 {
   local out=$1 && shift
-  script -q -c "clickhouse-local -q \"select * from system.settings into outfile '$out'\"" --log-out /dev/null
+  script -q -c "clickhouse-local --implicit-select 0 -q \"select * from system.settings into outfile '$out'\"" --log-out /dev/null
 }
 
 # We save the (numeric) version of the old server to compare setting changes between the 2
@@ -147,7 +147,6 @@ then
   FROM new_settings
   LEFT JOIN old_settings ON new_settings.name = old_settings.name
   WHERE (new_value != old_value)
-      AND NOT (startsWith(new_value, 'auto(') AND old_value LIKE '%auto(%')
       AND (name NOT IN (
       SELECT arrayJoin(tupleElement(changes, 'name'))
       FROM

tests/integration/compose/docker_compose_mysql.yml

@@ -5,7 +5,7 @@ services:
         environment:
             MYSQL_ROOT_PASSWORD: clickhouse
             MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST}
-            DATADIR: /mysql/
+            DATADIR: /var/log/mysql/
         expose:
             - ${MYSQL_PORT:-3306}
         command: --server_id=100
@@ -14,11 +14,11 @@ services:
             --gtid-mode="ON"
             --enforce-gtid-consistency
             --log-error-verbosity=3
-            --log-error=/mysql/error.log
+            --log-error=/var/log/mysql/error.log
             --general-log=ON
-            --general-log-file=/mysql/general.log
+            --general-log-file=/var/log/mysql/general.log
         volumes:
             - type: ${MYSQL_LOGS_FS:-tmpfs}
               source: ${MYSQL_LOGS:-}
-              target: /mysql/
+              target: /var/log/mysql/
         user: ${MYSQL_DOCKER_USER}

tests/integration/compose/docker_compose_mysql_8_0.yml

@@ -4,8 +4,8 @@ services:
         restart: always
         environment:
             MYSQL_ROOT_PASSWORD: clickhouse
-            MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST}
-            DATADIR: /mysql/
+            MYSQL_ROOT_HOST: ${MYSQL8_ROOT_HOST}
+            DATADIR: /var/log/mysql/
         expose:
             - ${MYSQL8_PORT:-3306}
         command: --server_id=100 --log-bin='mysql-bin-1.log'
@@ -13,11 +13,11 @@ services:
             --default-time-zone='+3:00' --gtid-mode="ON"
             --enforce-gtid-consistency
             --log-error-verbosity=3
-            --log-error=/mysql/error.log
+            --log-error=/var/log/mysql/error.log
             --general-log=ON
-            --general-log-file=/mysql/general.log
+            --general-log-file=/var/log/mysql/general.log
         volumes:
             - type: ${MYSQL8_LOGS_FS:-tmpfs}
               source: ${MYSQL8_LOGS:-}
-              target: /mysql/
+              target: /var/log/mysql/
         user: ${MYSQL8_DOCKER_USER}

tests/integration/test_storage_s3_queue/test.py

@@ -1000,6 +1000,9 @@ def test_max_set_age(started_cluster):
     assert "Cannot parse input" in node.query(
         f"SELECT exception FROM system.s3queue WHERE file_name ilike '%{file_with_error}'"
     )
+    assert "Cannot parse input" in node.query(
+        f"SELECT exception FROM system.s3queue_log WHERE file_name ilike '%{file_with_error}' ORDER BY processing_end_time DESC LIMIT 1"
+    )
 
     assert 1 == int(
         node.query(

tests/queries/0_stateless/01284_fuzz_bits.sql

@@ -18,7 +18,7 @@ FROM
           reinterpretAsUInt8(
             substring(
               fuzzBits(
-                arrayStringConcat(arrayMap(x -> toString('\0'), range(10000))),
+                materialize(arrayStringConcat(arrayMap(x -> toString('\0'), range(10000)))),
                 0.3
               ),
               id + 1,