Merge branch 'master' of github.com:ClickHouse/ClickHouse into variant-dynamic-null-subcolumn

cmake/target.cmake

@@ -84,5 +84,5 @@ if (CMAKE_CROSSCOMPILING)
         message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
     endif ()
 
-    message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}")
+    message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILER_TARGET}")
 endif ()

docker/keeper/Dockerfile

@@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.6.1.4423"
+ARG VERSION="24.6.2.17"
 ARG PACKAGES="clickhouse-keeper"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/reqgenerator.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# To run this script you must install docker and piddeptree python package
+#
+
+import subprocess
+import os
+import sys
+
+
+def build_docker_deps(image_name, imagedir):
+    cmd = f"""docker run  --entrypoint "/bin/bash" {image_name} -c "pip install pipdeptree 2>/dev/null 1>/dev/null && pipdeptree --freeze  --warn silence | sed 's/ \+//g' | sort | uniq" > {imagedir}/requirements.txt"""
+    subprocess.check_call(cmd, shell=True)
+
+
+def check_docker_file_install_with_pip(filepath):
+    image_name = None
+    with open(filepath, "r") as f:
+        for line in f:
+            if "docker build" in line:
+                arr = line.split(" ")
+                if len(arr) > 4:
+                    image_name = arr[4]
+            if "pip3 install" in line or "pip install" in line:
+                return image_name, True
+    return image_name, False
+
+
+def process_affected_images(images_dir):
+    for root, _dirs, files in os.walk(images_dir):
+        for f in files:
+            if f == "Dockerfile":
+                docker_file_path = os.path.join(root, f)
+                print("Checking image on path", docker_file_path)
+                image_name, has_pip = check_docker_file_install_with_pip(
+                    docker_file_path
+                )
+                if has_pip:
+                    print("Find pip in", image_name)
+                    try:
+                        build_docker_deps(image_name, root)
+                    except Exception as ex:
+                        print(ex)
+                else:
+                    print("Pip not found in", docker_file_path)
+
+
+process_affected_images(sys.argv[1])

docker/server/Dockerfile.alpine

@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.6.1.4423"
+ARG VERSION="24.6.2.17"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/server/Dockerfile.ubuntu

@@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="24.6.1.4423"
+ARG VERSION="24.6.2.17"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 #docker-official-library:off

docker/test/fasttest/Dockerfile

@@ -19,10 +19,7 @@ RUN apt-get update \
         odbcinst \
         psmisc \
         python3 \
-        python3-lxml \
         python3-pip \
-        python3-requests \
-        python3-termcolor \
         unixodbc \
         pv \
         jq \
@@ -31,7 +28,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r /requirements.txt
 
 # This symlink is required by gcc to find the lld linker
 RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
@@ -39,6 +37,10 @@ RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
 # https://salsa.debian.org/pkg-llvm-team/llvm-toolchain/-/commit/992e52c0b156a5ba9c6a8a54f8c4857ddd3d371d
 RUN sed -i '/_IMPORT_CHECK_FILES_FOR_\(mlir-\|llvm-bolt\|merge-fdata\|MLIR\)/ {s|^|#|}' /usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/LLVMExports-*.cmake
 
+# LLVM changes paths for compiler-rt libraries. For some reason clang-18.1.8 cannot catch up libraries from default install path.
+# It's very dirty workaround, better to build compiler and LLVM ourself and use it. Details: https://github.com/llvm/llvm-project/issues/95792
+RUN test ! -d /usr/lib/llvm-18/lib/clang/18/lib/x86_64-pc-linux-gnu || ln -s /usr/lib/llvm-18/lib/clang/18/lib/x86_64-pc-linux-gnu /usr/lib/llvm-18/lib/clang/18/lib/x86_64-unknown-linux-gnu
+
 ARG CCACHE_VERSION=4.6.1
 RUN mkdir /tmp/ccache \
     && cd /tmp/ccache \

docker/test/fasttest/requirements.txt

@@ -0,0 +1,41 @@
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+PyJWT==2.3.0
+PyYAML==6.0.1
+Pygments==2.11.2
+SecretStorage==3.3.1
+blinker==1.4
+certifi==2020.6.20
+chardet==4.0.0
+cryptography==3.4.8
+dbus-python==1.2.18
+distro==1.7.0
+httplib2==0.20.2
+idna==3.3
+importlib-metadata==4.6.4
+jeepney==0.7.1
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+lxml==4.8.0
+more-itertools==8.10.0
+numpy==1.26.3
+oauthlib==3.2.0
+packaging==24.1
+pandas==1.5.3
+pip==24.1.1
+pipdeptree==2.23.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu3
+python-dateutil==2.9.0.post0
+pytz==2024.1
+requests==2.32.3
+scipy==1.12.0
+setuptools==59.6.0
+six==1.16.0
+termcolor==1.1.0
+urllib3==1.26.5
+wadllib==1.3.6
+wheel==0.37.1
+zipp==1.0.0

docker/test/fuzzer/Dockerfile

@@ -31,7 +31,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-RUN pip3 install Jinja2
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r /requirements.txt
 
 COPY * /
 

docker/test/fuzzer/requirements.txt

@@ -0,0 +1,27 @@
+blinker==1.4
+cryptography==3.4.8
+dbus-python==1.2.18
+distro==1.7.0
+httplib2==0.20.2
+importlib-metadata==4.6.4
+jeepney==0.7.1
+Jinja2==3.1.4
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+MarkupSafe==2.1.5
+more-itertools==8.10.0
+oauthlib==3.2.0
+packaging==24.1
+pip==24.1.1
+pipdeptree==2.23.0
+PyJWT==2.3.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu3
+SecretStorage==3.3.1
+setuptools==59.6.0
+six==1.16.0
+wadllib==1.3.6
+wheel==0.37.1
+zipp==1.0.0

docker/test/integration/base/Dockerfile

@@ -33,7 +33,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-RUN pip3 install pycurl
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r requirements.txt && rm -rf /root/.cache/pip
 
 # Architecture of the image when BuildKit/buildx is used
 ARG TARGETARCH

docker/test/integration/base/requirements.txt

@@ -0,0 +1,26 @@
+blinker==1.4
+cryptography==3.4.8
+dbus-python==1.2.18
+distro==1.7.0
+httplib2==0.20.2
+importlib-metadata==4.6.4
+jeepney==0.7.1
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+more-itertools==8.10.0
+oauthlib==3.2.0
+packaging==24.1
+pip==24.1.1
+pipdeptree==2.23.0
+pycurl==7.45.3
+PyJWT==2.3.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu3
+SecretStorage==3.3.1
+setuptools==59.6.0
+six==1.16.0
+wadllib==1.3.6
+wheel==0.37.1
+zipp==1.0.0

docker/test/integration/resolver/Dockerfile

@@ -2,4 +2,5 @@
 # Helper docker container to run python bottle apps
 
 FROM python:3
-RUN python -m pip install bottle
+COPY requirements.txt /
+RUN python -m pip install --no-cache-dir -r requirements.txt

docker/test/integration/resolver/requirements.txt

@@ -0,0 +1,6 @@
+bottle==0.12.25
+packaging==24.1
+pip==23.2.1
+pipdeptree==2.23.0
+setuptools==69.0.3
+wheel==0.42.0

docker/test/integration/runner/Dockerfile

@@ -26,7 +26,6 @@ RUN apt-get update \
     libicu-dev \
     bsdutils \
     curl \
-    python3-pika \
     liblua5.1-dev \
     luajit \
     libssl-dev \
@@ -61,49 +60,8 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
 
 # kazoo 2.10.0 is broken
 # https://s3.amazonaws.com/clickhouse-test-reports/59337/524625a1d2f4cc608a3f1059e3df2c30f353a649/integration_tests__asan__analyzer__[5_6].html
-RUN python3 -m pip install --no-cache-dir \
+COPY requirements.txt /
-    PyMySQL==1.1.0 \
+RUN python3 -m pip install --no-cache-dir -r requirements.txt
-    asyncio==3.4.3 \
-    avro==1.10.2 \
-    azure-storage-blob==12.19.0 \
-    boto3==1.34.24 \
-    cassandra-driver==3.29.0 \
-    confluent-kafka==2.3.0 \
-    delta-spark==2.3.0 \
-    dict2xml==1.7.4 \
-    dicttoxml==1.7.16 \
-    docker==6.1.3 \
-    docker-compose==1.29.2 \
-    grpcio==1.60.0 \
-    grpcio-tools==1.60.0 \
-    kafka-python==2.0.2 \
-    lz4==4.3.3 \
-    minio==7.2.3 \
-    nats-py==2.6.0 \
-    protobuf==4.25.2 \
-    kazoo==2.9.0 \
-    psycopg2-binary==2.9.6 \
-    pyhdfs==0.3.1 \
-    pymongo==3.11.0 \
-    pyspark==3.3.2 \
-    pytest==7.4.4 \
-    pytest-order==1.0.0 \
-    pytest-random==0.2 \
-    pytest-repeat==0.9.3 \
-    pytest-timeout==2.2.0 \
-    pytest-xdist==3.5.0 \
-    pytest-reportlog==0.4.0 \
-    pytz==2023.3.post1 \
-    pyyaml==5.3.1 \
-    redis==5.0.1 \
-    requests-kerberos==0.14.0 \
-    tzlocal==2.1 \
-    retry==0.9.2 \
-    bs4==0.0.2 \
-    lxml==5.1.0 \
-    urllib3==2.0.7 \
-    jwcrypto==1.5.6
-# bs4, lxml are for cloud tests, do not delete
 
 # Hudi supports only spark 3.3.*, not 3.4
 RUN curl -fsSL -O https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \

docker/test/integration/runner/requirements.txt

@@ -0,0 +1,113 @@
+PyHDFS==0.3.1
+PyJWT==2.3.0
+PyMySQL==1.1.0
+PyNaCl==1.5.0
+PyYAML==5.3.1
+SecretStorage==3.3.1
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+async-timeout==4.0.3
+asyncio==3.4.3
+attrs==23.2.0
+avro==1.10.2
+azure-core==1.30.1
+azure-storage-blob==12.19.0
+bcrypt==4.1.3
+beautifulsoup4==4.12.3
+blinker==1.4
+boto3==1.34.24
+botocore==1.34.101
+bs4==0.0.2
+cassandra-driver==3.29.0
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+confluent-kafka==2.3.0
+cryptography==3.4.8
+dbus-python==1.2.18
+decorator==5.1.1
+delta-spark==2.3.0
+dict2xml==1.7.4
+dicttoxml==1.7.16
+distro-info==1.1+ubuntu0.2
+distro==1.7.0
+docker-compose==1.29.2
+docker==6.1.3
+dockerpty==0.4.1
+docopt==0.6.2
+exceptiongroup==1.2.1
+execnet==2.1.1
+geomet==0.2.1.post1
+grpcio-tools==1.60.0
+grpcio==1.60.0
+gssapi==1.8.3
+httplib2==0.20.2
+idna==3.7
+importlib-metadata==4.6.4
+iniconfig==2.0.0
+isodate==0.6.1
+jeepney==0.7.1
+jmespath==1.0.1
+jsonschema==3.2.0
+jwcrypto==1.5.6
+kafka-python==2.0.2
+kazoo==2.9.0
+keyring==23.5.0
+krb5==0.5.1
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+lxml==5.1.0
+lz4==4.3.3
+minio==7.2.3
+more-itertools==8.10.0
+nats-py==2.6.0
+oauthlib==3.2.0
+packaging==24.0
+paramiko==3.4.0
+pika==1.2.0
+pip==24.1.1
+pipdeptree==2.23.0
+pluggy==1.5.0
+protobuf==4.25.2
+psycopg2-binary==2.9.6
+py4j==0.10.9.5
+py==1.11.0
+pycparser==2.22
+pycryptodome==3.20.0
+pymongo==3.11.0
+pyparsing==2.4.7
+pyrsistent==0.20.0
+pyspark==3.3.2
+pyspnego==0.10.2
+pytest-order==1.0.0
+pytest-random==0.2
+pytest-repeat==0.9.3
+pytest-reportlog==0.4.0
+pytest-timeout==2.2.0
+pytest-xdist==3.5.0
+pytest==7.4.4
+python-apt==2.4.0+ubuntu3
+python-dateutil==2.9.0.post0
+python-dotenv==0.21.1
+pytz==2023.3.post1
+redis==5.0.1
+requests-kerberos==0.14.0
+requests==2.31.0
+retry==0.9.2
+s3transfer==0.10.1
+setuptools==59.6.0
+simplejson==3.19.2
+six==1.16.0
+soupsieve==2.5
+texttable==1.7.0
+tomli==2.0.1
+typing_extensions==4.11.0
+tzlocal==2.1
+unattended-upgrades==0.1
+urllib3==2.0.7
+wadllib==1.3.6
+websocket-client==0.59.0
+wheel==0.37.1
+zipp==1.0.0

docker/test/libfuzzer/Dockerfile

@@ -1,3 +1,4 @@
+# docker build -t clickhouse/libfuzzer .
 ARG FROM_TAG=latest
 FROM clickhouse/test-base:$FROM_TAG
 
@@ -29,7 +30,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-RUN pip3 install Jinja2
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r /requirements.txt
 
 COPY * /
 

docker/test/libfuzzer/requirements.txt

@@ -0,0 +1,27 @@
+blinker==1.4
+cryptography==3.4.8
+dbus-python==1.2.18
+distro==1.7.0
+httplib2==0.20.2
+importlib-metadata==4.6.4
+jeepney==0.7.1
+Jinja2==3.1.4
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+MarkupSafe==2.1.5
+more-itertools==8.10.0
+oauthlib==3.2.0
+packaging==24.1
+pip==24.1.1
+pipdeptree==2.23.0
+PyJWT==2.3.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu3
+SecretStorage==3.3.1
+setuptools==59.6.0
+six==1.16.0
+wadllib==1.3.6
+wheel==0.37.1
+zipp==1.0.0

docker/test/performance-comparison/Dockerfile

@@ -23,7 +23,6 @@ RUN apt-get update \
             python3 \
             python3-dev \
             python3-pip \
-            python3-setuptools \
             rsync \
             tree \
             tzdata \
@@ -33,12 +32,14 @@ RUN apt-get update \
             cargo \
             ripgrep \
             zstd \
-    && pip3 --no-cache-dir install 'clickhouse-driver==0.2.1' scipy \
     && apt-get purge --yes python3-dev g++ \
     && apt-get autoremove --yes \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
+COPY requirements.txt /
+RUN pip3 --no-cache-dir install -r requirements.txt
+
 COPY run.sh /
 
 CMD ["bash", "/run.sh"]

docker/test/performance-comparison/requirements.txt

@@ -0,0 +1,32 @@
+blinker==1.4
+clickhouse-driver==0.2.7
+cryptography==3.4.8
+dbus-python==1.2.18
+distro==1.7.0
+httplib2==0.20.2
+importlib-metadata==4.6.4
+jeepney==0.7.1
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+more-itertools==8.10.0
+numpy==1.26.3
+oauthlib==3.2.0
+packaging==24.1
+pip==24.1.1
+pipdeptree==2.23.0
+Pygments==2.11.2
+PyJWT==2.3.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu3
+pytz==2023.4
+PyYAML==6.0.1
+scipy==1.12.0
+SecretStorage==3.3.1
+setuptools==59.6.0
+six==1.16.0
+tzlocal==2.1
+wadllib==1.3.6
+wheel==0.37.1
+zipp==1.0.0

docker/test/sqllogic/Dockerfile

@@ -18,11 +18,8 @@ RUN apt-get update --yes \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-RUN pip3 install \
+COPY requirements.txt /
-    numpy \
+RUN pip3 install --no-cache-dir -r /requirements.txt
-    pyodbc \
-    deepdiff \
-    sqlglot
 
 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
 

docker/test/sqllogic/requirements.txt

@@ -0,0 +1,30 @@
+blinker==1.4
+cryptography==3.4.8
+dbus-python==1.2.18
+deepdiff==7.0.1
+distro==1.7.0
+httplib2==0.20.2
+importlib-metadata==4.6.4
+jeepney==0.7.1
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+more-itertools==8.10.0
+numpy==1.26.4
+oauthlib==3.2.0
+ordered-set==4.1.0
+packaging==24.1
+pip==24.1.1
+pipdeptree==2.23.0
+PyJWT==2.3.0
+pyodbc==5.1.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu3
+SecretStorage==3.3.1
+setuptools==59.6.0
+six==1.16.0
+sqlglot==23.16.0
+wadllib==1.3.6
+wheel==0.37.1
+zipp==1.0.0

docker/test/sqltest/Dockerfile

@@ -14,9 +14,8 @@ RUN apt-get update --yes \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-RUN pip3 install \
+COPY requirements.txt /
-    pyyaml \
+RUN pip3 install --no-cache-dir -r /requirements.txt
-    clickhouse-driver
 
 ARG sqltest_repo="https://github.com/elliotchance/sqltest/"
 

docker/test/sqltest/requirements.txt

@@ -0,0 +1,29 @@
+blinker==1.4
+clickhouse-driver==0.2.7
+cryptography==3.4.8
+dbus-python==1.2.18
+distro==1.7.0
+httplib2==0.20.2
+importlib-metadata==4.6.4
+jeepney==0.7.1
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+more-itertools==8.10.0
+oauthlib==3.2.0
+packaging==24.1
+pip==24.1.1
+pipdeptree==2.23.0
+PyJWT==2.3.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu3
+pytz==2024.1
+PyYAML==6.0.1
+SecretStorage==3.3.1
+setuptools==59.6.0
+six==1.16.0
+tzlocal==5.2
+wadllib==1.3.6
+wheel==0.37.1
+zipp==1.0.0

docker/test/stateful/Dockerfile

@@ -6,7 +6,6 @@ FROM clickhouse/stateless-test:$FROM_TAG
 RUN apt-get update -y \
     && env DEBIAN_FRONTEND=noninteractive \
         apt-get install --yes --no-install-recommends \
-        python3-requests \
         nodejs \
         npm \
     && apt-get clean \

docker/test/stateless/Dockerfile

@@ -25,10 +25,7 @@ RUN apt-get update -y \
             openssl \
             postgresql-client \
             python3 \
-            python3-lxml \
             python3-pip \
-            python3-requests \
-            python3-termcolor \
             qemu-user-static \
             sqlite3 \
             sudo \
@@ -51,7 +48,8 @@ RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PR
     && unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local \
     && rm protoc-${PROTOC_VERSION}-linux-x86_64.zip
 
-RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3 pyarrow==15.0.0
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r /requirements.txt
 
 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
   && cd /tmp/clickhouse-odbc-tmp \

docker/test/stateless/requirements.txt

@@ -0,0 +1,51 @@
+awscli==1.22.34
+blinker==1.4
+botocore==1.23.34
+certifi==2020.6.20
+chardet==4.0.0
+colorama==0.4.4
+cryptography==3.4.8
+dbus-python==1.2.18
+distro==1.7.0
+docutils==0.17.1
+gyp==0.1
+httplib2==0.20.2
+idna==3.3
+importlib-metadata==4.6.4
+jeepney==0.7.1
+Jinja2==3.1.3
+jmespath==0.10.0
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+lxml==4.8.0
+MarkupSafe==2.1.5
+more-itertools==8.10.0
+numpy==1.26.3
+oauthlib==3.2.0
+packaging==24.1
+pandas==1.5.3
+pip==24.1.1
+pipdeptree==2.23.0
+pyarrow==15.0.0
+pyasn1==0.4.8
+PyJWT==2.3.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu3
+python-dateutil==2.8.1
+pytz==2024.1
+PyYAML==6.0.1
+requests==2.32.3
+roman==3.3
+rsa==4.8
+s3transfer==0.5.0
+scipy==1.12.0
+SecretStorage==3.3.1
+setuptools==59.6.0
+six==1.16.0
+termcolor==1.1.0
+urllib3==1.26.5
+wadllib==1.3.6
+wheel==0.37.1
+zipp==1.0.0

docker/test/style/Dockerfile

@@ -23,22 +23,8 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
 # python-magic is the same version as in Ubuntu 22.04
-RUN pip3 install \
+COPY requirements.txt /
-        PyGithub \
+RUN pip3 install --no-cache-dir -r requirements.txt
-        black==23.12.0 \
-        boto3 \
-        codespell==2.2.1 \
-        mypy==1.8.0 \
-        pylint==3.1.0 \
-        python-magic==0.4.24 \
-        flake8==4.0.1 \
-        requests \
-        thefuzz \
-        tqdm==4.66.4 \
-        types-requests \
-        unidiff \
-        jwt \
-    && rm -rf /root/.cache/pip
 
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8
 ENV LC_ALL en_US.UTF-8

docker/test/style/requirements.txt

@@ -0,0 +1,58 @@
+aiohttp==3.9.5
+aiosignal==1.3.1
+astroid==3.1.0
+async-timeout==4.0.3
+attrs==23.2.0
+black==23.12.0
+boto3==1.34.131
+botocore==1.34.131
+certifi==2024.6.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+codespell==2.2.1
+cryptography==42.0.8
+Deprecated==1.2.14
+dill==0.3.8
+flake8==4.0.1
+frozenlist==1.4.1
+idna==3.7
+isort==5.13.2
+jmespath==1.0.1
+jwt==1.3.1
+mccabe==0.6.1
+multidict==6.0.5
+mypy==1.8.0
+mypy-extensions==1.0.0
+packaging==24.1
+pathspec==0.9.0
+pip==24.1.1
+pipdeptree==2.23.0
+platformdirs==4.2.2
+pycodestyle==2.8.0
+pycparser==2.22
+pyflakes==2.4.0
+PyGithub==2.3.0
+PyJWT==2.8.0
+pylint==3.1.0
+PyNaCl==1.5.0
+python-dateutil==2.9.0.post0
+python-magic==0.4.24
+PyYAML==6.0.1
+rapidfuzz==3.9.3
+requests==2.32.3
+s3transfer==0.10.1
+setuptools==59.6.0
+six==1.16.0
+thefuzz==0.22.1
+tomli==2.0.1
+tomlkit==0.12.5
+tqdm==4.66.4
+types-requests==2.32.0.20240622
+typing_extensions==4.12.2
+unidiff==0.7.5
+urllib3==2.2.2
+wheel==0.37.1
+wrapt==1.16.0
+yamllint==1.26.3
+yarl==1.9.4

docs/changelogs/v24.6.2.17-stable.md

@@ -0,0 +1,26 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.6.2.17-stable (5710a8b5c0c) FIXME as compared to v24.6.1.4423-stable (dcced7c8478)
+
+#### New Feature
+* Backported in [#66002](https://github.com/ClickHouse/ClickHouse/issues/66002): Add AzureQueue storage. [#65458](https://github.com/ClickHouse/ClickHouse/pull/65458) ([Kseniia Sumarokova](https://github.com/kssenii)).
+
+#### Improvement
+* Backported in [#65898](https://github.com/ClickHouse/ClickHouse/issues/65898): Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#65935](https://github.com/ClickHouse/ClickHouse/issues/65935): For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65907](https://github.com/ClickHouse/ClickHouse/issues/65907): Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65962](https://github.com/ClickHouse/ClickHouse/issues/65962): Add missing workload identity changes. [#65848](https://github.com/ClickHouse/ClickHouse/pull/65848) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Backported in [#66033](https://github.com/ClickHouse/ClickHouse/issues/66033): Follow up to [#65046](https://github.com/ClickHouse/ClickHouse/issues/65046). [#65928](https://github.com/ClickHouse/ClickHouse/pull/65928) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#66076](https://github.com/ClickHouse/ClickHouse/issues/66076): Fix support of non-const scale arguments in rounding functions. [#65983](https://github.com/ClickHouse/ClickHouse/pull/65983) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
+* Backported in [#66017](https://github.com/ClickHouse/ClickHouse/issues/66017): Fix race in s3queue. [#65986](https://github.com/ClickHouse/ClickHouse/pull/65986) ([Kseniia Sumarokova](https://github.com/kssenii)).
+

docs/en/sql-reference/aggregate-functions/reference/aggthrow.md

@@ -0,0 +1,37 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/aggthrow
+sidebar_position: 101
+---
+
+# aggThrow
+
+This function can be used for the purpose of testing exception safety. It will throw an exception on creation with the specified probability.
+
+**Syntax**
+
+```sql
+aggThrow(throw_prob)
+```
+
+**Arguments**
+
+- `throw_prob` — Probability to throw on creation. [Float64](../../data-types/float.md).
+
+**Returned value**
+
+- An exception: `Code: 503. DB::Exception: Aggregate function aggThrow has thrown exception successfully`.
+
+**Example**
+
+Query:
+
+```sql
+SELECT number % 2 AS even, aggThrow(number) FROM numbers(10) GROUP BY even;
+```
+
+Result:
+
+```response
+Received exception:
+Code: 503. DB::Exception: Aggregate function aggThrow has thrown exception successfully: While executing AggregatingTransform. (AGGREGATE_FUNCTION_THROW)
+```

docs/en/sql-reference/aggregate-functions/reference/index.md

@@ -43,6 +43,7 @@ Standard aggregate functions:
 
 ClickHouse-specific aggregate functions:
 
+- [aggThrow](../reference/aggthrow.md)
 - [analysisOfVariance](../reference/analysis_of_variance.md)
 - [any](../reference/any_respect_nulls.md)
 - [anyHeavy](../reference/anyheavy.md)

docs/en/sql-reference/functions/other-functions.md

@@ -86,7 +86,7 @@ Returns the fully qualified domain name of the ClickHouse server.
 fqdn();
 ```
 
-This function is case-insensitive.
+Aliases: `fullHostName`, 'FQDN'. 
 
 **Returned value**
 

src/Columns/ColumnTuple.cpp

@@ -711,7 +711,13 @@ void ColumnTuple::takeDynamicStructureFromSourceColumns(const Columns & source_c
 ColumnPtr ColumnTuple::compress() const
 {
     if (columns.empty())
-        return Ptr();
+    {
+        return ColumnCompressed::create(size(), 0,
+            [n = column_length]
+            {
+                return ColumnTuple::create(n);
+            });
+    }
 
     size_t byte_size = 0;
     Columns compressed;

src/Common/BinStringDecodeHelper.h

@@ -5,7 +5,7 @@
 namespace DB
 {
 
-static void inline hexStringDecode(const char * pos, const char * end, char *& out, size_t word_size = 2)
+static void inline hexStringDecode(const char * pos, const char * end, char *& out, size_t word_size)
 {
     if ((end - pos) & 1)
     {
@@ -23,7 +23,7 @@ static void inline hexStringDecode(const char * pos, const char * end, char *& o
     ++out;
 }
 
-static void inline binStringDecode(const char * pos, const char * end, char *& out)
+static void inline binStringDecode(const char * pos, const char * end, char *& out, size_t word_size)
 {
     if (pos == end)
     {
@@ -53,7 +53,7 @@ static void inline binStringDecode(const char * pos, const char * end, char *& o
         ++out;
     }
 
-    assert((end - pos) % 8 == 0);
+    chassert((end - pos) % word_size == 0);
 
     while (end - pos != 0)
     {

src/Functions/FunctionsBinaryRepresentation.cpp

@@ -3,14 +3,14 @@
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnVector.h>
 #include <Columns/ColumnsNumber.h>
-#include <Common/BitHelpers.h>
-#include <Common/BinStringDecodeHelper.h>
 #include <DataTypes/DataTypeString.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/IFunction.h>
 #include <IO/WriteHelpers.h>
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/castColumn.h>
+#include <Common/BinStringDecodeHelper.h>
+#include <Common/BitHelpers.h>
 
 namespace DB
 {
@@ -218,10 +218,7 @@ struct UnbinImpl
     static constexpr auto name = "unbin";
     static constexpr size_t word_size = 8;
 
-    static void decode(const char * pos, const char * end, char *& out)
+    static void decode(const char * pos, const char * end, char *& out) { binStringDecode(pos, end, out, word_size); }
-    {
-        binStringDecode(pos, end, out);
-    }
 };
 
 /// Encode number or string to string with binary or hexadecimal representation
@@ -651,7 +648,15 @@ public:
 
             size_t size = in_offsets.size();
             out_offsets.resize(size);
-            out_vec.resize(in_vec.size() / word_size + size);
+
+            size_t max_out_len = 0;
+            for (size_t i = 0; i < in_offsets.size(); ++i)
+            {
+                const size_t len = in_offsets[i] - (i == 0 ? 0 : in_offsets[i - 1])
+                    - /* trailing zero symbol that is always added in ColumnString and that is ignored while decoding */ 1;
+                max_out_len += (len + word_size - 1) / word_size + /* trailing zero symbol that is always added by Impl::decode */ 1;
+            }
+            out_vec.resize(max_out_len);
 
             char * begin = reinterpret_cast<char *>(out_vec.data());
             char * pos = begin;
@@ -661,6 +666,7 @@ public:
             {
                 size_t new_offset = in_offsets[i];
 
+                /// `new_offset - 1` because in ColumnString each string is stored with trailing zero byte
                 Impl::decode(reinterpret_cast<const char *>(&in_vec[prev_offset]), reinterpret_cast<const char *>(&in_vec[new_offset - 1]), pos);
 
                 out_offsets[i] = pos - begin;
@@ -668,6 +674,9 @@ public:
                 prev_offset = new_offset;
             }
 
+            chassert(
+                static_cast<size_t>(pos - begin) <= out_vec.size(),
+                fmt::format("too small amount of memory was preallocated: needed {}, but have only {}", pos - begin, out_vec.size()));
             out_vec.resize(pos - begin);
 
             return col_res;
@@ -680,11 +689,11 @@ public:
             ColumnString::Offsets & out_offsets = col_res->getOffsets();
 
             const ColumnString::Chars & in_vec = col_fix_string->getChars();
-            size_t n = col_fix_string->getN();
+            const size_t n = col_fix_string->getN();
 
             size_t size = col_fix_string->size();
             out_offsets.resize(size);
-            out_vec.resize(in_vec.size() / word_size + size);
+            out_vec.resize(((n + word_size - 1) / word_size + /* trailing zero symbol that is always added by Impl::decode */ 1) * size);
 
             char * begin = reinterpret_cast<char *>(out_vec.data());
             char * pos = begin;
@@ -694,6 +703,7 @@ public:
             {
                 size_t new_offset = prev_offset + n;
 
+                /// here we don't subtract 1 from `new_offset` because in ColumnFixedString strings are stored without trailing zero byte
                 Impl::decode(reinterpret_cast<const char *>(&in_vec[prev_offset]), reinterpret_cast<const char *>(&in_vec[new_offset]), pos);
 
                 out_offsets[i] = pos - begin;
@@ -701,6 +711,9 @@ public:
                 prev_offset = new_offset;
             }
 
+            chassert(
+                static_cast<size_t>(pos - begin) <= out_vec.size(),
+                fmt::format("too small amount of memory was preallocated: needed {}, but have only {}", pos - begin, out_vec.size()));
             out_vec.resize(pos - begin);
 
             return col_res;

src/IO/BufferWithOwnMemory.h

@@ -44,7 +44,7 @@ struct Memory : boost::noncopyable, Allocator
     char * m_data = nullptr;
     size_t alignment = 0;
 
-    [[maybe_unused]] bool allow_gwp_asan_force_sample;
+    [[maybe_unused]] bool allow_gwp_asan_force_sample{false};
 
     Memory() = default;
 

src/Parsers/ExpressionElementParsers.cpp

@@ -1129,11 +1129,11 @@ inline static bool makeHexOrBinStringLiteral(IParser::Pos & pos, ASTPtr & node,
 
     if (hex)
     {
-        hexStringDecode(str_begin, str_end, res_pos);
+        hexStringDecode(str_begin, str_end, res_pos, word_size);
     }
     else
     {
-        binStringDecode(str_begin, str_end, res_pos);
+        binStringDecode(str_begin, str_end, res_pos, word_size);
     }
 
     return makeStringLiteral(pos, node, String(reinterpret_cast<char *>(res.data()), (res_pos - res_begin - 1)));

tests/queries/0_stateless/01563_distributed_query_finish.reference

@@ -1,2 +1 @@
-1,0
 NETWORK_ERROR=0

tests/queries/0_stateless/01563_distributed_query_finish.sh

@@ -19,20 +19,25 @@ create table dist_01247 as data_01247 engine=Distributed(test_cluster_two_shards
 select * from dist_01247 format Null;
 EOL
 
-network_errors_before=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.errors WHERE name = 'NETWORK_ERROR'")
+# NOTE: it is possible to got NETWORK_ERROR even with no-parallel, at least due to system.*_log_sender to the cloud
+for ((i = 0; i < 100; ++i)); do
+    network_errors_before=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.errors WHERE name = 'NETWORK_ERROR'")
 
-opts=(
+    opts=(
-    "--max_distributed_connections=1"
+        "--max_distributed_connections=1"
-    "--optimize_skip_unused_shards=1"
+        "--optimize_skip_unused_shards=1"
-    "--optimize_distributed_group_by_sharding_key=1"
+        "--optimize_distributed_group_by_sharding_key=1"
-    "--prefer_localhost_replica=0"
+        "--prefer_localhost_replica=0"
-)
+    )
-$CLICKHOUSE_CLIENT "${opts[@]}" --format CSV -nm <<EOL
+    $CLICKHOUSE_CLIENT "${opts[@]}" --format CSV -nm -q "select count(), * from dist_01247 group by number order by number limit 1 format Null"
-select count(), * from dist_01247 group by number order by number limit 1;
-EOL
 
-# expect zero new network errors
+    # expect zero new network errors
-network_errors_after=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.errors WHERE name = 'NETWORK_ERROR'")
+    network_errors_after=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.errors WHERE name = 'NETWORK_ERROR'")
+
+    if [[ $((network_errors_after-network_errors_before)) -eq 0 ]]; then
+        break
+    fi
+done
 echo NETWORK_ERROR=$(( network_errors_after-network_errors_before ))
 
 $CLICKHOUSE_CLIENT -q "drop table data_01247"

tests/queries/0_stateless/03199_unbin_buffer_overflow.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+# check for buffer overflow in unbin (due to not enough memory preallocate for output buffer)
+# we iterate over all remainders of input string length modulo word_size and check that no assertions are triggered
+
+word_size=8
+for i in $(seq 1 $((word_size+1))); do
+  str=$(printf "%${i}s" | tr ' ' 'x')
+  $CLICKHOUSE_CLIENT -q "SELECT count() FROM numbers(99) GROUP BY unbin(toFixedString(materialize('$str'), $i)) WITH ROLLUP WITH TOTALS FORMAT NULL"
+done
+
+word_size=8
+for i in $(seq 1 $((word_size+1))); do
+  str=$(printf "%${i}s" | tr ' ' 'x')
+  $CLICKHOUSE_CLIENT -q "SELECT count() FROM numbers(99) GROUP BY unbin(materialize('$str')) WITH ROLLUP WITH TOTALS FORMAT NULL"
+done
+
+word_size=2
+for i in $(seq 1 $((word_size+1))); do
+  str=$(printf "%${i}s" | tr ' ' 'x')
+  $CLICKHOUSE_CLIENT -q "SELECT count() FROM numbers(99) GROUP BY unhex(toFixedString(materialize('$str'), $i)) WITH ROLLUP WITH TOTALS FORMAT NULL"
+done
+
+word_size=2
+for i in $(seq 1 $((word_size+1))); do
+  str=$(printf "%${i}s" | tr ' ' 'x')
+  $CLICKHOUSE_CLIENT -q "SELECT count() FROM numbers(99) GROUP BY unhex(materialize('$str')) WITH ROLLUP WITH TOTALS FORMAT NULL"
+done

tests/queries/0_stateless/03201_query_cache_empty_tuple.reference

@@ -0,0 +1,2 @@
+()	0
+()	0

tests/queries/0_stateless/03201_query_cache_empty_tuple.sql

@@ -0,0 +1,2 @@
+SELECT tuple(), 0 FROM numbers(1) SETTINGS use_query_cache = true;
+SELECT tuple(), 0 FROM numbers(1) SETTINGS use_query_cache = true;

utils/check-style/aspell-ignore/en/aspell-dict.txt

@@ -1098,6 +1098,8 @@ aggregatefunction
 aggregatingmergetree
 aggregatio
 aggretate
+aggthrow
+aggThrow
 aiochclient
 allocator
 alphaTokens

utils/list-versions/version_date.tsv

@@ -1,3 +1,4 @@
+v24.6.2.17-stable	2024-07-05
 v24.6.1.4423-stable	2024-07-01
 v24.5.4.49-stable	2024-07-01
 v24.5.3.5-stable	2024-06-13
@@ -6,6 +7,7 @@ v24.5.1.1763-stable	2024-06-01
 v24.4.3.25-stable	2024-06-14
 v24.4.2.141-stable	2024-06-07
 v24.4.1.2088-stable	2024-05-01
+v24.3.5.46-lts	2024-07-03
 v24.3.4.147-lts	2024-06-13
 v24.3.3.102-lts	2024-05-01
 v24.3.2.23-lts	2024-04-03