From cd5852f11a7611fb2199bbfc6adcbdbecd9a6287 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 9 Feb 2023 13:11:33 +0100
Subject: [PATCH 01/16] Add typing and rewrite packager to pathlib.Path

---
 docker/packager/packager | 47 +++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index 58dd299fd6d..6a5b6cabce1 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -5,13 +5,14 @@ import os
 import argparse
 import logging
 import sys
+from pathlib import Path
 from typing import List
 
-SCRIPT_PATH = os.path.realpath(__file__)
+SCRIPT_PATH = Path(__file__).absolute()
 IMAGE_TYPE = "binary"
 
 
-def check_image_exists_locally(image_name):
+def check_image_exists_locally(image_name: str) -> bool:
     try:
         output = subprocess.check_output(
             f"docker images -q {image_name} 2> /dev/null", shell=True
@@ -21,7 +22,7 @@ def check_image_exists_locally(image_name):
         return False
 
 
-def pull_image(image_name):
+def pull_image(image_name: str) -> bool:
     try:
         subprocess.check_call(f"docker pull {image_name}", shell=True)
         return True
@@ -30,8 +31,8 @@ def pull_image(image_name):
         return False
 
 
-def build_image(image_name, filepath):
-    context = os.path.dirname(filepath)
+def build_image(image_name: str, filepath: Path) -> None:
+    context = filepath.parent
     build_cmd = f"docker build --network=host -t {image_name} -f {filepath} {context}"
     logging.info("Will build image with cmd: '%s'", build_cmd)
     subprocess.check_call(
@@ -40,7 +41,7 @@ def build_image(image_name, filepath):
     )
 
 
-def pre_build(repo_path: str, env_variables: List[str]):
+def pre_build(repo_path: Path, env_variables: List[str]):
     if "WITH_PERFORMANCE=1" in env_variables:
         current_branch = subprocess.check_output(
             "git branch --show-current", shell=True, encoding="utf-8"
@@ -67,14 +68,15 @@ def pre_build(repo_path: str, env_variables: List[str]):
 
 
 def run_docker_image_with_env(
-    image_name,
-    as_root,
-    output,
-    env_variables,
-    ch_root,
-    ccache_dir,
-    docker_image_version,
+    image_name: str,
+    as_root: bool,
+    output_dir: Path,
+    env_variables: List[str],
+    ch_root: Path,
+    ccache_dir: Path,
+    docker_image_version: str,
 ):
+    output_dir.mkdir(parents=True, exist_ok=True)
     env_part = " -e ".join(env_variables)
     if env_part:
         env_part = " -e " + env_part
@@ -90,7 +92,7 @@ def run_docker_image_with_env(
         user = f"{os.geteuid()}:{os.getegid()}"
 
     cmd = (
-        f"docker run --network=host --user={user} --rm --volume={output}:/output "
+        f"docker run --network=host --user={user} --rm --volume={output_dir}:/output "
         f"--volume={ch_root}:/build --volume={ccache_dir}:/ccache {env_part} "
         f"{interactive} {image_name}:{docker_image_version}"
     )
@@ -100,7 +102,7 @@ def run_docker_image_with_env(
     subprocess.check_call(cmd, shell=True)
 
 
-def is_release_build(build_type, package_type, sanitizer):
+def is_release_build(build_type: str, package_type: str, sanitizer: str) -> bool:
     return build_type == "" and package_type == "deb" and sanitizer == ""
 
 
@@ -312,10 +314,11 @@ def parse_env_variables(
     return result
 
 
-def dir_name(name: str) -> str:
-    if not os.path.isabs(name):
-        name = os.path.abspath(os.path.join(os.getcwd(), name))
-    return name
+def dir_name(name: str) -> Path:
+    path = Path(name)
+    if not path.is_absolute():
+        path = Path.cwd() / name
+    return path
 
 
 if __name__ == "__main__":
@@ -331,7 +334,7 @@ if __name__ == "__main__":
     )
     parser.add_argument(
         "--clickhouse-repo-path",
-        default=os.path.join(os.path.dirname(SCRIPT_PATH), os.pardir, os.pardir),
+        default=SCRIPT_PATH.parents[2],
         type=dir_name,
         help="ClickHouse git repository",
     )
@@ -364,7 +367,7 @@ if __name__ == "__main__":
     parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
     parser.add_argument(
         "--ccache_dir",
-        default=os.getenv("HOME", "") + "/.ccache",
+        default=Path.home() / ".ccache",
         type=dir_name,
         help="a directory with ccache",
     )
@@ -400,7 +403,7 @@ if __name__ == "__main__":
     if args.with_binaries != "" and args.package_type == "deb":
         logging.info("Should place %s to output", args.with_binaries)
 
-    dockerfile = os.path.join(ch_root, "docker/packager", IMAGE_TYPE, "Dockerfile")
+    dockerfile = ch_root / "docker/packager" / IMAGE_TYPE / "Dockerfile"
     image_with_version = image_name + ":" + args.docker_image_version
     if not check_image_exists_locally(image_name) or args.force_build_image:
         if not pull_image(image_with_version) or args.force_build_image:

From 648697f085a6260713a7f86e375aa5f559bf680b Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 9 Feb 2023 13:25:56 +0100
Subject: [PATCH 02/16] Add main function, use specific exception in packager

---
 docker/packager/packager | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index 6a5b6cabce1..a11673759e7 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -12,6 +12,10 @@ SCRIPT_PATH = Path(__file__).absolute()
 IMAGE_TYPE = "binary"
 
 
+class BuildException(Exception):
+    pass
+
+
 def check_image_exists_locally(image_name: str) -> bool:
     try:
         output = subprocess.check_output(
@@ -57,7 +61,9 @@ def pre_build(repo_path: Path, env_variables: List[str]):
             # conclusion is: in the current state the easiest way to go is to force
             # unshallow repository for performance artifacts.
             # To change it we need to rework our performance tests docker image
-            raise Exception("shallow repository is not suitable for performance builds")
+            raise BuildException(
+                "shallow repository is not suitable for performance builds"
+            )
         if current_branch != "master":
             cmd = (
                 f"git -C {repo_path} fetch --no-recurse-submodules "
@@ -321,7 +327,7 @@ def dir_name(name: str) -> Path:
     return path
 
 
-if __name__ == "__main__":
+def main():
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
@@ -395,12 +401,11 @@ if __name__ == "__main__":
     ch_root = args.clickhouse_repo_path
 
     if args.additional_pkgs and args.package_type != "deb":
-        raise Exception("Can build additional packages only in deb build")
+        raise BuildException("Can build additional packages only in deb build")
 
-    if args.with_binaries != "" and args.package_type != "deb":
-        raise Exception("Can add additional binaries only in deb build")
-
-    if args.with_binaries != "" and args.package_type == "deb":
+    if args.with_binaries != "":
+        if args.package_type != "deb":
+            raise BuildException("Can add additional binaries only in deb build")
         logging.info("Should place %s to output", args.with_binaries)
 
     dockerfile = ch_root / "docker/packager" / IMAGE_TYPE / "Dockerfile"
@@ -435,3 +440,7 @@ if __name__ == "__main__":
         args.docker_image_version,
     )
     logging.info("Output placed into %s", args.output_dir)
+
+
+if __name__ == "__main__":
+    main()

From c994c41e744f4cdf254e61df6b59f8e72c0e2a60 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 9 Feb 2023 13:44:38 +0100
Subject: [PATCH 03/16] Fix logging formatting

---
 docker/packager/packager | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index a11673759e7..d59aeea50af 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -31,7 +31,7 @@ def pull_image(image_name: str) -> bool:
         subprocess.check_call(f"docker pull {image_name}", shell=True)
         return True
     except subprocess.CalledProcessError:
-        logging.info(f"Cannot pull image {image_name}".format())
+        logging.info("Cannot pull image %s", image_name)
         return False
 
 

From 612d1d88762ebecd1f8269db8ba15a73b86d1c13 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 9 Feb 2023 16:43:24 +0100
Subject: [PATCH 04/16] Install sccache in binary-builder and fasttests

---
 docker/packager/binary/Dockerfile |  1 +
 docker/packager/binary/build.sh   |  1 +
 docker/test/fasttest/Dockerfile   |  6 ------
 docker/test/fasttest/run.sh       | 11 +++++++++--
 docker/test/util/Dockerfile       | 13 +++++++++++++
 5 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 62e6d47c183..bc2327b9d12 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -69,6 +69,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
         libc6 \
         libc6-dev \
         libc6-dev-arm64-cross \
+        python3-boto3 \
         yasm \
         zstd \
     && apt-get clean \
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
index 24dca72e946..2cd0a011013 100755
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@@ -6,6 +6,7 @@ exec &> >(ts)
 ccache_status () {
     ccache --show-config ||:
     ccache --show-stats ||:
+    SCCACHE_NO_DAEMON=1 sccache --show-stats ||:
 }
 
 [ -O /build ] || git config --global --add safe.directory /build
diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile
index 32546b71eb8..ffb13fc774d 100644
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@@ -20,12 +20,6 @@ RUN apt-get update \
         zstd \
     --yes --no-install-recommends
 
-# Install CMake 3.20+ for Rust compilation
-RUN apt purge cmake --yes
-RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
-RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main'
-RUN apt update && apt install cmake --yes
-
 RUN pip3 install numpy scipy pandas Jinja2
 
 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 086276bed55..ac2eb5b403e 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -37,6 +37,13 @@ export FASTTEST_DATA
 export FASTTEST_OUT
 export PATH
 
+function ccache_status
+{
+    ccache --show-config ||:
+    ccache --show-stats ||:
+    SCCACHE_NO_DAEMON=1 sccache --show-stats ||:
+}
+
 function start_server
 {
     set -m # Spawn server in its own process groups
@@ -171,7 +178,7 @@ function run_cmake
     export CCACHE_COMPILERCHECK=content
     export CCACHE_MAXSIZE=15G
 
-    ccache --show-stats ||:
+    ccache_status
     ccache --zero-stats ||:
 
     mkdir "$FASTTEST_BUILD" ||:
@@ -193,7 +200,7 @@ function build
             strip programs/clickhouse -o "$FASTTEST_OUTPUT/clickhouse-stripped"
             zstd --threads=0 "$FASTTEST_OUTPUT/clickhouse-stripped"
         fi
-        ccache --show-stats ||:
+        ccache_status
         ccache --evict-older-than 1d ||:
     )
 }
diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile
index 0ee426f4e4d..911cadc3c58 100644
--- a/docker/test/util/Dockerfile
+++ b/docker/test/util/Dockerfile
@@ -92,4 +92,17 @@ RUN mkdir /tmp/ccache \
     && cd / \
     && rm -rf /tmp/ccache
 
+ARG TARGETARCH
+ARG SCCACHE_VERSION=v0.4.1
+RUN arch=${TARGETARCH:-amd64} \
+  && case $arch in \
+    amd64) rarch=x86_64 ;; \
+    arm64) rarch=aarch64 ;; \
+  esac \
+  && curl -Ls "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl.tar.gz" | \
+    tar xz -C /tmp \
+  && mv "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl/sccache" /usr/bin \
+  && rm "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl" -r
+
+
 COPY process_functional_tests_result.py /

From e1531688bc049c654c5e59eeec0ebbc00ddfaab7 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 9 Feb 2023 16:45:57 +0100
Subject: [PATCH 05/16] Add sccache to cmake configuration

---
 cmake/ccache.cmake | 64 +++++++++++++++++++++++++++-------------------
 1 file changed, 38 insertions(+), 26 deletions(-)

diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake
index f0769f337d0..6103a5212e3 100644
--- a/cmake/ccache.cmake
+++ b/cmake/ccache.cmake
@@ -8,45 +8,57 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA
     return()
 endif()
 
-option(ENABLE_CCACHE "Speedup re-compilations using ccache (external tool)" ON)
+option(ENABLE_CCACHE "Speedup re-compilations using ccache (default) or sccache (external tools)" ON)
+option(FORCE_SCCACHE "Use sccache by default" OFF)
 
-if (NOT ENABLE_CCACHE)
-    message(STATUS "Using ccache: no (disabled via configuration)")
+if (NOT ENABLE_CCACHE AND NOT FORCE_SCCACHE)
+    message(STATUS "Using *ccache: no (disabled via configuration)")
     return()
 endif()
 
-find_program (CCACHE_EXECUTABLE ccache)
+# We support ccache and sccache
+if (FORCE_SCCACHE)
+    find_program (CCACHE_EXECUTABLE sccache)
+else()
+    find_program (CCACHE_EXECUTABLE ccache sccache)
+endif()
+
 
 if (NOT CCACHE_EXECUTABLE)
-    message(${RECONFIGURE_MESSAGE_LEVEL} "Using ccache: no (Could not find find ccache. To significantly reduce compile times for the 2nd, 3rd, etc. build, it is highly recommended to install ccache. To suppress this message, run cmake with -DENABLE_CCACHE=0)")
+    message(${RECONFIGURE_MESSAGE_LEVEL} "Using *ccache: no (Could not find find ccache or sccache. To significantly reduce compile times for the 2nd, 3rd, etc. build, it is highly recommended to install one of them. To suppress this message, run cmake with -DENABLE_CCACHE=0)")
     return()
 endif()
 
-execute_process(COMMAND ${CCACHE_EXECUTABLE} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
-string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
+if (CCACHE_EXECUTABLE MATCHES "/ccache$")
+    execute_process(COMMAND ${CCACHE_EXECUTABLE} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
+    string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
 
-set (CCACHE_MINIMUM_VERSION 3.3)
+    set (CCACHE_MINIMUM_VERSION 3.3)
 
-if (CCACHE_VERSION VERSION_LESS_EQUAL ${CCACHE_MINIMUM_VERSION})
-    message(${RECONFIGURE_MESSAGE_LEVEL} "Using ccache: no (found ${CCACHE_EXECUTABLE} (version ${CCACHE_VERSION}), the minimum required version is ${CCACHE_MINIMUM_VERSION}")
-    return()
-endif()
+    if (CCACHE_VERSION VERSION_LESS_EQUAL ${CCACHE_MINIMUM_VERSION})
+        message(${RECONFIGURE_MESSAGE_LEVEL} "Using ccache: no (found ${CCACHE_EXECUTABLE} (version ${CCACHE_VERSION}), the minimum required version is ${CCACHE_MINIMUM_VERSION}")
+        return()
+    endif()
 
-message(STATUS "Using ccache: ${CCACHE_EXECUTABLE} (version ${CCACHE_VERSION})")
-set(LAUNCHER ${CCACHE_EXECUTABLE})
+    message(STATUS "Using ccache: ${CCACHE_EXECUTABLE} (version ${CCACHE_VERSION})")
+    set(LAUNCHER ${CCACHE_EXECUTABLE})
 
-# Work around a well-intended but unfortunate behavior of ccache 4.0 & 4.1 with
-# environment variable SOURCE_DATE_EPOCH. This variable provides an alternative
-# to source-code embedded timestamps (__DATE__/__TIME__) and therefore helps with
-# reproducible builds (*). SOURCE_DATE_EPOCH is set automatically by the
-# distribution, e.g. Debian. Ccache 4.0 & 4.1 incorporate SOURCE_DATE_EPOCH into
-# the hash calculation regardless they contain timestamps or not. This invalidates
-# the cache whenever SOURCE_DATE_EPOCH changes. As a fix, ignore SOURCE_DATE_EPOCH.
-#
-# (*) https://reproducible-builds.org/specs/source-date-epoch/
-if (CCACHE_VERSION VERSION_GREATER_EQUAL "4.0" AND CCACHE_VERSION VERSION_LESS "4.2")
-    message(STATUS "Ignore SOURCE_DATE_EPOCH for ccache 4.0 / 4.1")
-    set(LAUNCHER env -u SOURCE_DATE_EPOCH ${CCACHE_EXECUTABLE})
+    # Work around a well-intended but unfortunate behavior of ccache 4.0 & 4.1 with
+    # environment variable SOURCE_DATE_EPOCH. This variable provides an alternative
+    # to source-code embedded timestamps (__DATE__/__TIME__) and therefore helps with
+    # reproducible builds (*). SOURCE_DATE_EPOCH is set automatically by the
+    # distribution, e.g. Debian. Ccache 4.0 & 4.1 incorporate SOURCE_DATE_EPOCH into
+    # the hash calculation regardless they contain timestamps or not. This invalidates
+    # the cache whenever SOURCE_DATE_EPOCH changes. As a fix, ignore SOURCE_DATE_EPOCH.
+    #
+    # (*) https://reproducible-builds.org/specs/source-date-epoch/
+    if (CCACHE_VERSION VERSION_GREATER_EQUAL "4.0" AND CCACHE_VERSION VERSION_LESS "4.2")
+        message(STATUS "Ignore SOURCE_DATE_EPOCH for ccache 4.0 / 4.1")
+        set(LAUNCHER env -u SOURCE_DATE_EPOCH ${CCACHE_EXECUTABLE})
+    endif()
+elseif(CCACHE_EXECUTABLE MATCHES "/sccache$")
+    message(STATUS "Using sccache: ${CCACHE_EXECUTABLE}")
+    set(LAUNCHER ${CCACHE_EXECUTABLE})
 endif()
 
 set (CMAKE_CXX_COMPILER_LAUNCHER ${LAUNCHER} ${CMAKE_CXX_COMPILER_LAUNCHER})

From 3b1291eb0ebf8740b715f572c15811b8ae61a7a9 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 9 Feb 2023 16:45:57 +0100
Subject: [PATCH 06/16] Replace distcc with sccache in packager

---
 docker/packager/packager | 28 ++++++----------------------
 1 file changed, 6 insertions(+), 22 deletions(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index d59aeea50af..41d093f4596 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -118,10 +118,8 @@ def parse_env_variables(
     sanitizer,
     package_type,
     cache,
-    distcc_hosts,
     clang_tidy,
     version,
-    author,
     official,
     additional_pkgs,
     with_coverage,
@@ -251,10 +249,7 @@ def parse_env_variables(
     else:
         result.append("BUILD_TYPE=None")
 
-    if cache == "distcc":
-        result.append(f"CCACHE_PREFIX={cache}")
-
-    if cache:
+    if cache == "ccache":
         result.append("CCACHE_DIR=/ccache")
         result.append("CCACHE_COMPRESSLEVEL=5")
         result.append("CCACHE_BASEDIR=/build")
@@ -270,14 +265,6 @@ def parse_env_variables(
             result.append("CTCACHE_DIR=/ccache/clang-tidy-cache")
         result.append(f"CCACHE_MAXSIZE={cache_maxsize}")
 
-    if distcc_hosts:
-        hosts_with_params = [f"{host}/24,lzo" for host in distcc_hosts] + [
-            "localhost/`nproc`"
-        ]
-        result.append('DISTCC_HOSTS="' + " ".join(hosts_with_params) + '"')
-    elif cache == "distcc":
-        result.append('DISTCC_HOSTS="localhost/`nproc`"')
-
     if additional_pkgs:
         # NOTE: This are the env for packages/build script
         result.append("MAKE_APK=true")
@@ -308,9 +295,6 @@ def parse_env_variables(
     if version:
         result.append(f"VERSION_STRING='{version}'")
 
-    if author:
-        result.append(f"AUTHOR='{author}'")
-
     if official:
         cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1")
 
@@ -370,17 +354,19 @@ def main():
     )
 
     parser.add_argument("--clang-tidy", action="store_true")
-    parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
+    parser.add_argument("--cache", choices=("ccache", "sccache", ""), default="")
     parser.add_argument(
         "--ccache_dir",
         default=Path.home() / ".ccache",
         type=dir_name,
         help="a directory with ccache",
     )
-    parser.add_argument("--distcc-hosts", nargs="+")
+    parser.add_argument(
+        "--s3-bucket",
+        help="a bucket with cache for clang-tidy-cache and sscache",
+    )
     parser.add_argument("--force-build-image", action="store_true")
     parser.add_argument("--version")
-    parser.add_argument("--author", default="clickhouse", help="a package author")
     parser.add_argument("--official", action="store_true")
     parser.add_argument("--additional-pkgs", action="store_true")
     parser.add_argument("--with-coverage", action="store_true")
@@ -419,10 +405,8 @@ def main():
         args.sanitizer,
         args.package_type,
         args.cache,
-        args.distcc_hosts,
         args.clang_tidy,
         args.version,
-        args.author,
         args.official,
         args.additional_pkgs,
         args.with_coverage,

From 419f234294d62eee6b282a5a47a13c41310525a6 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Mon, 13 Feb 2023 15:25:23 +0100
Subject: [PATCH 07/16] Fix logic for --force-build-image

---
 docker/packager/packager | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index 41d093f4596..d92a949a956 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -396,9 +396,11 @@ def main():
 
     dockerfile = ch_root / "docker/packager" / IMAGE_TYPE / "Dockerfile"
     image_with_version = image_name + ":" + args.docker_image_version
-    if not check_image_exists_locally(image_name) or args.force_build_image:
-        if not pull_image(image_with_version) or args.force_build_image:
-            build_image(image_with_version, dockerfile)
+    if args.force_build_image:
+        build_image(image_with_version, dockerfile)
+    elif not (check_image_exists_locally(image_name) or pull_image(image_with_version)):
+        build_image(image_with_version, dockerfile)
+
     env_prepared = parse_env_variables(
         args.build_type,
         args.compiler,

From 2d872ff23b9c0952f738e1a90b96122d19f03721 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 28 Feb 2023 20:18:50 +0100
Subject: [PATCH 08/16] Add a function to parse arguments

---
 docker/packager/packager | 37 +++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index d92a949a956..2a259cde874 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -10,6 +10,7 @@ from typing import List
 
 SCRIPT_PATH = Path(__file__).absolute()
 IMAGE_TYPE = "binary"
+IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}-builder"
 
 
 class BuildException(Exception):
@@ -80,7 +81,6 @@ def run_docker_image_with_env(
     env_variables: List[str],
     ch_root: Path,
     ccache_dir: Path,
-    docker_image_version: str,
 ):
     output_dir.mkdir(parents=True, exist_ok=True)
     env_part = " -e ".join(env_variables)
@@ -100,7 +100,7 @@ def run_docker_image_with_env(
     cmd = (
         f"docker run --network=host --user={user} --rm --volume={output_dir}:/output "
         f"--volume={ch_root}:/build --volume={ccache_dir}:/ccache {env_part} "
-        f"{interactive} {image_name}:{docker_image_version}"
+        f"{interactive} {image_name}"
     )
 
     logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd)
@@ -311,8 +311,7 @@ def dir_name(name: str) -> Path:
     return path
 
 
-def main():
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
+def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description="ClickHouse building script using prebuilt Docker image",
@@ -382,23 +381,34 @@ def main():
 
     args = parser.parse_args()
 
-    image_name = f"clickhouse/{IMAGE_TYPE}-builder"
-
-    ch_root = args.clickhouse_repo_path
-
     if args.additional_pkgs and args.package_type != "deb":
-        raise BuildException("Can build additional packages only in deb build")
+        raise argparse.ArgumentTypeError(
+            "Can build additional packages only in deb build"
+        )
 
     if args.with_binaries != "":
         if args.package_type != "deb":
-            raise BuildException("Can add additional binaries only in deb build")
+            raise argparse.ArgumentTypeError(
+                "Can add additional binaries only in deb build"
+            )
         logging.info("Should place %s to output", args.with_binaries)
 
+    return args
+
+
+def main():
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
+    args = parse_args()
+
+    ch_root = args.clickhouse_repo_path
+
     dockerfile = ch_root / "docker/packager" / IMAGE_TYPE / "Dockerfile"
-    image_with_version = image_name + ":" + args.docker_image_version
+    image_with_version = IMAGE_NAME + ":" + args.docker_image_version
     if args.force_build_image:
         build_image(image_with_version, dockerfile)
-    elif not (check_image_exists_locally(image_name) or pull_image(image_with_version)):
+    elif not (
+        check_image_exists_locally(image_with_version) or pull_image(image_with_version)
+    ):
         build_image(image_with_version, dockerfile)
 
     env_prepared = parse_env_variables(
@@ -417,13 +427,12 @@ def main():
 
     pre_build(args.clickhouse_repo_path, env_prepared)
     run_docker_image_with_env(
-        image_name,
+        image_with_version,
         args.as_root,
         args.output_dir,
         env_prepared,
         ch_root,
         args.ccache_dir,
-        args.docker_image_version,
     )
     logging.info("Output placed into %s", args.output_dir)
 

From df17a354103113e17be7f94971a7e5b5d09acf88 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 21 Mar 2023 16:39:21 +0100
Subject: [PATCH 09/16] Get rid of useless cache_maxsize parameter

---
 docker/packager/packager | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index 2a259cde874..fdde5ae8bc3 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -255,15 +255,11 @@ def parse_env_variables(
         result.append("CCACHE_BASEDIR=/build")
         result.append("CCACHE_NOHASHDIR=true")
         result.append("CCACHE_COMPILERCHECK=content")
-        cache_maxsize = "15G"
+        result.append("CCACHE_MAXSIZE=15G")
         if clang_tidy:
-            # 15G is not enough for tidy build
-            cache_maxsize = "25G"
-
             # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above.
             # It's there to have the clang-tidy cache embedded into our standard `CCACHE_DIR`
             result.append("CTCACHE_DIR=/ccache/clang-tidy-cache")
-        result.append(f"CCACHE_MAXSIZE={cache_maxsize}")
 
     if additional_pkgs:
         # NOTE: This are the env for packages/build script

From 6d8da8f88bad64bb3437b2cbb0570719ba3a8c8e Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 23 Mar 2023 12:43:05 +0100
Subject: [PATCH 10/16] Add sccache and ctcache S3 parameters, don't use

---
 docker/packager/binary/Dockerfile |  4 +-
 docker/packager/packager          | 79 ++++++++++++++++++++++++-------
 2 files changed, 64 insertions(+), 19 deletions(-)

diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index bc2327b9d12..fa860b2207f 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -76,7 +76,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
     && rm -rf /var/lib/apt/lists
 
 # Download toolchain and SDK for Darwin
-RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
+RUN curl -sL -O https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
 
 # Architecture of the image when BuildKit/buildx is used
 ARG TARGETARCH
@@ -98,7 +98,7 @@ ENV PATH="$PATH:/usr/local/go/bin"
 ENV GOPATH=/workdir/go
 ENV GOCACHE=/workdir/
 
-ARG CLANG_TIDY_SHA1=03644275e794b0587849bfc2ec6123d5ae0bdb1c
+ARG CLANG_TIDY_SHA1=c191254ea00d47ade11d7170ef82fe038c213774
 RUN curl -Lo /usr/bin/clang-tidy-cache \
         "https://raw.githubusercontent.com/matus-chochlik/ctcache/$CLANG_TIDY_SHA1/clang-tidy-cache" \
     && chmod +x /usr/bin/clang-tidy-cache
diff --git a/docker/packager/packager b/docker/packager/packager
index fdde5ae8bc3..7df9ebe04ec 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -113,17 +113,20 @@ def is_release_build(build_type: str, package_type: str, sanitizer: str) -> bool
 
 
 def parse_env_variables(
-    build_type,
-    compiler,
-    sanitizer,
-    package_type,
-    cache,
-    clang_tidy,
-    version,
-    official,
-    additional_pkgs,
-    with_coverage,
-    with_binaries,
+    build_type: str,
+    compiler: str,
+    sanitizer: str,
+    package_type: str,
+    cache: str,
+    s3_bucket: str,
+    s3_directory: str,
+    s3_rw_access: bool,
+    clang_tidy: bool,
+    version: str,
+    official: bool,
+    additional_pkgs: bool,
+    with_coverage: bool,
+    with_binaries: str,
 ):
     DARWIN_SUFFIX = "-darwin"
     DARWIN_ARM_SUFFIX = "-darwin-aarch64"
@@ -256,10 +259,30 @@ def parse_env_variables(
         result.append("CCACHE_NOHASHDIR=true")
         result.append("CCACHE_COMPILERCHECK=content")
         result.append("CCACHE_MAXSIZE=15G")
-        if clang_tidy:
-            # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above.
-            # It's there to have the clang-tidy cache embedded into our standard `CCACHE_DIR`
-            result.append("CTCACHE_DIR=/ccache/clang-tidy-cache")
+
+    if cache == "sccache":
+        # see https://github.com/mozilla/sccache/blob/main/docs/S3.md
+        result.append(f"SCCACHE_BUCKET={s3_bucket}")
+        sccache_dir = "sccache"
+        if s3_directory:
+            sccache_dir = f"{s3_directory}/{sccache_dir}"
+        result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}")
+        if not s3_rw_access:
+            result.append("SCCACHE_S3_NO_CREDENTIALS=true")
+
+    if clang_tidy:
+        # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above.
+        # It's there to have the clang-tidy cache embedded into our standard `CCACHE_DIR`
+        result.append("CTCACHE_DIR=/ccache/clang-tidy-cache")
+        if s3_bucket:
+            # see https://github.com/matus-chochlik/ctcache#environment-variables
+            ctcache_dir = "clang-tidy-cache"
+            if s3_directory:
+                ctcache_dir = f"{s3_directory}/{ctcache_dir}"
+            result.append(f"CTCACHE_S3_BUCKET={s3_bucket}")
+            result.append(f"CTCACHE_S3_FOLDER={ctcache_dir}")
+            if not s3_rw_access:
+                result.append("CTCACHE_S3_NO_CREDENTIALS=true")
 
     if additional_pkgs:
         # NOTE: This are the env for packages/build script
@@ -349,7 +372,12 @@ def parse_args() -> argparse.Namespace:
     )
 
     parser.add_argument("--clang-tidy", action="store_true")
-    parser.add_argument("--cache", choices=("ccache", "sccache", ""), default="")
+    parser.add_argument(
+        "--cache",
+        choices=("ccache", "sccache", ""),
+        default="",
+        help="ccache or sccache for objects caching; sccache uses only S3 buckets",
+    )
     parser.add_argument(
         "--ccache_dir",
         default=Path.home() / ".ccache",
@@ -358,7 +386,17 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--s3-bucket",
-        help="a bucket with cache for clang-tidy-cache and sscache",
+        help="an S3 bucket used for sscache and clang-tidy-cache",
+    )
+    parser.add_argument(
+        "--s3-directory",
+        default="ccache",
+        help="an S3 directory prefix used for sscache and clang-tidy-cache",
+    )
+    parser.add_argument(
+        "--s3-rw-access",
+        action="store_true",
+        help="if set, the build fails on errors writing cache to S3",
     )
     parser.add_argument("--force-build-image", action="store_true")
     parser.add_argument("--version")
@@ -389,6 +427,10 @@ def parse_args() -> argparse.Namespace:
             )
         logging.info("Should place %s to output", args.with_binaries)
 
+    if args.cache == "sccache":
+        if not args.s3_bucket:
+            raise argparse.ArgumentTypeError("sccache must have --s3-bucket set")
+
     return args
 
 
@@ -413,6 +455,9 @@ def main():
         args.sanitizer,
         args.package_type,
         args.cache,
+        args.s3_bucket,
+        args.s3_directory,
+        args.s3_rw_access,
         args.clang_tidy,
         args.version,
         args.official,

From fcc14ae72429d9ed8b004f2d6e2edd8aa6cac642 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Fri, 24 Mar 2023 17:24:21 +0100
Subject: [PATCH 11/16] Enable s3 caching

---
 docker/packager/packager | 23 +++++++++++++++++++----
 tests/ci/build_check.py  | 31 +++----------------------------
 2 files changed, 22 insertions(+), 32 deletions(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index 7df9ebe04ec..8acbf0669b5 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -6,7 +6,7 @@ import argparse
 import logging
 import sys
 from pathlib import Path
-from typing import List
+from typing import List, Optional
 
 SCRIPT_PATH = Path(__file__).absolute()
 IMAGE_TYPE = "binary"
@@ -80,7 +80,7 @@ def run_docker_image_with_env(
     output_dir: Path,
     env_variables: List[str],
     ch_root: Path,
-    ccache_dir: Path,
+    ccache_dir: Optional[Path],
 ):
     output_dir.mkdir(parents=True, exist_ok=True)
     env_part = " -e ".join(env_variables)
@@ -97,9 +97,13 @@ def run_docker_image_with_env(
     else:
         user = f"{os.geteuid()}:{os.getegid()}"
 
+    ccache_mount = f"--volume={ccache_dir}:/ccache"
+    if ccache_dir is None:
+        ccache_mount = ""
+
     cmd = (
-        f"docker run --network=host --user={user} --rm --volume={output_dir}:/output "
-        f"--volume={ch_root}:/build --volume={ccache_dir}:/ccache {env_part} "
+        f"docker run --network=host --user={user} --rm {ccache_mount}"
+        f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
         f"{interactive} {image_name}"
     )
 
@@ -252,7 +256,13 @@ def parse_env_variables(
     else:
         result.append("BUILD_TYPE=None")
 
+    if not cache:
+        cmake_flags.append("-DENABLE_CCACHE=OFF")
+        cmake_flags.append("-DFORCE_SCCACHE=OFF")
+
     if cache == "ccache":
+        cmake_flags.append("-DENABLE_CCACHE=ON")
+        cmake_flags.append("-DFORCE_SCCACHE=OFF")
         result.append("CCACHE_DIR=/ccache")
         result.append("CCACHE_COMPRESSLEVEL=5")
         result.append("CCACHE_BASEDIR=/build")
@@ -261,6 +271,8 @@ def parse_env_variables(
         result.append("CCACHE_MAXSIZE=15G")
 
     if cache == "sccache":
+        cmake_flags.append("-DENABLE_CCACHE=OFF")
+        cmake_flags.append("-DFORCE_SCCACHE=ON")
         # see https://github.com/mozilla/sccache/blob/main/docs/S3.md
         result.append(f"SCCACHE_BUCKET={s3_bucket}")
         sccache_dir = "sccache"
@@ -420,6 +432,9 @@ def parse_args() -> argparse.Namespace:
             "Can build additional packages only in deb build"
         )
 
+    if args.cache != "ccache":
+        args.ccache_dir = None
+
     if args.with_binaries != "":
         if args.package_type != "deb":
             raise argparse.ArgumentTypeError(
diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index 4db601be803..a829069985d 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -6,15 +6,12 @@ import json
 import os
 import sys
 import time
-from shutil import rmtree
 from typing import List, Tuple
 
-from ccache_utils import get_ccache_if_not_exists, upload_ccache
 from ci_config import CI_CONFIG, BuildConfig
 from commit_status_helper import get_commit_filtered_statuses, get_commit
 from docker_pull_helper import get_image_with_version
 from env_helper import (
-    CACHES_PATH,
     GITHUB_JOB,
     IMAGES_PATH,
     REPO_COPY,
@@ -54,7 +51,6 @@ def get_packager_cmd(
     output_path: str,
     build_version: str,
     image_version: str,
-    ccache_path: str,
     official: bool,
 ) -> str:
     package_type = build_config["package_type"]
@@ -72,8 +68,9 @@ def get_packager_cmd(
     if build_config["tidy"] == "enable":
         cmd += " --clang-tidy"
 
-    cmd += " --cache=ccache"
-    cmd += f" --ccache_dir={ccache_path}"
+    cmd += " --cache=sccache"
+    cmd += " --s3-rw-access"
+    cmd += f" --s3-bucket={S3_BUILDS_BUCKET}"
 
     if "additional_pkgs" in build_config and build_config["additional_pkgs"]:
         cmd += " --additional-pkgs"
@@ -314,29 +311,12 @@ def main():
     if not os.path.exists(build_output_path):
         os.makedirs(build_output_path)
 
-    ccache_path = os.path.join(CACHES_PATH, build_name + "_ccache")
-
-    logging.info("Will try to fetch cache for our build")
-    try:
-        get_ccache_if_not_exists(
-            ccache_path, s3_helper, pr_info.number, TEMP_PATH, pr_info.release_pr
-        )
-    except Exception as e:
-        # In case there are issues with ccache, remove the path and do not fail a build
-        logging.info("Failed to get ccache, building without it. Error: %s", e)
-        rmtree(ccache_path, ignore_errors=True)
-
-    if not os.path.exists(ccache_path):
-        logging.info("cache was not fetched, will create empty dir")
-        os.makedirs(ccache_path)
-
     packager_cmd = get_packager_cmd(
         build_config,
         os.path.join(REPO_COPY, "docker/packager"),
         build_output_path,
         version.string,
         image_version,
-        ccache_path,
         official_flag,
     )
 
@@ -352,13 +332,8 @@ def main():
     subprocess.check_call(
         f"sudo chown -R ubuntu:ubuntu {build_output_path}", shell=True
     )
-    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {ccache_path}", shell=True)
     logging.info("Build finished with %s, log path %s", success, log_path)
 
-    # Upload the ccache first to have the least build time in case of problems
-    logging.info("Will upload cache")
-    upload_ccache(ccache_path, s3_helper, pr_info.number, TEMP_PATH)
-
     # FIXME performance
     performance_urls = []
     performance_path = os.path.join(build_output_path, "performance.tar.zst")

From f1b1f307645a7399a61db560bde855649b175c30 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Mon, 27 Mar 2023 09:14:27 +0200
Subject: [PATCH 12/16] Make --ccache-dir equal to --ccache_dir

---
 docker/packager/packager | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index 8acbf0669b5..2a64721d5de 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -391,7 +391,7 @@ def parse_args() -> argparse.Namespace:
         help="ccache or sccache for objects caching; sccache uses only S3 buckets",
     )
     parser.add_argument(
-        "--ccache_dir",
+        "--ccache-dir",
         default=Path.home() / ".ccache",
         type=dir_name,
         help="a directory with ccache",

From 79177e198872343b8fbd4324971529d951f7f623 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Mon, 27 Mar 2023 15:35:51 +0200
Subject: [PATCH 13/16] Enable sccache in fasttests

---
 docker/test/fasttest/run.sh |  5 +++--
 tests/ci/fast_test_check.py | 40 +++++++------------------------------
 2 files changed, 10 insertions(+), 35 deletions(-)

diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index ac2eb5b403e..e0e30a63bb4 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -16,7 +16,8 @@ export LLVM_VERSION=${LLVM_VERSION:-13}
 # it being undefined. Also read it as array so that we can pass an empty list
 # of additional variable to cmake properly, and it doesn't generate an extra
 # empty parameter.
-read -ra FASTTEST_CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}"
+# Read it as CMAKE_FLAGS to not lose exported FASTTEST_CMAKE_FLAGS on subsequential launch
+read -ra CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}"
 
 # Run only matching tests.
 FASTTEST_FOCUS=${FASTTEST_FOCUS:-""}
@@ -185,7 +186,7 @@ function run_cmake
 
     (
         cd "$FASTTEST_BUILD"
-        cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" "${CMAKE_LIBS_CONFIG[@]}" "${FASTTEST_CMAKE_FLAGS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt"
+        cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" "${CMAKE_LIBS_CONFIG[@]}" "${CMAKE_FLAGS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt"
     )
 }
 
diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py
index a5bb64889d1..9981234f87c 100644
--- a/tests/ci/fast_test_check.py
+++ b/tests/ci/fast_test_check.py
@@ -11,7 +11,6 @@ from typing import List, Tuple
 
 from github import Github
 
-from ccache_utils import get_ccache_if_not_exists, upload_ccache
 from clickhouse_helper import (
     ClickHouseHelper,
     mark_flaky_tests,
@@ -22,7 +21,7 @@ from commit_status_helper import (
     update_mergeable_check,
 )
 from docker_pull_helper import get_image_with_version
-from env_helper import CACHES_PATH, TEMP_PATH
+from env_helper import S3_BUILDS_BUCKET, TEMP_PATH
 from get_robot_token import get_best_robot_token
 from pr_info import FORCE_TESTS_LABEL, PRInfo
 from report import TestResults, read_test_results
@@ -38,24 +37,22 @@ NAME = "Fast test"
 csv.field_size_limit(sys.maxsize)
 
 
-def get_fasttest_cmd(
-    workspace, output_path, ccache_path, repo_path, pr_number, commit_sha, image
-):
+def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image):
     return (
         f"docker run --cap-add=SYS_PTRACE "
+        "--network=host "  # required to get access to IAM credentials
         f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output "
         f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE "
+        f"-e FASTTEST_CMAKE_FLAGS='-DENABLE_CCACHE=OFF -DFORCE_SCCACHE=ON' "
         f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} "
         f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 "
+        f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache "
         f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse "
-        f"--volume={output_path}:/test_output "
-        f"--volume={ccache_path}:/fasttest-workspace/ccache {image}"
+        f"--volume={output_path}:/test_output {image}"
     )
 
 
-def process_results(
-    result_folder: str,
-) -> Tuple[str, str, TestResults, List[str]]:
+def process_results(result_folder: str) -> Tuple[str, str, TestResults, List[str]]:
     test_results = []  # type: TestResults
     additional_files = []
     # Just upload all files from result_folder.
@@ -129,21 +126,6 @@ def main():
     if not os.path.exists(output_path):
         os.makedirs(output_path)
 
-    if not os.path.exists(CACHES_PATH):
-        os.makedirs(CACHES_PATH)
-    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {CACHES_PATH}", shell=True)
-    cache_path = os.path.join(CACHES_PATH, "fasttest")
-
-    logging.info("Will try to fetch cache for our build")
-    ccache_for_pr = get_ccache_if_not_exists(
-        cache_path, s3_helper, pr_info.number, temp_path, pr_info.release_pr
-    )
-    upload_master_ccache = ccache_for_pr in (-1, 0)
-
-    if not os.path.exists(cache_path):
-        logging.info("cache was not fetched, will create empty dir")
-        os.makedirs(cache_path)
-
     repo_path = os.path.join(temp_path, "fasttest-repo")
     if not os.path.exists(repo_path):
         os.makedirs(repo_path)
@@ -151,7 +133,6 @@ def main():
     run_cmd = get_fasttest_cmd(
         workspace,
         output_path,
-        cache_path,
         repo_path,
         pr_info.number,
         pr_info.sha,
@@ -172,7 +153,6 @@ def main():
             logging.info("Run failed")
 
     subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
-    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {cache_path}", shell=True)
 
     test_output_files = os.listdir(output_path)
     additional_logs = []
@@ -202,12 +182,6 @@ def main():
     else:
         state, description, test_results, additional_logs = process_results(output_path)
 
-    logging.info("Will upload cache")
-    upload_ccache(cache_path, s3_helper, pr_info.number, temp_path)
-    if upload_master_ccache:
-        logging.info("Will upload a fallback cache for master")
-        upload_ccache(cache_path, s3_helper, 0, temp_path)
-
     ch_helper = ClickHouseHelper()
     mark_flaky_tests(ch_helper, NAME, test_results)
 

From b140915b0a3d83dd31f58d20118f522c413793d1 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Mon, 27 Mar 2023 20:26:40 +0200
Subject: [PATCH 14/16] Use a network-independent cluster in
 01293_show_clusters

---
 tests/queries/0_stateless/01293_show_clusters.reference | 3 ++-
 tests/queries/0_stateless/01293_show_clusters.sh        | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01293_show_clusters.reference b/tests/queries/0_stateless/01293_show_clusters.reference
index c62f8cdfa2d..9f8a44ebd0a 100644
--- a/tests/queries/0_stateless/01293_show_clusters.reference
+++ b/tests/queries/0_stateless/01293_show_clusters.reference
@@ -1,2 +1,3 @@
 test_shard_localhost
-test_shard_localhost	1	1	1	localhost	::1	9000	1	default	
+test_cluster_one_shard_two_replicas	1	1	1	127.0.0.1	127.0.0.1	9000	1	default	
+test_cluster_one_shard_two_replicas	1	1	2	127.0.0.2	127.0.0.2	9000	0	default	
diff --git a/tests/queries/0_stateless/01293_show_clusters.sh b/tests/queries/0_stateless/01293_show_clusters.sh
index 2fdf17ec25e..ae027210383 100755
--- a/tests/queries/0_stateless/01293_show_clusters.sh
+++ b/tests/queries/0_stateless/01293_show_clusters.sh
@@ -6,4 +6,5 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -q "show clusters like 'test_shard%' limit 1"
 # cluster,shard_num,shard_weight,replica_num,host_name,host_address,port,is_local,user,default_database[,errors_count,slowdowns_count,estimated_recovery_time]
-$CLICKHOUSE_CLIENT -q "show cluster 'test_shard_localhost'" | cut -f-10
+# use a cluster with static IPv4
+$CLICKHOUSE_CLIENT -q "show cluster 'test_cluster_one_shard_two_replicas'" | cut -f-10

From def5ae851e681ae3e0147141df5dd95c765f6807 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Tue, 28 Mar 2023 17:36:05 +0200
Subject: [PATCH 15/16] Use CTCACHE_DIR only with ccache

---
 docker/packager/packager | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index 2a64721d5de..e4b17a3a1da 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -285,7 +285,8 @@ def parse_env_variables(
     if clang_tidy:
         # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above.
         # It's there to have the clang-tidy cache embedded into our standard `CCACHE_DIR`
-        result.append("CTCACHE_DIR=/ccache/clang-tidy-cache")
+        if cache == "ccache":
+            result.append("CTCACHE_DIR=/ccache/clang-tidy-cache")
         if s3_bucket:
             # see https://github.com/matus-chochlik/ctcache#environment-variables
             ctcache_dir = "clang-tidy-cache"

From 7ff582b963bdfca74f942704fd0c1deb96c27e3e Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 29 Mar 2023 16:52:04 +0200
Subject: [PATCH 16/16] Deprecate ENABLE_CCACHE and replace it by
 COMPILER_CACHE

---
 CMakeLists.txt              |  2 +-
 cmake/ccache.cmake          | 29 +++++++++++++++++++----------
 docker/packager/packager    |  9 +++------
 tests/ci/fast_test_check.py |  2 +-
 4 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bd7de46474b..ce615c11f2b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -568,7 +568,7 @@ if (NATIVE_BUILD_TARGETS
         COMMAND ${CMAKE_COMMAND}
             "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
             "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
-            "-DENABLE_CCACHE=${ENABLE_CCACHE}"
+            "-DCOMPILER_CACHE=${COMPILER_CACHE}"
             # Avoid overriding .cargo/config.toml with native toolchain.
             "-DENABLE_RUST=OFF"
             "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=${ENABLE_CLICKHOUSE_SELF_EXTRACTING}"
diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake
index 6103a5212e3..285c8efb9e1 100644
--- a/cmake/ccache.cmake
+++ b/cmake/ccache.cmake
@@ -1,5 +1,6 @@
 # Setup integration with ccache to speed up builds, see https://ccache.dev/
 
+# Matches both ccache and sccache
 if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MATCHES "ccache")
     # custom compiler launcher already defined, most likely because cmake was invoked with like "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" or
     # via environment variable --> respect setting and trust that the launcher was specified correctly
@@ -8,24 +9,32 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA
     return()
 endif()
 
-option(ENABLE_CCACHE "Speedup re-compilations using ccache (default) or sccache (external tools)" ON)
-option(FORCE_SCCACHE "Use sccache by default" OFF)
-
-if (NOT ENABLE_CCACHE AND NOT FORCE_SCCACHE)
-    message(STATUS "Using *ccache: no (disabled via configuration)")
-    return()
+set(ENABLE_CCACHE "default" CACHE STRING "Deprecated, use COMPILER_CACHE=(auto|ccache|sccache|disabled)")
+if (NOT ENABLE_CCACHE STREQUAL "default")
+    message(WARNING "The -DENABLE_CCACHE is deprecated in favor of -DCOMPILER_CACHE")
 endif()
 
-# We support ccache and sccache
-if (FORCE_SCCACHE)
+set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (ccache, then sccache), 'ccache', 'sccache', or 'disabled'")
+
+# It has pretty complex logic, because the ENABLE_CCACHE is deprecated, but still should
+# control the COMPILER_CACHE
+# After it will be completely removed, the following block will be much simpler
+if (COMPILER_CACHE STREQUAL "ccache" OR (ENABLE_CCACHE AND NOT ENABLE_CCACHE STREQUAL "default"))
+    find_program (CCACHE_EXECUTABLE ccache)
+elseif(COMPILER_CACHE STREQUAL "disabled" OR NOT ENABLE_CCACHE STREQUAL "default")
+    message(STATUS "Using *ccache: no (disabled via configuration)")
+    return()
+elseif(COMPILER_CACHE STREQUAL "auto")
+    find_program (CCACHE_EXECUTABLE ccache sccache)
+elseif(COMPILER_CACHE STREQUAL "sccache")
     find_program (CCACHE_EXECUTABLE sccache)
 else()
-    find_program (CCACHE_EXECUTABLE ccache sccache)
+    message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|ccache|sccache|disabled), given '${COMPILER_CACHE}'")
 endif()
 
 
 if (NOT CCACHE_EXECUTABLE)
-    message(${RECONFIGURE_MESSAGE_LEVEL} "Using *ccache: no (Could not find find ccache or sccache. To significantly reduce compile times for the 2nd, 3rd, etc. build, it is highly recommended to install one of them. To suppress this message, run cmake with -DENABLE_CCACHE=0)")
+    message(${RECONFIGURE_MESSAGE_LEVEL} "Using *ccache: no (Could not find find ccache or sccache. To significantly reduce compile times for the 2nd, 3rd, etc. build, it is highly recommended to install one of them. To suppress this message, run cmake with -DCOMPILER_CACHE=disabled)")
     return()
 endif()
 
diff --git a/docker/packager/packager b/docker/packager/packager
index e4b17a3a1da..7d022df52e6 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -257,12 +257,10 @@ def parse_env_variables(
         result.append("BUILD_TYPE=None")
 
     if not cache:
-        cmake_flags.append("-DENABLE_CCACHE=OFF")
-        cmake_flags.append("-DFORCE_SCCACHE=OFF")
+        cmake_flags.append("-DCOMPILER_CACHE=disabled")
 
     if cache == "ccache":
-        cmake_flags.append("-DENABLE_CCACHE=ON")
-        cmake_flags.append("-DFORCE_SCCACHE=OFF")
+        cmake_flags.append("-DCOMPILER_CACHE=ccache")
         result.append("CCACHE_DIR=/ccache")
         result.append("CCACHE_COMPRESSLEVEL=5")
         result.append("CCACHE_BASEDIR=/build")
@@ -271,8 +269,7 @@ def parse_env_variables(
         result.append("CCACHE_MAXSIZE=15G")
 
     if cache == "sccache":
-        cmake_flags.append("-DENABLE_CCACHE=OFF")
-        cmake_flags.append("-DFORCE_SCCACHE=ON")
+        cmake_flags.append("-DCOMPILER_CACHE=sccache")
         # see https://github.com/mozilla/sccache/blob/main/docs/S3.md
         result.append(f"SCCACHE_BUCKET={s3_bucket}")
         sccache_dir = "sccache"
diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py
index 9981234f87c..54367f70b3f 100644
--- a/tests/ci/fast_test_check.py
+++ b/tests/ci/fast_test_check.py
@@ -43,7 +43,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i
         "--network=host "  # required to get access to IAM credentials
         f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output "
         f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE "
-        f"-e FASTTEST_CMAKE_FLAGS='-DENABLE_CCACHE=OFF -DFORCE_SCCACHE=ON' "
+        f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' "
         f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} "
         f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 "
         f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache "