Merge remote-tracking branch 'origin' into vectorshine-master

.github/workflows/create_release.yml

@@ -3,6 +3,9 @@ name: CreateRelease
 concurrency:
   group: release
 
+env:
+  PYTHONUNBUFFERED: 1
+
 'on':
   workflow_dispatch:
     inputs:

.github/workflows/docker_test_images.yml

@@ -1,4 +1,5 @@
 name: Build docker images
+
 'on':
   workflow_call:
     inputs:
@@ -12,6 +13,9 @@ name: Build docker images
         type: boolean
         default: false
 
+env:
+  PYTHONUNBUFFERED: 1
+
 jobs:
   DockerBuildAarch64:
     runs-on: [self-hosted, style-checker-aarch64]

.github/workflows/reusable_build_stage.yml

@@ -1,7 +1,11 @@
 ### FIXME: merge reusable_test.yml and reusable_build.yml as they are almost identical
 #          and then merge reusable_build_stage.yml and reusable_test_stage.yml
 
+env:
+  PYTHONUNBUFFERED: 1
+
 name: BuildStageWF
+
 'on':
   workflow_call:
     inputs:

.github/workflows/reusable_test_stage.yml

@@ -1,4 +1,8 @@
+env:
+  PYTHONUNBUFFERED: 1
+
 name: StageWF
+
 'on':
   workflow_call:
     inputs:

README.md

@@ -34,7 +34,7 @@ curl https://clickhouse.com/ | sh
 
 Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
 
-* [v24.9 Community Call](https://clickhouse.com/company/events/v24-9-community-release-call) - September 26
+* [v24.10 Community Call](https://clickhouse.com/company/events/v24-10-community-release-call) - October 31
 
 ## Upcoming Events
 

SECURITY.md

@@ -14,9 +14,10 @@ The following versions of ClickHouse server are currently supported with securit
 
 | Version | Supported |
 |:-|:-|
+| 24.9 | ✔️ |
 | 24.8 | ✔️ |
 | 24.7 | ✔️ |
-| 24.6 | ✔️ |
+| 24.6 | ❌ |
 | 24.5 | ❌ |
 | 24.4 | ❌ |
 | 24.3 | ✔️ |

ci_v2/docker/style-test/Dockerfile

@@ -0,0 +1,17 @@
+# docker build -t clickhouse/style-test .
+FROM ubuntu:22.04
+
+RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+        aspell \
+        libxml2-utils \
+        python3-pip \
+        locales \
+        git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
+
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r requirements.txt

ci_v2/docker/style-test/requirements.txt

@@ -0,0 +1,4 @@
+requests==2.32.3
+yamllint==1.26.3
+codespell==2.2.1
+https://clickhouse-builds.s3.amazonaws.com/packages/praktika-0.1-py3-none-any.whl

ci_v2/jobs/check_style.py

@@ -0,0 +1,410 @@
+import math
+import multiprocessing
+import os
+import re
+import sys
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+
+from praktika.result import Result
+from praktika.utils import Shell, Utils
+
+NPROC = multiprocessing.cpu_count()
+
+
+def chunk_list(data, n):
+    """Split the data list into n nearly equal-sized chunks."""
+    chunk_size = math.ceil(len(data) / n)
+    for i in range(0, len(data), chunk_size):
+        yield data[i : i + chunk_size]
+
+
+def run_check_concurrent(check_name, check_function, files, nproc=NPROC):
+    stop_watch = Utils.Stopwatch()
+
+    if not files:
+        print(f"File list is empty [{files}]")
+        raise
+
+    file_chunks = list(chunk_list(files, nproc))
+    results = []
+
+    # Run check_function concurrently on each chunk
+    with ProcessPoolExecutor(max_workers=NPROC) as executor:
+        futures = [executor.submit(check_function, chunk) for chunk in file_chunks]
+        # Wait for results and process them (optional)
+        for future in futures:
+            try:
+                res = future.result()
+                if res and res not in results:
+                    results.append(res)
+            except Exception as e:
+                results.append(f"Exception in {check_name}: {e}")
+
+    result = Result(
+        name=check_name,
+        status=Result.Status.SUCCESS if not results else Result.Status.FAILED,
+        start_time=stop_watch.start_time,
+        duration=stop_watch.duration,
+        info=f"errors: {results}" if results else "",
+    )
+    return result
+
+
+def run_simple_check(check_name, check_function, **kwargs):
+    stop_watch = Utils.Stopwatch()
+
+    error = check_function(**kwargs)
+
+    result = Result(
+        name=check_name,
+        status=Result.Status.SUCCESS if not error else Result.Status.FAILED,
+        start_time=stop_watch.start_time,
+        duration=stop_watch.duration,
+        info=error,
+    )
+    return result
+
+
+def run_check(check_name, check_function, files):
+    return run_check_concurrent(check_name, check_function, files, nproc=1)
+
+
+def check_duplicate_includes(file_path):
+    includes = []
+    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+        for line in f:
+            if re.match(r"^#include ", line):
+                includes.append(line.strip())
+
+    include_counts = {line: includes.count(line) for line in includes}
+    duplicates = {line: count for line, count in include_counts.items() if count > 1}
+
+    if duplicates:
+        return f"{file_path}: {duplicates}"
+    return ""
+
+
+def check_whitespaces(file_paths):
+    for file in file_paths:
+        exit_code, out, err = Shell.get_res_stdout_stderr(
+            f'./ci_v2/jobs/scripts/check_style/double_whitespaces.pl "{file}"',
+            verbose=False,
+        )
+        if out or err:
+            return out + " err: " + err
+    return ""
+
+
+def check_yamllint(file_paths):
+    file_paths = " ".join([f"'{file}'" for file in file_paths])
+    exit_code, out, err = Shell.get_res_stdout_stderr(
+        f"yamllint --config-file=./.yamllint {file_paths}", verbose=False
+    )
+    return out or err
+
+
+def check_xmllint(file_paths):
+    if not isinstance(file_paths, list):
+        file_paths = [file_paths]
+    file_paths = " ".join([f"'{file}'" for file in file_paths])
+    exit_code, out, err = Shell.get_res_stdout_stderr(
+        f"xmllint --noout --nonet {file_paths}", verbose=False
+    )
+    return out or err
+
+
+def check_functional_test_cases(files):
+    """
+    Queries with event_date should have yesterday() not today()
+    NOTE: it is not that accuate, but at least something.
+    """
+
+    patterns = [
+        re.compile(
+            r"(?i)where.*?\bevent_date\s*(=|>=)\s*today\(\)(?!\s*-\s*1)",
+            re.IGNORECASE | re.DOTALL,
+        )
+    ]
+
+    errors = []
+    for test_case in files:
+        try:
+            with open(test_case, "r", encoding="utf-8", errors="replace") as f:
+                file_content = " ".join(
+                    f.read().splitlines()
+                )  # Combine lines into a single string
+
+            # Check if any pattern matches in the concatenated string
+            if any(pattern.search(file_content) for pattern in patterns):
+                errors.append(
+                    f"event_date should be filtered using >=yesterday() in {test_case} (to avoid flakiness)"
+                )
+
+        except Exception as e:
+            errors.append(f"Error checking {test_case}: {e}")
+
+    for test_case in files:
+        if "fail" in test_case:
+            errors.append(f"test case {test_case} includes 'fail' in its name")
+
+    return " ".join(errors)
+
+
+def check_gaps_in_tests_numbers(file_paths, gap_threshold=100):
+    test_numbers = set()
+
+    pattern = re.compile(r"(\d+)")
+
+    for file in file_paths:
+        file_name = os.path.basename(file)
+        match = pattern.search(file_name)
+        if match:
+            test_numbers.add(int(match.group(1)))
+
+    sorted_numbers = sorted(test_numbers)
+    large_gaps = []
+    for i in range(1, len(sorted_numbers)):
+        prev_num = sorted_numbers[i - 1]
+        next_num = sorted_numbers[i]
+        diff = next_num - prev_num
+        if diff >= gap_threshold:
+            large_gaps.append(f"Gap ({prev_num}, {next_num}) > {gap_threshold}")
+
+    return large_gaps
+
+
+def check_broken_links(path, exclude_paths):
+    broken_symlinks = []
+
+    for path in Path(path).rglob("*"):
+        if any(exclude_path in str(path) for exclude_path in exclude_paths):
+            continue
+        if path.is_symlink():
+            if not path.exists():
+                broken_symlinks.append(str(path))
+
+    if broken_symlinks:
+        for symlink in broken_symlinks:
+            print(symlink)
+        return f"Broken symlinks found: {broken_symlinks}"
+    else:
+        return ""
+
+
+def check_cpp_code():
+    res, out, err = Shell.get_res_stdout_stderr(
+        "./ci_v2/jobs/scripts/check_style/check_cpp.sh"
+    )
+    if err:
+        out += err
+    return out
+
+
+def check_repo_submodules():
+    res, out, err = Shell.get_res_stdout_stderr(
+        "./ci_v2/jobs/scripts/check_style/check_submodules.sh"
+    )
+    if err:
+        out += err
+    return out
+
+
+def check_other():
+    res, out, err = Shell.get_res_stdout_stderr(
+        "./ci_v2/jobs/scripts/check_style/checks_to_refactor.sh"
+    )
+    if err:
+        out += err
+    return out
+
+
+def check_codespell():
+    res, out, err = Shell.get_res_stdout_stderr(
+        "./ci_v2/jobs/scripts/check_style/check_typos.sh"
+    )
+    if err:
+        out += err
+    return out
+
+
+def check_aspell():
+    res, out, err = Shell.get_res_stdout_stderr(
+        "./ci_v2/jobs/scripts/check_style/check_aspell.sh"
+    )
+    if err:
+        out += err
+    return out
+
+
+def check_mypy():
+    res, out, err = Shell.get_res_stdout_stderr(
+        "./ci_v2/jobs/scripts/check_style/check-mypy"
+    )
+    if err:
+        out += err
+    return out
+
+
+def check_pylint():
+    res, out, err = Shell.get_res_stdout_stderr(
+        "./ci_v2/jobs/scripts/check_style/check-pylint"
+    )
+    if err:
+        out += err
+    return out
+
+
+def check_file_names(files):
+    files_set = set()
+    for file in files:
+        file_ = file.lower()
+        if file_ in files_set:
+            return f"Non-uniq file name in lower case: {file}"
+        files_set.add(file_)
+    return ""
+
+
+if __name__ == "__main__":
+    results = []
+    stop_watch = Utils.Stopwatch()
+
+    all_files = Utils.traverse_paths(
+        include_paths=["."],
+        exclude_paths=[
+            "./.git",
+            "./contrib",
+            "./build",
+        ],
+        not_exists_ok=True,  # ./build may exist if runs locally
+    )
+
+    cpp_files = Utils.traverse_paths(
+        include_paths=["./src", "./base", "./programs", "./utils"],
+        exclude_paths=[
+            "./base/glibc-compatibility",
+            "./contrib/consistent-hashing",
+            "./base/widechar_width",
+        ],
+        file_suffixes=[".h", ".cpp"],
+    )
+
+    yaml_workflow_files = Utils.traverse_paths(
+        include_paths=["./.github"],
+        exclude_paths=[],
+        file_suffixes=[".yaml", ".yml"],
+    )
+
+    xml_files = Utils.traverse_paths(
+        include_paths=["."],
+        exclude_paths=["./.git", "./contrib/"],
+        file_suffixes=[".xml"],
+    )
+
+    functional_test_files = Utils.traverse_paths(
+        include_paths=["./tests/queries"],
+        exclude_paths=[],
+        file_suffixes=[".sql", ".sh", ".py", ".j2"],
+    )
+
+    results.append(
+        Result(
+            name="Read Files",
+            status=Result.Status.SUCCESS,
+            start_time=stop_watch.start_time,
+            duration=stop_watch.duration,
+        )
+    )
+
+    results.append(
+        run_check_concurrent(
+            check_name="Whitespace Check",
+            check_function=check_whitespaces,
+            files=cpp_files,
+        )
+    )
+    results.append(
+        run_check_concurrent(
+            check_name="YamlLint Check",
+            check_function=check_yamllint,
+            files=yaml_workflow_files,
+        )
+    )
+    results.append(
+        run_check_concurrent(
+            check_name="XmlLint Check",
+            check_function=check_xmllint,
+            files=xml_files,
+        )
+    )
+    results.append(
+        run_check_concurrent(
+            check_name="Functional Tests scripts smoke check",
+            check_function=check_functional_test_cases,
+            files=functional_test_files,
+        )
+    )
+    results.append(
+        run_check(
+            check_name="Check Tests Numbers",
+            check_function=check_gaps_in_tests_numbers,
+            files=functional_test_files,
+        )
+    )
+    results.append(
+        run_simple_check(
+            check_name="Check Broken Symlinks",
+            check_function=check_broken_links,
+            path="./",
+            exclude_paths=["contrib/", "metadata/", "programs/server/data"],
+        )
+    )
+    results.append(
+        run_simple_check(
+            check_name="Check CPP code",
+            check_function=check_cpp_code,
+        )
+    )
+    results.append(
+        run_simple_check(
+            check_name="Check Submodules",
+            check_function=check_repo_submodules,
+        )
+    )
+    results.append(
+        run_check(
+            check_name="Check File Names",
+            check_function=check_file_names,
+            files=all_files,
+        )
+    )
+    results.append(
+        run_simple_check(
+            check_name="Check Many Different Things",
+            check_function=check_other,
+        )
+    )
+    results.append(
+        run_simple_check(
+            check_name="Check Codespell",
+            check_function=check_codespell,
+        )
+    )
+    results.append(
+        run_simple_check(
+            check_name="Check Aspell",
+            check_function=check_aspell,
+        )
+    )
+
+    res = Result.create_from(results=results, stopwatch=stop_watch).dump()
+
+    if not res.is_ok():
+        print("Style check: failed")
+        for result in results:
+            if not result.is_ok():
+                print("Failed check:")
+                print("  |  ", result)
+        sys.exit(1)
+    else:
+        print("Style check: ok")

ci_v2/jobs/scripts/check_style/check_aspell.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# force-enable double star globbing
+shopt -s globstar
+
+# Perform spell checking on the docs
+
+if [[ ${1:-} == "--help" ]] || [[ ${1:-} == "-h" ]]; then
+    echo "Usage $0 [--help|-h] [-i [filename]]"
+    echo "  --help|-h: print this help"
+    echo "  -i: interactive mode. If filename is specified, check only this file, otherwise check all files"
+    exit 0
+fi
+
+ROOT_PATH="."
+
+CHECK_LANG=en
+
+ASPELL_IGNORE_PATH="${ROOT_PATH}/utils/check-style/aspell-ignore/${CHECK_LANG}"
+
+if [[ ${1:-} == "-i" ]]; then
+    if [[ ! -z ${2:-} ]]; then
+        FILES_TO_CHECK=${ROOT_PATH}/docs/${CHECK_LANG}/${2}
+    else
+        FILES_TO_CHECK=${ROOT_PATH}/docs/${CHECK_LANG}/**/*.md
+    fi
+    for fname in ${FILES_TO_CHECK}; do
+        echo "Checking $fname"
+        aspell --personal=aspell-dict.txt --add-sgml-skip=code --encoding=utf-8 --mode=markdown -W 3 --lang=${CHECK_LANG} --home-dir=${ASPELL_IGNORE_PATH} -c "$fname"
+    done
+    exit
+fi
+
+STATUS=0
+for fname in ${ROOT_PATH}/docs/${CHECK_LANG}/**/*.md; do
+    errors=$(cat "$fname" \
+        | aspell list \
+            -W 3 \
+            --personal=aspell-dict.txt \
+            --add-sgml-skip=code \
+            --encoding=utf-8 \
+            --mode=markdown \
+            --lang=${CHECK_LANG} \
+            --home-dir=${ASPELL_IGNORE_PATH} \
+        | sort | uniq)
+    if [ ! -z "$errors" ]; then
+        STATUS=1
+        echo "====== $fname ======"
+        echo "$errors"
+    fi
+done
+
+if (( STATUS != 0 )); then
+    echo "====== Errors found ======"
+    echo "To exclude some words add them to the dictionary file \"${ASPELL_IGNORE_PATH}/aspell-dict.txt\""
+    echo "You can also run ${0} -i to see the errors interactively and fix them or add to the dictionary file"
+fi
+
+exit ${STATUS}

ci_v2/jobs/scripts/check_style/check_cpp.sh

@@ -0,0 +1,339 @@
+#!/usr/bin/env bash
+
+# For code formatting we have clang-format.
+#
+# But it's not sane to apply clang-format for whole code base,
+#  because it sometimes makes worse for properly formatted files.
+#
+# It's only reasonable to blindly apply clang-format only in cases
+#  when the code is likely to be out of style.
+#
+# For this purpose we have a script that will use very primitive heuristics
+#  (simple regexps) to check if the code is likely to have basic style violations.
+#  and then to run formatter only for the specified files.
+
+LC_ALL="en_US.UTF-8"
+ROOT_PATH="."
+EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|utils/keeper-bench/example.yaml'
+
+# From [1]:
+#     But since array_to_string_internal() in array.c still loops over array
+#     elements and concatenates them into a string, it's probably not more
+#     efficient than the looping solutions proposed, but it's more readable.
+#
+#  [1]: https://stackoverflow.com/a/15394738/328260
+function in_array()
+{
+    local IFS="|"
+    local value=$1 && shift
+
+    [[ "${IFS}${*}${IFS}" =~ "${IFS}${value}${IFS}" ]]
+}
+
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep $@ -P '((class|struct|namespace|enum|if|for|while|else|throw|switch).*|\)(\s*const)?(\s*override)?\s*)\{$|\s$|^ {1,3}[^\* ]\S|\t|^\s*(if|else if|if constexpr|else if constexpr|for|while|catch|switch)\(|\( [^\s\\]|\S \)' |
+# a curly brace not in a new line, but not for the case of C++11 init or agg. initialization | trailing whitespace | number of ws not a multiple of 4, but not in the case of comment continuation | missing whitespace after for/if/while... before opening brace | whitespaces inside braces
+    grep -v -P '(//|:\s+\*|\$\(\()| \)"'
+# single-line comment | continuation of a multiline comment | a typical piece of embedded shell code | something like ending of raw string literal
+
+# Tabs
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep $@ -F $'\t'
+
+# // namespace comments are unneeded
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep $@ -P '}\s*//+\s*namespace\s*'
+
+# Broken symlinks
+find -L $ROOT_PATH -type l 2>/dev/null | grep -v contrib && echo "^ Broken symlinks found"
+
+# Duplicated or incorrect setting declarations
+SETTINGS_FILE=$(mktemp)
+cat $ROOT_PATH/src/Core/Settings.cpp $ROOT_PATH/src/Core/FormatFactorySettingsDeclaration.h | grep "M(" | awk '{print substr($2, 0, length($2) - 1) " " substr($1, 3, length($1) - 3) " SettingsDeclaration" }' > ${SETTINGS_FILE}
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep "extern const Settings" -T | awk '{print substr($5, 0, length($5) -1) " " substr($4, 9) " " substr($1, 0, length($1) - 1)}' >> ${SETTINGS_FILE}
+
+# Duplicate extern declarations for settings
+awk '{if (seen[$0]++) print $3 " -> " $1 ;}' ${SETTINGS_FILE} | while read line;
+do
+    echo "Found duplicated setting declaration in: $line"
+done
+
+# Incorrect declarations for settings
+for setting in $(awk '{print $1 " " $2}' ${SETTINGS_FILE} | sort | uniq | awk '{ print $1 }' | sort | uniq -d);
+do
+    expected=$(grep "^$setting " ${SETTINGS_FILE} | grep SettingsDeclaration | awk '{ print $2 }')
+    grep "^$setting " ${SETTINGS_FILE} | grep -v " $expected" | awk '{ print $3 " found setting " $1 " with type " $2 }' | while read line;
+    do
+      echo "In $line but it should be $expected"
+    done
+done
+
+rm ${SETTINGS_FILE}
+
+# Unused/Undefined/Duplicates ErrorCodes/ProfileEvents/CurrentMetrics
+declare -A EXTERN_TYPES
+EXTERN_TYPES[ErrorCodes]=int
+EXTERN_TYPES[ProfileEvents]=Event
+EXTERN_TYPES[CurrentMetrics]=Metric
+
+EXTERN_TYPES_EXCLUDES=(
+    ProfileEvents::global_counters
+    ProfileEvents::Event
+    ProfileEvents::Count
+    ProfileEvents::Counters
+    ProfileEvents::end
+    ProfileEvents::increment
+    ProfileEvents::incrementForLogMessage
+    ProfileEvents::getName
+    ProfileEvents::Timer
+    ProfileEvents::Type
+    ProfileEvents::TypeEnum
+    ProfileEvents::dumpToMapColumn
+    ProfileEvents::getProfileEvents
+    ProfileEvents::ThreadIdToCountersSnapshot
+    ProfileEvents::LOCAL_NAME
+    ProfileEvents::keeper_profile_events
+    ProfileEvents::CountersIncrement
+
+    CurrentMetrics::add
+    CurrentMetrics::sub
+    CurrentMetrics::get
+    CurrentMetrics::set
+    CurrentMetrics::end
+    CurrentMetrics::Increment
+    CurrentMetrics::Metric
+    CurrentMetrics::values
+    CurrentMetrics::Value
+    CurrentMetrics::keeper_metrics
+
+    ErrorCodes::ErrorCode
+    ErrorCodes::getName
+    ErrorCodes::increment
+    ErrorCodes::end
+    ErrorCodes::values
+    ErrorCodes::values[i]
+    ErrorCodes::getErrorCodeByName
+    ErrorCodes::Value
+)
+for extern_type in ${!EXTERN_TYPES[@]}; do
+    type_of_extern=${EXTERN_TYPES[$extern_type]}
+    allowed_chars='[_A-Za-z]+'
+
+    # Unused
+    # NOTE: to fix automatically, replace echo with:
+    # sed -i "/extern const $type_of_extern $val/d" $file
+    find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | {
+        # NOTE: the check is pretty dumb and distinguish only by the type_of_extern,
+        # and this matches with zkutil::CreateMode
+        grep -v -e 'src/Common/ZooKeeper/Types.h' -e 'src/Coordination/KeeperConstants.cpp'
+    } | {
+        grep -vP $EXCLUDE_DIRS | xargs grep -l -P "extern const $type_of_extern $allowed_chars"
+    } | while read file; do
+        grep -P "extern const $type_of_extern $allowed_chars;" $file | sed -r -e "s/^.*?extern const $type_of_extern ($allowed_chars);.*?$/\1/" | while read val; do
+            if ! grep -q "$extern_type::$val" $file; then
+                # Excludes for SOFTWARE_EVENT/HARDWARE_EVENT/CACHE_EVENT in ThreadProfileEvents.cpp
+                if [[ ! $extern_type::$val =~ ProfileEvents::Perf.* ]]; then
+                    echo "$extern_type::$val is defined but not used in file $file"
+                fi
+            fi
+        done
+    done
+
+    # Undefined
+    # NOTE: to fix automatically, replace echo with:
+    # ( grep -q -F 'namespace $extern_type' $file && \
+    #   sed -i -r "0,/(\s*)extern const $type_of_extern [$allowed_chars]+/s//\1extern const $type_of_extern $val;\n&/" $file || \
+    #     awk '{ print; if (ns == 1) { ns = 2 }; if (ns == 2) { ns = 0; print "namespace $extern_type\n{\n    extern const $type_of_extern '$val';\n}" } }; /namespace DB/ { ns = 1; };' < $file > ${file}.tmp && mv ${file}.tmp $file )
+    find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | {
+        grep -vP $EXCLUDE_DIRS | xargs grep -l -P "$extern_type::$allowed_chars"
+    } | while read file; do
+        grep -P "$extern_type::$allowed_chars" $file | grep -P -v '^\s*//' | sed -r -e "s/^.*?$extern_type::($allowed_chars).*?$/\1/" | while read val; do
+            if ! grep -q "extern const $type_of_extern $val" $file; then
+                if ! in_array "$extern_type::$val" "${EXTERN_TYPES_EXCLUDES[@]}"; then
+                    echo "$extern_type::$val is used in file $file but not defined"
+                fi
+            fi
+        done
+    done
+
+    # Duplicates
+    find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | {
+        grep -vP $EXCLUDE_DIRS | xargs grep -l -P "$extern_type::$allowed_chars"
+    } | while read file; do
+        grep -P "extern const $type_of_extern $allowed_chars;" $file | sort | uniq -c | grep -v -P ' +1 ' && echo "Duplicate $extern_type in file $file"
+    done
+done
+
+# Three or more consecutive empty lines
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    while read file; do awk '/^$/ { ++i; if (i > 2) { print "More than two consecutive empty lines in file '$file'" } } /./ { i = 0 }' $file; done
+
+# Check that every header file has #pragma once in first line
+find $ROOT_PATH/{src,programs,utils} -name '*.h' |
+    grep -vP $EXCLUDE_DIRS |
+    while read file; do [[ $(head -n1 $file) != '#pragma once' ]] && echo "File $file must have '#pragma once' in first line"; done
+
+# Too many exclamation marks
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -F '!!!' | grep -P '.' && echo "Too many exclamation marks (looks dirty, unconfident)."
+
+# Exclamation mark in a message
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -F '!",' | grep -P '.' && echo "No need for an exclamation mark (looks dirty, unconfident)."
+
+# Trailing whitespaces
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -n -P ' $' | grep -n -P '.' && echo "^ Trailing whitespaces."
+
+# Forbid stringstream because it's easy to use them incorrectly and hard to debug possible issues
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -P 'std::[io]?stringstream' | grep -v "STYLE_CHECK_ALLOW_STD_STRING_STREAM" && echo "Use WriteBufferFromOwnString or ReadBufferFromString instead of std::stringstream"
+
+# Forbid std::cerr/std::cout in src (fine in programs/utils)
+std_cerr_cout_excludes=(
+    /examples/
+    /tests/
+    _fuzzer
+    # OK
+    src/Common/ProgressIndication.cpp
+    # only under #ifdef DBMS_HASH_MAP_DEBUG_RESIZES, that is used only in tests
+    src/Common/HashTable/HashTable.h
+    # SensitiveDataMasker::printStats()
+    src/Common/SensitiveDataMasker.cpp
+    # StreamStatistics::print()
+    src/Compression/LZ4_decompress_faster.cpp
+    # ContextSharedPart with subsequent std::terminate()
+    src/Interpreters/Context.cpp
+    # IProcessor::dump()
+    src/Processors/IProcessor.cpp
+    src/Client/ClientApplicationBase.cpp
+    src/Client/ClientBase.cpp
+    src/Client/LineReader.cpp
+    src/Client/QueryFuzzer.cpp
+    src/Client/Suggest.cpp
+    src/Client/ClientBase.h
+    src/Client/LineReader.h
+    src/Client/ReplxxLineReader.h
+    src/Bridge/IBridge.cpp
+    src/Daemon/BaseDaemon.cpp
+    src/Loggers/Loggers.cpp
+    src/Common/GWPAsan.cpp
+    src/Common/ProgressIndication.h
+)
+sources_with_std_cerr_cout=( $(
+    find $ROOT_PATH/{src,base} -name '*.h' -or -name '*.cpp' | \
+        grep -vP $EXCLUDE_DIRS | \
+        grep -F -v $(printf -- "-e %s " "${std_cerr_cout_excludes[@]}") | \
+        xargs grep -F --with-filename -e std::cerr -e std::cout | cut -d: -f1 | sort -u
+) )
+
+# Exclude comments
+for src in "${sources_with_std_cerr_cout[@]}"; do
+    # suppress stderr, since it may contain warning for #pargma once in headers
+    if gcc -fpreprocessed -dD -E "$src" 2>/dev/null | grep -F -q -e std::cerr -e std::cout; then
+        echo "$src: uses std::cerr/std::cout"
+    fi
+done
+
+expect_tests=( $(find $ROOT_PATH/tests/queries -name '*.expect') )
+for test_case in "${expect_tests[@]}"; do
+    pattern="^exp_internal -f \$CLICKHOUSE_TMP/\$basename.debuglog 0$"
+    grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'"
+
+    if grep -q "^spawn.*CLICKHOUSE_CLIENT_BINARY$" "$test_case"; then
+        pattern="^spawn.*CLICKHOUSE_CLIENT_BINARY.*--history_file$"
+        grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'"
+    fi
+
+    # Otherwise expect_after/expect_before will not bail without stdin attached
+    # (and actually this is a hack anyway, correct way is to use $any_spawn_id)
+    pattern="-i \$any_spawn_id timeout"
+    grep -q -- "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'"
+    pattern="-i \$any_spawn_id eof"
+    grep -q -- "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'"
+done
+
+# Forbid non-unique error codes
+if [[ "$(grep -Po "M\([0-9]*," $ROOT_PATH/src/Common/ErrorCodes.cpp | wc -l)" != "$(grep -Po "M\([0-9]*," $ROOT_PATH/src/Common/ErrorCodes.cpp | sort | uniq | wc -l)" ]]
+then
+    echo "ErrorCodes.cpp contains non-unique error codes"
+fi
+
+# Check that there is no system-wide libraries/headers in use.
+#
+# NOTE: it is better to override find_path/find_library in cmake, but right now
+# it is not possible, see [1] for the reference.
+#
+#   [1]: git grep --recurse-submodules -e find_library -e find_path contrib
+if git grep -e find_path -e find_library -- :**CMakeLists.txt; then
+    echo "There is find_path/find_library usage. ClickHouse should use everything bundled. Consider adding one more contrib module."
+fi
+
+# Forbid std::filesystem::is_symlink and std::filesystem::read_symlink, because it's easy to use them incorrectly
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -P '::(is|read)_symlink' | grep -v "STYLE_CHECK_ALLOW_STD_FS_SYMLINK" && echo "Use DB::FS::isSymlink and DB::FS::readSymlink instead"
+
+# Forbid __builtin_unreachable(), because it's hard to debug when it becomes reachable
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -P '__builtin_unreachable' && echo "Use UNREACHABLE() from defines.h instead"
+
+# Forbid mt19937() and random_device() which are outdated and slow
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -P '(std::mt19937|std::mersenne_twister_engine|std::random_device)' && echo "Use pcg64_fast (from pcg_random.h) and randomSeed (from Common/randomSeed.h) instead"
+
+# Require checking return value of close(),
+# since it can hide fd misuse and break other places.
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -e ' close(.*fd' -e ' ::close(' | grep -v = && echo "Return value of close() should be checked"
+
+# A small typo can lead to debug code in release builds, see https://github.com/ClickHouse/ClickHouse/pull/47647
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -l -F '#ifdef NDEBUG' | xargs -I@FILE awk '/#ifdef NDEBUG/ { inside = 1; dirty = 1 } /#endif/ { if (inside && dirty) { print "File @FILE has suspicious #ifdef NDEBUG, possibly confused with #ifndef NDEBUG" }; inside = 0 } /#else/ { dirty = 0 }' @FILE
+
+# If a user is doing dynamic or typeid cast with a pointer, and immediately dereferencing it, it is unsafe.
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep --line-number -P '(dynamic|typeid)_cast<[^>]+\*>\([^\(\)]+\)->' | grep -P '.' && echo "It's suspicious when you are doing a dynamic_cast or typeid_cast with a pointer and immediately dereferencing it. Use references instead of pointers or check a pointer to nullptr."
+
+# Check for bad punctuation: whitespace before comma.
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P --line-number '\w ,' | grep -v 'bad punctuation is ok here' && echo "^ There is bad punctuation: whitespace before comma. You should write it like this: 'Hello, world!'"
+
+# Check usage of std::regex which is too bloated and slow.
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P --line-number 'std::regex' | grep -P '.' && echo "^ Please use re2 instead of std::regex"
+
+# Cyrillic characters hiding inside Latin.
+find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | grep -v StorageSystemContributors.generated.cpp | xargs grep -P --line-number '[a-zA-Z][а-яА-ЯёЁ]|[а-яА-ЯёЁ][a-zA-Z]' && echo "^ Cyrillic characters found in unexpected place."
+
+# Orphaned header files.
+join -v1 <(find $ROOT_PATH/{src,programs,utils} -name '*.h' -printf '%f\n' | sort | uniq) <(find $ROOT_PATH/{src,programs,utils} -name '*.cpp' -or -name '*.c' -or -name '*.h' -or -name '*.S' | xargs grep --no-filename -o -P '[\w-]+\.h' | sort | uniq) |
+    grep . && echo '^ Found orphan header files.'
+
+# Don't allow dynamic compiler check with CMake, because we are using hermetic, reproducible, cross-compiled, static (TLDR, good) builds.
+ls -1d $ROOT_PATH/contrib/*-cmake | xargs -I@ find @ -name 'CMakeLists.txt' -or -name '*.cmake' | xargs grep --with-filename -i -P 'check_c_compiler_flag|check_cxx_compiler_flag|check_c_source_compiles|check_cxx_source_compiles|check_include_file|check_symbol_exists|cmake_push_check_state|cmake_pop_check_state|find_package|CMAKE_REQUIRED_FLAGS|CheckIncludeFile|CheckCCompilerFlag|CheckCXXCompilerFlag|CheckCSourceCompiles|CheckCXXSourceCompiles|CheckCSymbolExists|CheckCXXSymbolExists' | grep -v Rust && echo "^ It's not allowed to have dynamic compiler checks with CMake."
+
+# Wrong spelling of abbreviations, e.g. SQL is right, Sql is wrong. XMLHttpRequest is very wrong.
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -P 'Sql|Html|Xml|Cpu|Tcp|Udp|Http|Db|Json|Yaml' | grep -v -P 'RabbitMQ|Azure|Aws|aws|Avro|IO/S3' &&
+    echo "Abbreviations such as SQL, XML, HTTP, should be in all caps. For example, SQL is right, Sql is wrong. XMLHttpRequest is very wrong."
+
+find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
+    grep -vP $EXCLUDE_DIRS |
+    xargs grep -F -i 'ErrorCodes::LOGICAL_ERROR, "Logical error:' &&
+    echo "If an exception has LOGICAL_ERROR code, there is no need to include the text 'Logical error' in the exception message, because then the phrase 'Logical error' will be printed twice."
+
+# There shouldn't be any code snippets under GPL or LGPL
+find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL"
+
+PATTERN="allow_";
+DIFF=$(comm -3 <(grep -o "\b$PATTERN\w*\b" $ROOT_PATH/src/Core/Settings.cpp | sort -u) <(grep -o -h "\b$PATTERN\w*\b" $ROOT_PATH/src/Databases/enableAllExperimentalSettings.cpp $ROOT_PATH/utils/check-style/experimental_settings_ignore.txt | sort -u));
+[ -n "$DIFF" ] && echo "$DIFF" && echo "^^ Detected 'allow_*' settings that might need to be included in src/Databases/enableAllExperimentalSettings.cpp" && echo "Alternatively, consider adding an exception to utils/check-style/experimental_settings_ignore.txt"

ci_v2/jobs/scripts/check_style/check_submodules.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# The script checks if all submodules defined in $GIT_ROOT/.gitmodules exist in $GIT_ROOT/contrib
+
+set -e
+
+GIT_ROOT="."
+
+cd "$GIT_ROOT"
+
+# Remove keys for submodule.*.path parameters, the values are separated by \0
+# and check if the directory exists
+git config --file .gitmodules --null --get-regexp path | sed -z 's|.*\n||' | \
+  xargs -P100 -0 --no-run-if-empty -I{} bash -c 'if ! test -d '"'{}'"'; then echo Directory for submodule {} is not found; exit 1; fi' 2>&1
+
+
+# And check that the submodule is fine
+git config --file .gitmodules --null --get-regexp path | sed -z 's|.*\n||' | \
+  xargs -P100 -0 --no-run-if-empty -I{} git submodule status -q '{}' 2>&1
+
+
+# All submodules should be from https://github.com/
+git config --file "$ROOT_PATH/.gitmodules" --get-regexp 'submodule\..+\.url' | \
+while read -r line; do
+    name=${line#submodule.}; name=${name%.url*}
+    url=${line#* }
+    [[ "$url" != 'https://github.com/'* ]] && echo "All submodules should be from https://github.com/, submodule '$name' has '$url'"
+done
+
+# All submodules should be of this form: [submodule "contrib/libxyz"] (for consistency, the submodule name does matter too much)
+# - restrict the check to top-level .gitmodules file
+git config --file "$ROOT_PATH/.gitmodules" --get-regexp 'submodule\..+\.path' | \
+while read -r line; do
+    name=${line#submodule.}; name=${name%.path*}
+    path=${line#* }
+    [ "$name" != "$path" ] && echo "Submodule name '$name' is not equal to it's path '$path'"
+done

ci_v2/jobs/scripts/check_style/check_typos.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Check for typos in code.
+
+ROOT_PATH="."
+
+#FIXME: check all (or almost all) repo
+codespell \
+    --skip "*generated*,*gperf*,*.bin,*.mrk*,*.idx,checksums.txt,*.dat,*.pyc,*.kate-swp,*obfuscateQueries.cpp,d3-*.js,*.min.js,*.sum,${ROOT_PATH}/utils/check-style/aspell-ignore" \
+    --ignore-words "${ROOT_PATH}/utils/check-style/codespell-ignore-words.list" \
+    --exclude-file "${ROOT_PATH}/utils/check-style/codespell-ignore-lines.list" \
+    --quiet-level 2 \
+    "$ROOT_PATH"/{src,base,programs,utils} \
+    $@ | grep -P '.' \
+    && echo -e "\nFound some typos in code.\nSee the files utils/check-style/codespell* if you want to add an exception."

ci_v2/jobs/scripts/check_style/checks_to_refactor.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+ROOT_PATH="."
+
+# Queries to system.query_log/system.query_thread_log should have current_database = currentDatabase() condition
+# NOTE: it is not that accurate, but at least something.
+tests_with_query_log=( $(
+    find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' |
+        xargs grep --with-filename -e system.query_log -e system.query_thread_log | cut -d: -f1 | sort -u
+) )
+for test_case in "${tests_with_query_log[@]}"; do
+    grep -qE current_database.*currentDatabase "$test_case" || {
+        grep -qE 'current_database.*\$CLICKHOUSE_DATABASE' "$test_case"
+    } || echo "Queries to system.query_log/system.query_thread_log does not have current_database = currentDatabase() condition in $test_case"
+done
+
+grep -iE 'SYSTEM STOP MERGES;?$' -R $ROOT_PATH/tests/queries && echo "Merges cannot be disabled globally in fast/stateful/stateless tests, because it will break concurrently running queries"
+
+
+# Queries to:
+tables_with_database_column=(
+    system.tables
+    system.parts
+    system.detached_parts
+    system.parts_columns
+    system.columns
+    system.projection_parts
+    system.mutations
+)
+# should have database = currentDatabase() condition
+#
+# NOTE: it is not that accuate, but at least something.
+tests_with_database_column=( $(
+    find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' |
+        xargs grep --with-filename $(printf -- "-e %s " "${tables_with_database_column[@]}") |
+        grep -v -e ':--' -e ':#' |
+        cut -d: -f1 | sort -u
+) )
+for test_case in "${tests_with_database_column[@]}"; do
+    grep -qE database.*currentDatabase "$test_case" || {
+        grep -qE 'database.*\$CLICKHOUSE_DATABASE' "$test_case"
+    } || {
+        # explicit database
+        grep -qE "database[ ]*=[ ]*'" "$test_case"
+    } || {
+        echo "Queries to ${tables_with_database_column[*]} does not have database = currentDatabase()/\$CLICKHOUSE_DATABASE condition in $test_case"
+    }
+done
+
+# Queries with ReplicatedMergeTree
+# NOTE: it is not that accuate, but at least something.
+tests_with_replicated_merge_tree=( $(
+    find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' |
+        xargs grep --with-filename -e "Replicated.*MergeTree[ ]*(.*" | cut -d: -f1 | sort -u
+) )
+for test_case in "${tests_with_replicated_merge_tree[@]}"; do
+    case "$test_case" in
+        *.gen.*)
+            ;;
+        *.sh)
+            test_case_zk_prefix="\(\$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX\|{database}\)"
+            grep -q -e "Replicated.*MergeTree[ ]*(.*$test_case_zk_prefix" "$test_case" || echo "Replicated.*MergeTree should contain '$test_case_zk_prefix' in zookeeper path to avoid overlaps ($test_case)"
+            ;;
+        *.sql|*.sql.j2)
+            test_case_zk_prefix="\({database}\|currentDatabase()\|{uuid}\|{default_path_test}\)"
+            grep -q -e "Replicated.*MergeTree[ ]*(.*$test_case_zk_prefix" "$test_case" || echo "Replicated.*MergeTree should contain '$test_case_zk_prefix' in zookeeper path to avoid overlaps ($test_case)"
+            ;;
+        *.py)
+            # Right now there is not such tests anyway
+            echo "No ReplicatedMergeTree style check for *.py ($test_case)"
+            ;;
+    esac
+done
+
+# The stateful directory should only contain the tests that depend on the test dataset (hits or visits).
+find $ROOT_PATH/tests/queries/1_stateful -name '*.sql' -or -name '*.sh' | grep -v '00076_system_columns_bytes' | xargs -I{} bash -c 'grep -q -P "hits|visits" "{}" || echo "The test {} does not depend on the test dataset (hits or visits table) and should be located in the 0_stateless directory. You can also add an exception to the check-style script."'
+
+# Check for existence of __init__.py files
+for i in "${ROOT_PATH}"/tests/integration/test_*; do FILE="${i}/__init__.py"; [ ! -f "${FILE}" ] && echo "${FILE} should exist for every integration test"; done
+
+# Check for executable bit on non-executable files
+find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} '(' -name '*.cpp' -or -name '*.h' -or -name '*.sql' -or -name '*.j2' -or -name '*.xml' -or -name '*.reference' -or -name '*.txt' -or -name '*.md' ')' -and -executable | grep -P '.' && echo "These files should not be executable."
+
+# Check for BOM
+find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xEF\xBB\xBF' | grep -P '.' && echo "Files should not have UTF-8 BOM"
+find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFF\xFE' | grep -P '.' && echo "Files should not have UTF-16LE BOM"
+find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFE\xFF' | grep -P '.' && echo "Files should not have UTF-16BE BOM"
+
+# Conflict markers
+find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' |
+    xargs grep -P '^(<<<<<<<|=======|>>>>>>>)$' | grep -P '.' && echo "Conflict markers are found in files"
+
+# DOS/Windows newlines
+find $ROOT_PATH/{base,src,programs,utils,docs} -name '*.md' -or -name '*.h' -or -name '*.cpp' -or -name '*.js' -or -name '*.py' -or -name '*.html' | xargs grep -l -P '\r$' && echo "^ Files contain DOS/Windows newlines (\r\n instead of \n)."
+
+# # workflows check
+# act --list --directory="$ROOT_PATH" 1>/dev/null 2>&1 || act --list --directory="$ROOT_PATH" 2>&1
+# actionlint -ignore 'reusable workflow call.+' || :

ci_v2/jobs/scripts/check_style/double_whitespaces.pl

@@ -0,0 +1,37 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Find double whitespace such as "a,  b, c" that looks very ugly and annoying.
+# But skip double whitespaces if they are used as an alignment - by comparing to surrounding lines.
+
+my $ret = 0;
+
+foreach my $file (@ARGV)
+{
+    my @array;
+
+    open (FH,'<',$file);
+    while (<FH>)
+    {
+        push @array, $_;
+    }
+
+    for (my $i = 1; $i < $#array; ++$i)
+    {
+        if ($array[$i] =~ ',( {2,3})[^ /]')
+        {
+            # https://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
+
+            if ((substr($array[$i - 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) # whitespaces are not part of alignment
+             && (substr($array[$i + 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/)
+             && $array[$i] !~ /(-?\d+\w*,\s+){3,}/) # this is not a number table like { 10, -1,  2 }
+            {
+                print($file . ":" . ($i + 1) . $array[$i]);
+                $ret = 1;
+            }
+        }
+    }
+}
+
+exit $ret;

ci_v2/settings/definitions.py

@@ -0,0 +1,251 @@
+from praktika import Docker, Secret
+
+S3_BUCKET_NAME = "clickhouse-builds"
+S3_BUCKET_HTTP_ENDPOINT = "clickhouse-builds.s3.amazonaws.com"
+
+
+class RunnerLabels:
+    CI_SERVICES = "ci_services"
+    CI_SERVICES_EBS = "ci_services_ebs"
+
+
+BASE_BRANCH = "master"
+
+SECRETS = [
+    Secret.Config(
+        name="dockerhub_robot_password",
+        type=Secret.Type.AWS_SSM_VAR,
+    ),
+    Secret.Config(
+        name="woolenwolf_gh_app.clickhouse-app-id",
+        type=Secret.Type.AWS_SSM_SECRET,
+    ),
+    Secret.Config(
+        name="woolenwolf_gh_app.clickhouse-app-key",
+        type=Secret.Type.AWS_SSM_SECRET,
+    ),
+]
+
+DOCKERS = [
+    # Docker.Config(
+    #     name="clickhouse/binary-builder",
+    #     path="./docker/packager/binary-builder",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=[],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/cctools",
+    #     path="./docker/packager/cctools",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=[],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/test-old-centos",
+    #     path="./docker/test/compatibility/centos",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=[],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/test-old-ubuntu",
+    #     path="./docker/test/compatibility/ubuntu",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=[],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/test-util",
+    #     path="./docker/test/util",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=[],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/integration-test",
+    #     path="./docker/test/integration/base",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/fuzzer",
+    #     path="./docker/test/fuzzer",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/performance-comparison",
+    #     path="./docker/test/performance-comparison",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=[],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/fasttest",
+    #     path="./docker/test/fasttest",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-util"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/test-base",
+    #     path="./docker/test/base",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-util"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/clickbench",
+    #     path="./docker/test/clickbench",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/keeper-jepsen-test",
+    #     path="./docker/test/keeper-jepsen",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/server-jepsen-test",
+    #     path="./docker/test/server-jepsen",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/sqllogic-test",
+    #     path="./docker/test/sqllogic",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/sqltest",
+    #     path="./docker/test/sqltest",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/stateless-test",
+    #     path="./docker/test/stateless",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/stateful-test",
+    #     path="./docker/test/stateful",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/stateless-test"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/stress-test",
+    #     path="./docker/test/stress",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/stateful-test"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/unit-test",
+    #     path="./docker/test/unit",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    # Docker.Config(
+    #     name="clickhouse/integration-tests-runner",
+    #     path="./docker/test/integration/runner",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+    Docker.Config(
+        name="clickhouse/style-test",
+        path="./ci_v2/docker/style-test",
+        platforms=Docker.Platforms.arm_amd,
+        depends_on=[],
+    ),
+    # Docker.Config(
+    #     name="clickhouse/docs-builder",
+    #     path="./docker/docs/builder",
+    #     arm64=True,
+    #     amd64=True,
+    #     depends_on=["clickhouse/test-base"],
+    # ),
+]
+
+# TODO:
+# "docker/test/integration/s3_proxy": {
+#     "name": "clickhouse/s3-proxy",
+#     "dependent": []
+# },
+# "docker/test/integration/resolver": {
+#     "name": "clickhouse/python-bottle",
+#     "dependent": []
+# },
+# "docker/test/integration/helper_container": {
+#     "name": "clickhouse/integration-helper",
+#     "dependent": []
+# },
+# "docker/test/integration/mysql_golang_client": {
+#     "name": "clickhouse/mysql-golang-client",
+#     "dependent": []
+# },
+# "docker/test/integration/dotnet_client": {
+#     "name": "clickhouse/dotnet-client",
+#     "dependent": []
+# },
+# "docker/test/integration/mysql_java_client": {
+#     "name": "clickhouse/mysql-java-client",
+#     "dependent": []
+# },
+# "docker/test/integration/mysql_js_client": {
+#     "name": "clickhouse/mysql-js-client",
+#     "dependent": []
+# },
+# "docker/test/integration/mysql_php_client": {
+#     "name": "clickhouse/mysql-php-client",
+#     "dependent": []
+# },
+# "docker/test/integration/postgresql_java_client": {
+#     "name": "clickhouse/postgresql-java-client",
+#     "dependent": []
+# },
+# "docker/test/integration/kerberos_kdc": {
+#     "only_amd64": true,
+#     "name": "clickhouse/kerberos-kdc",
+#     "dependent": []
+# },
+# "docker/test/integration/kerberized_hadoop": {
+#     "only_amd64": true,
+#     "name": "clickhouse/kerberized-hadoop",
+#     "dependent": []
+# },
+# "docker/test/sqlancer": {
+#     "name": "clickhouse/sqlancer-test",
+#     "dependent": []
+# },
+# "docker/test/install/deb": {
+#     "name": "clickhouse/install-deb-test",
+#     "dependent": []
+# },
+# "docker/test/install/rpm": {
+#     "name": "clickhouse/install-rpm-test",
+#     "dependent": []
+# },
+# "docker/test/integration/nginx_dav": {
+#     "name": "clickhouse/nginx-dav",
+#     "dependent": []
+# }
+
+
+class JobNames:
+    STYLE_CHECK = "Style Check"

ci_v2/settings/settings.py

@@ -0,0 +1,20 @@
+from ci_v2.settings.definitions import (
+    S3_BUCKET_HTTP_ENDPOINT,
+    S3_BUCKET_NAME,
+    RunnerLabels,
+)
+
+S3_ARTIFACT_PATH = f"{S3_BUCKET_NAME}/artifacts"
+CI_CONFIG_RUNS_ON = [RunnerLabels.CI_SERVICES]
+DOCKER_BUILD_RUNS_ON = [RunnerLabels.CI_SERVICES_EBS]
+CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache"
+HTML_S3_PATH = f"{S3_BUCKET_NAME}/reports"
+S3_BUCKET_TO_HTTP_ENDPOINT = {S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT}
+
+DOCKERHUB_USERNAME = "robotclickhouse"
+DOCKERHUB_SECRET = "dockerhub_robot_password"
+
+CI_DB_DB_NAME = "default"
+CI_DB_TABLE_NAME = "checks"
+
+INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS = ""

ci_v2/workflows/pull_request.py

@@ -0,0 +1,44 @@
+from typing import List
+
+from ci_v2.settings.definitions import (
+    BASE_BRANCH,
+    DOCKERS,
+    SECRETS,
+    JobNames,
+    RunnerLabels,
+)
+from praktika import Job, Workflow
+
+style_check_job = Job.Config(
+    name=JobNames.STYLE_CHECK,
+    runs_on=[RunnerLabels.CI_SERVICES],
+    command="python3 ./ci_v2/jobs/check_style.py",
+    run_in_docker="clickhouse/style-test",
+)
+
+workflow = Workflow.Config(
+    name="PR",
+    event=Workflow.Event.PULL_REQUEST,
+    base_branches=[BASE_BRANCH],
+    jobs=[
+        style_check_job,
+    ],
+    dockers=DOCKERS,
+    secrets=SECRETS,
+    enable_cache=True,
+    enable_report=True,
+    enable_merge_ready_status=True,
+)
+
+WORKFLOWS = [
+    workflow,
+]  # type: List[Workflow.Config]
+
+
+if __name__ == "__main__":
+    # example: local job test inside praktika environment
+    from praktika.runner import Runner
+
+    Runner.generate_dummy_environment(workflow, style_check_job)
+
+    Runner().run(workflow, style_check_job)

contrib/FP16-cmake/CMakeLists.txt

@@ -1 +1,4 @@
-# See contrib/usearch-cmake/CMakeLists.txt
+set (FP16_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/FP16/")
+
+add_library(_fp16 INTERFACE)
+target_include_directories(_fp16 SYSTEM INTERFACE ${FP16_PROJECT_DIR}/include)

contrib/SimSIMD

@@ -1 +1 @@
-Subproject commit 91a76d1ac519b3b9dc8957734a3dabd985f00c26
+Subproject commit ff51434d90c66f916e94ff05b24530b127aa4cff

contrib/SimSIMD-cmake/CMakeLists.txt

@@ -1 +1,4 @@
-# See contrib/usearch-cmake/CMakeLists.txt
+set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD")
+
+add_library(_simsimd INTERFACE)
+target_include_directories(_simsimd SYSTEM INTERFACE "${SIMSIMD_PROJECT_DIR}/include")

contrib/boost-cmake/CMakeLists.txt

@@ -69,6 +69,11 @@ set (SRCS_PROGRAM_OPTIONS
     "${LIBRARY_DIR}/libs/program_options/src/winmain.cpp"
 )
 
+# Always compile this file with the highest possible level of optimizations, even in Debug builds.
+# Otherwise the client takes too long to start and SQL stateless tests (many options to parse)
+# https://github.com/ClickHouse/ClickHouse/issues/65745
+set_source_files_properties(${SRCS_PROGRAM_OPTIONS} PROPERTIES COMPILE_FLAGS "-O3")
+
 add_library (_boost_program_options ${SRCS_PROGRAM_OPTIONS})
 add_library (boost::program_options ALIAS _boost_program_options)
 target_include_directories (_boost_program_options SYSTEM BEFORE PUBLIC ${LIBRARY_DIR})

contrib/jemalloc-cmake/CMakeLists.txt

@@ -164,15 +164,6 @@ target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
 # Because our coverage callbacks call malloc, and recursive call of malloc could not work.
 target_compile_options(_jemalloc PRIVATE ${WITHOUT_COVERAGE_FLAGS_LIST})
 
-if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
-    target_compile_definitions(_jemalloc PRIVATE
-        -DJEMALLOC_DEBUG=1
-        # Usage examples:
-        # - MALLOC_CONF=log:.
-        # - MALLOC_CONF='log:core.malloc.exit|core.sallocx.entry|core.sdallocx.entry'
-        -DJEMALLOC_LOG=1)
-endif ()
-
 target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1)
 
 # jemalloc provides support two unwind flavors:

contrib/mariadb-connector-c-cmake/CMakeLists.txt

@@ -50,7 +50,11 @@ if(NOT MARIADB_UNIX_ADDR)
   set(MARIADB_UNIX_ADDR "/tmp/mysql.sock")
 endif()
 
-set(HAVE_ALLOCA_H 1)
+if (OS_FREEBSD)
+  set(HAVE_ALLOCA_H 0)
+else()
+  set(HAVE_ALLOCA_H 1)
+endif()
 set(HAVE_ARPA_INET_H 1)
 set(HAVE_DLFCN_H 1)
 set(HAVE_FCNTL_H 1)

contrib/usearch

@@ -1 +1 @@
-Subproject commit 7a8967cb442b08ca20c3dd781414378e65957d37
+Subproject commit d1d33eac94acd3b628e0b446c927ec3295ef63c7

contrib/usearch-cmake/CMakeLists.txt

@@ -1,14 +1,9 @@
-set(FP16_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/FP16")
-set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD")
 set(USEARCH_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/usearch")
 
 add_library(_usearch INTERFACE)
+target_include_directories(_usearch SYSTEM INTERFACE ${USEARCH_PROJECT_DIR}/include)
 
-target_include_directories(_usearch SYSTEM INTERFACE
-    ${FP16_PROJECT_DIR}/include
-    ${SIMSIMD_PROJECT_DIR}/include
-    ${USEARCH_PROJECT_DIR}/include)
-
+target_link_libraries(_usearch INTERFACE _fp16)
 target_compile_definitions(_usearch INTERFACE USEARCH_USE_FP16LIB)
 
 # target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD)

docker/keeper/Dockerfile

@@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.8.4.13"
+ARG VERSION="24.9.1.3278"
 ARG PACKAGES="clickhouse-keeper"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/reqgenerator.py

@@ -2,19 +2,29 @@
 # To run this script you must install docker and piddeptree python package
 #
 
-import subprocess
 import os
+import subprocess
 import sys
 
 
-def build_docker_deps(image_name, imagedir):
-    cmd = f"""docker run  --entrypoint "/bin/bash" {image_name} -c "pip install pipdeptree 2>/dev/null 1>/dev/null && pipdeptree --freeze  --warn silence | sed 's/ \+//g' | sort | uniq" > {imagedir}/requirements.txt"""
+def build_docker_deps(image_name: str, imagedir: str) -> None:
+    print("Fetch the newest manifest for", image_name)
+    pip_cmd = (
+        "pip install pipdeptree 2>/dev/null 1>/dev/null && pipdeptree --freeze "
+        "--warn silence --exclude pipdeptree"
+    )
+    # /=/!d - remove dependencies without pin
+    # ubuntu - ignore system packages
+    # \s - remove spaces
+    sed = r"sed '/==/!d; /==.*+ubuntu/d; s/\s//g'"
+    cmd = rf"""docker run --rm --entrypoint "/bin/bash" {image_name} -c "{pip_cmd} | {sed} | sort -u" > {imagedir}/requirements.txt"""
+    print("Running the command:", cmd)
     subprocess.check_call(cmd, shell=True)
 
 
 def check_docker_file_install_with_pip(filepath):
     image_name = None
-    with open(filepath, "r") as f:
+    with open(filepath, "r", encoding="utf-8") as f:
         for line in f:
             if "docker build" in line:
                 arr = line.split(" ")
@@ -25,7 +35,7 @@ def check_docker_file_install_with_pip(filepath):
     return image_name, False
 
 
-def process_affected_images(images_dir):
+def process_affected_images(images_dir: str) -> None:
     for root, _dirs, files in os.walk(images_dir):
         for f in files:
             if f == "Dockerfile":

docker/server/Dockerfile.alpine

@@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.8.4.13"
+ARG VERSION="24.9.1.3278"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/server/Dockerfile.ubuntu

@@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="24.8.4.13"
+ARG VERSION="24.9.1.3278"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 #docker-official-library:off

docker/test/fuzzer/generate-test-j2.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
-from argparse import ArgumentParser
-import os
-import jinja2
 import itertools
+import os
+from argparse import ArgumentParser
+
+import jinja2
 
 
 def removesuffix(text, suffix):

docker/test/integration/hive_server/http_api_server.py

@@ -1,7 +1,8 @@
+import datetime
 import os
 import subprocess
-import datetime
-from flask import Flask, flash, request, redirect, url_for
+
+from flask import Flask, flash, redirect, request, url_for
 
 
 def run_command(command, wait=False):

docker/test/integration/runner/Dockerfile

@@ -48,7 +48,7 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \
     && apt-get update \
     && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
-        docker-ce='5:23.*' \
+        docker-ce='5:23.*' docker-compose-plugin='2.29.*' \
     && rm -rf \
         /var/lib/apt/lists/* \
         /var/cache/debconf \

docker/test/integration/runner/requirements.txt

@@ -1,15 +1,13 @@
 PyHDFS==0.3.1
-PyJWT==2.3.0
-PyMySQL==1.1.0
+PyJWT==2.4.0
+PyMySQL==1.1.1
 PyNaCl==1.5.0
-PyYAML==5.3.1
 SecretStorage==3.3.1
 argon2-cffi-bindings==21.2.0
 argon2-cffi==23.1.0
 async-timeout==4.0.3
 asyncio==3.4.3
-attrs==23.2.0
-avro==1.10.2
+avro==1.11.3
 azure-core==1.30.1
 azure-storage-blob==12.19.0
 bcrypt==4.1.3
@@ -24,18 +22,13 @@ cffi==1.16.0
 charset-normalizer==3.3.2
 click==8.1.7
 confluent-kafka==2.3.0
-cryptography==3.4.8
+cryptography==42.0.0
 dbus-python==1.2.18
-decorator==5.1.1
 delta-spark==2.3.0
+deltalake==0.16.0
 dict2xml==1.7.4
 dicttoxml==1.7.16
-distro-info==1.1+ubuntu0.2
-distro==1.7.0
-docker-compose==1.29.2
 docker==6.1.3
-dockerpty==0.4.1
-docopt==0.6.2
 exceptiongroup==1.2.1
 execnet==2.1.1
 geomet==0.2.1.post1
@@ -49,7 +42,6 @@ iniconfig==2.0.0
 isodate==0.6.1
 jeepney==0.7.1
 jmespath==1.0.1
-jsonschema==3.2.0
 jwcrypto==1.5.6
 kafka-python==2.0.2
 kazoo==2.9.0
@@ -63,23 +55,22 @@ lz4==4.3.3
 minio==7.2.3
 more-itertools==8.10.0
 nats-py==2.6.0
+numpy==2.1.0
 oauthlib==3.2.0
 packaging==24.0
 paramiko==3.4.0
 pika==1.2.0
 pip==24.1.1
-pipdeptree==2.23.0
 pluggy==1.5.0
 protobuf==4.25.2
 psycopg2-binary==2.9.6
 py4j==0.10.9.5
-py==1.11.0
+pyarrow-hotfix==0.6
 pyarrow==17.0.0
 pycparser==2.22
 pycryptodome==3.20.0
 pymongo==3.11.0
 pyparsing==2.4.7
-pyrsistent==0.20.0
 pyspark==3.3.2
 pyspnego==0.10.2
 pytest-order==1.0.0
@@ -89,28 +80,22 @@ pytest-reportlog==0.4.0
 pytest-timeout==2.2.0
 pytest-xdist==3.5.0
 pytest==7.4.4
-python-apt==2.4.0+ubuntu3
 python-dateutil==2.9.0.post0
-python-dotenv==0.21.1
 pytz==2023.3.post1
 redis==5.0.1
 requests-kerberos==0.14.0
 requests==2.31.0
-retry==0.9.2
 s3transfer==0.10.1
-setuptools==59.6.0
+setuptools==70.0.0
 simplejson==3.19.2
 six==1.16.0
 soupsieve==2.5
-texttable==1.7.0
 tomli==2.0.1
 typing_extensions==4.11.0
 tzlocal==2.1
 unattended-upgrades==0.1
 urllib3==2.0.7
 wadllib==1.3.6
-websocket-client==0.59.0
-wheel==0.37.1
+websocket-client==1.8.0
+wheel==0.38.1
 zipp==1.0.0
-deltalake==0.16.0
-

docker/test/sqlancer/process_sqlancer_result.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 
-import os
-import logging
 import argparse
 import csv
+import logging
+import os
 
 
 def process_result(result_folder):

docker/test/sqltest/test.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 
-import os
-import yaml
 import html
+import os
 import random
 import string
-from clickhouse_driver import Client
 
+import yaml
+from clickhouse_driver import Client
 
 client = Client(host="localhost", port=9000)
 settings = {

docs/changelogs/v24.7.5.37-stable.md

@@ -0,0 +1,32 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.7.5.37-stable (f2533ca97be) FIXME as compared to v24.7.4.51-stable (70fe2f6fa52)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#68971](https://github.com/ClickHouse/ClickHouse/issues/68971): Fix the upper bound of the function `fromModifiedJulianDay`. It was supposed to be `9999-12-31` but was mistakenly set to `9999-01-01`. [#67583](https://github.com/ClickHouse/ClickHouse/pull/67583) ([PHO](https://github.com/depressed-pho)).
+* Backported in [#68816](https://github.com/ClickHouse/ClickHouse/issues/68816): Fixed crash in Parquet filtering when data types in the file substantially differ from requested types (e.g. `... FROM file('a.parquet', Parquet, 'x String')`, but the file has `x Int64`). Without this fix, use `input_format_parquet_filter_push_down = 0` as a workaround. [#68131](https://github.com/ClickHouse/ClickHouse/pull/68131) ([Michael Kolupaev](https://github.com/al13n321)).
+* Backported in [#69006](https://github.com/ClickHouse/ClickHouse/issues/69006): After https://github.com/ClickHouse/ClickHouse/pull/61984 `schema_inference_make_columns_nullable=0` still can make columns `Nullable` in Parquet/Arrow formats. The change was backward incompatible and users noticed the changes in the behaviour. This PR makes `schema_inference_make_columns_nullable=0` to work as before (no Nullable columns will be inferred) and introduces new value `auto` for this setting that will make columns `Nullable` only if data has information about nullability. [#68298](https://github.com/ClickHouse/ClickHouse/pull/68298) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#68884](https://github.com/ClickHouse/ClickHouse/issues/68884): Fixes [#50868](https://github.com/ClickHouse/ClickHouse/issues/50868). Small DateTime64 constant values returned by a nested subquery inside a distributed query were wrongly transformed to Nulls, thus causing errors and possible incorrect query results. [#68323](https://github.com/ClickHouse/ClickHouse/pull/68323) ([Shankar](https://github.com/shiyer7474)).
+* Backported in [#69027](https://github.com/ClickHouse/ClickHouse/issues/69027): Added back virtual columns ` _table` and `_database` to distributed tables. They were available until version 24.3. [#68672](https://github.com/ClickHouse/ClickHouse/pull/68672) ([Anton Popov](https://github.com/CurtizJ)).
+* Backported in [#68862](https://github.com/ClickHouse/ClickHouse/issues/68862): Fix possible error `Size of permutation (0) is less than required (...)` during Variant column permutation. [#68681](https://github.com/ClickHouse/ClickHouse/pull/68681) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#68788](https://github.com/ClickHouse/ClickHouse/issues/68788): Fix issue with materialized constant keys when hashing maps with arrays as keys in functions `sipHash(64/128)Keyed`. [#68731](https://github.com/ClickHouse/ClickHouse/pull/68731) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Backported in [#68909](https://github.com/ClickHouse/ClickHouse/issues/68909): Fix complex types metadata parsing in DeltaLake. Closes [#68739](https://github.com/ClickHouse/ClickHouse/issues/68739). [#68836](https://github.com/ClickHouse/ClickHouse/pull/68836) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#69158](https://github.com/ClickHouse/ClickHouse/issues/69158): Fix possible wrong result during anyHeavy state merge. [#68950](https://github.com/ClickHouse/ClickHouse/pull/68950) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#69118](https://github.com/ClickHouse/ClickHouse/issues/69118): Fix logical error when we have empty async insert. [#69080](https://github.com/ClickHouse/ClickHouse/pull/69080) ([Han Fei](https://github.com/hanfei1991)).
+
+#### NO CL CATEGORY
+
+* Backported in [#68944](https://github.com/ClickHouse/ClickHouse/issues/68944):. [#68897](https://github.com/ClickHouse/ClickHouse/pull/68897) ([Alexander Gololobov](https://github.com/davenger)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#68832](https://github.com/ClickHouse/ClickHouse/issues/68832): Turn off fault injection for insert in `01396_inactive_replica_cleanup_nodes_zookeeper`. [#68715](https://github.com/ClickHouse/ClickHouse/pull/68715) ([alesapin](https://github.com/alesapin)).
+* Backported in [#68779](https://github.com/ClickHouse/ClickHouse/issues/68779): Fix flaky test 00989_parallel_parts_loading. [#68737](https://github.com/ClickHouse/ClickHouse/pull/68737) ([alesapin](https://github.com/alesapin)).
+* Backported in [#68960](https://github.com/ClickHouse/ClickHouse/issues/68960): Fix 2477 timeout. [#68752](https://github.com/ClickHouse/ClickHouse/pull/68752) ([jsc0218](https://github.com/jsc0218)).
+* Backported in [#69050](https://github.com/ClickHouse/ClickHouse/issues/69050): Fix 01114_database_atomic flakiness. [#68930](https://github.com/ClickHouse/ClickHouse/pull/68930) ([Raúl Marín](https://github.com/Algunenano)).
+

docs/changelogs/v24.9.1.3278-stable.md

@@ -0,0 +1,500 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.9.1.3278-stable (6d058d82a8e) FIXME as compared to v24.9.1.1-new (e02b434d2fc)
+
+#### Backward Incompatible Change
+* Allow to write `SETTINGS` before `FORMAT` in a chain of queries with `UNION` when subqueries are inside parentheses. This closes [#39712](https://github.com/ClickHouse/ClickHouse/issues/39712). Change the behavior when a query has the SETTINGS clause specified twice in a sequence. The closest SETTINGS clause will have a preference for the corresponding subquery. In the previous versions, the outermost SETTINGS clause could take a preference over the inner one. [#60197](https://github.com/ClickHouse/ClickHouse/pull/60197) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Do not allow explicitly specifying UUID when creating a table in Replicated database. Also, do not allow explicitly specifying ZooKeeper path and replica name for *MergeTree tables in Replicated databases. [#66104](https://github.com/ClickHouse/ClickHouse/pull/66104) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Reimplement Dynamic type. Now when the limit of dynamic data types is reached new types are not casted to String but stored in a special data structure in binary format with binary encoded data type. Now any type ever inserted into Dynamic column can be read from it as subcolumn. [#68132](https://github.com/ClickHouse/ClickHouse/pull/68132) ([Pavel Kruglov](https://github.com/Avogar)).
+* Expressions like `a[b].c` are supported for named tuples, as well as named subscripts from arbitrary expressions, e.g., `expr().name`. This is useful for processing JSON. This closes [#54965](https://github.com/ClickHouse/ClickHouse/issues/54965). In previous versions, an expression of form `expr().name` was parsed as `tupleElement(expr(), name)`, and the query analyzer was searching for a column `name` rather than for the corresponding tuple element; while in the new version, it is changed to `tupleElement(expr(), 'name')`. In most cases, the previous version was not working, but it is possible to imagine a very unusual scenario when this change could lead to incompatibility: if you stored names of tuple elements in a column or an alias, that was named differently than the tuple element's name: `SELECT 'b' AS a, CAST([tuple(123)] AS 'Array(Tuple(b UInt8))') AS t, t[1].a`. It is very unlikely that you used such queries, but we still have to mark this change as potentially backward incompatible. [#68435](https://github.com/ClickHouse/ClickHouse/pull/68435) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* When the setting `print_pretty_type_names` is enabled, it will print `Tuple` data type in a pretty form in `SHOW CREATE TABLE` statements, `formatQuery` function, and in the interactive mode in `clickhouse-client` and `clickhouse-local`. In previous versions, this setting was only applied to `DESCRIBE` queries and `toTypeName`. This closes [#65753](https://github.com/ClickHouse/ClickHouse/issues/65753). [#68492](https://github.com/ClickHouse/ClickHouse/pull/68492) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### New Feature
+* Function `toStartOfInterval()` now has a new overload which emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function. ([#55619](https://github.com/ClickHouse/ClickHouse/issues/55619)). It allows to align date or timestamp values to multiples of a given interval from an *arbitrary* origin (instead of 0000-01-01 00:00:00.000 as *fixed* origin). For example, `SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));` returns `2023-01-01 14:44:30` which is a multiple of 1 minute intervals, starting from origin `2023-01-01 14:35:30`. [#56738](https://github.com/ClickHouse/ClickHouse/pull/56738) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Add support for `ATTACH PARTITION ALL FROM`. [#61987](https://github.com/ClickHouse/ClickHouse/pull/61987) ([Kirill Nikiforov](https://github.com/allmazz)).
+* Adds a setting `input_format_try_infer_variants` which allows Variant type to be inferred during schema inference for text formats when there is more than one possible type for column/array elements. [#63798](https://github.com/ClickHouse/ClickHouse/pull/63798) ([Shaun Struwig](https://github.com/Blargian)).
+* Introduced JSONCompactWithProgress format where ClickHouse outputs each row as a newline-delimited JSON object, including metadata, data, progress, totals, and statistics. [#66205](https://github.com/ClickHouse/ClickHouse/pull/66205) ([Alexey Korepanov](https://github.com/alexkorep)).
+* Implement new JSON data type. [#66444](https://github.com/ClickHouse/ClickHouse/pull/66444) ([Pavel Kruglov](https://github.com/Avogar)).
+* Add the `input_format_json_empty_as_default` setting which, when enabled, treats empty fields in JSON inputs as default values. Closes [#59339](https://github.com/ClickHouse/ClickHouse/issues/59339). [#66782](https://github.com/ClickHouse/ClickHouse/pull/66782) ([Alexis Arnaud](https://github.com/a-a-f)).
+* Added functions `overlay` and `overlayUTF8` which replace parts of a string by another string. Example: `SELECT overlay('Hello New York', 'Jersey', 11)` returns `Hello New Jersey`. [#66933](https://github.com/ClickHouse/ClickHouse/pull/66933) ([李扬](https://github.com/taiyang-li)).
+* Add new Command, Lightweight Delete In Partition ``` DELETE FROM [db.]table [ON CLUSTER cluster] [IN PARTITION partition_expr] WHERE expr; ``` ``` VM-114-29-tos :) select * from ads_app_poster_ip_source_channel_di_replicated_local;. [#67805](https://github.com/ClickHouse/ClickHouse/pull/67805) ([sunny](https://github.com/sunny19930321)).
+* Implemented comparison for `Interval` data type values so they are converting now to the least supertype. [#68057](https://github.com/ClickHouse/ClickHouse/pull/68057) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Add create_if_not_exists setting to default to IF NOT EXISTS behavior during CREATE statements. [#68164](https://github.com/ClickHouse/ClickHouse/pull/68164) ([Peter Nguyen](https://github.com/petern48)).
+* Makes possible to read Iceberg tables in Azure and locally. [#68210](https://github.com/ClickHouse/ClickHouse/pull/68210) ([Daniil Ivanik](https://github.com/divanik)).
+* Add aggregate functions distinctDynamicTypes/distinctJSONPaths/distinctJSONPathsAndTypes for better introspection of JSON column type content. [#68463](https://github.com/ClickHouse/ClickHouse/pull/68463) ([Pavel Kruglov](https://github.com/Avogar)).
+* Query cache entries can now be dropped by tag. For example, the query cache entry created by `SELECT 1 SETTINGS use_query_cache = true, query_cache_tag = 'abc'` can now be dropped by `SYSTEM DROP QUERY CACHE TAG 'abc'` (or of course just: `SYSTEM DROP QUERY CACHE` which will clear the entire query cache). [#68477](https://github.com/ClickHouse/ClickHouse/pull/68477) ([Michał Tabaszewski](https://github.com/pinsvin00)).
+* Add storage encryption for named collections. [#68615](https://github.com/ClickHouse/ClickHouse/pull/68615) ([Pablo Marcos](https://github.com/pamarcos)).
+* Added `ripeMD160` function, which computes the RIPEMD-160 cryptographic hash of a string. Example: `SELECT hex(ripeMD160('The quick brown fox jumps over the lazy dog'))` returns `37F332F68DB77BD9D7EDD4969571AD671CF9DD3B`. [#68639](https://github.com/ClickHouse/ClickHouse/pull/68639) ([Dergousov Maxim](https://github.com/m7kss1)).
+* Add virtual column _headers for url table engine. Closes [#65026](https://github.com/ClickHouse/ClickHouse/issues/65026). [#68867](https://github.com/ClickHouse/ClickHouse/pull/68867) ([flynn](https://github.com/ucasfl)).
+* Adding `system.projections` table to track available projections. [#68901](https://github.com/ClickHouse/ClickHouse/pull/68901) ([Jordi Villar](https://github.com/jrdi)).
+* Add new function `arrayZipUnaligned` for spark compatiablity(arrays_zip), which allowed unaligned arrays based on original `arrayZip`. ``` sql SELECT arrayZipUnaligned([1], [1, 2, 3]). [#69030](https://github.com/ClickHouse/ClickHouse/pull/69030) ([李扬](https://github.com/taiyang-li)).
+* Adding `RealTimeMicroseconds` metric in the HTTP Header `X-ClickHouse-Summary`. This way we can know the CPU time of a request without having to go to check it in `system.query_log`. [#69032](https://github.com/ClickHouse/ClickHouse/pull/69032) ([Alejandro](https://github.com/alexon1234)).
+* Added cp/mv commands for keeper client which atomically copies/moves node. [#69034](https://github.com/ClickHouse/ClickHouse/pull/69034) ([Mikhail Artemenko](https://github.com/Michicosun)).
+* Adds argument `scale` (default: `true`) to function `arrayAUC` which allows to skip the normalization step (issue [#69609](https://github.com/ClickHouse/ClickHouse/issues/69609)). [#69717](https://github.com/ClickHouse/ClickHouse/pull/69717) ([gabrielmcg44](https://github.com/gabrielmcg44)).
+
+#### Performance Improvement
+* Improve the join performance by rerange the right table by keys while the table keys are dense in left or inner hash join. [#60341](https://github.com/ClickHouse/ClickHouse/pull/60341) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Improve all join perfromance by append `RowRefList` or `RowRef` to AddedColumns for lazy output, while buildOutput, we use `RowRefList`/`RowRef` for output, and remove `is_join_get` condition from `buildOutput` for loop. [#63677](https://github.com/ClickHouse/ClickHouse/pull/63677) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Load filesystem cache metadata asynchronously to boot process, in order to make restarts faster. [#65736](https://github.com/ClickHouse/ClickHouse/pull/65736) ([Daniel Pozo Escalona](https://github.com/danipozo)).
+* Functions `array` and `map` were optimized to process certain common cases much faster. [#67707](https://github.com/ClickHouse/ClickHouse/pull/67707) ([李扬](https://github.com/taiyang-li)).
+* Trivial optimize on orc string reading especially when column contains no NULLs. [#67794](https://github.com/ClickHouse/ClickHouse/pull/67794) ([李扬](https://github.com/taiyang-li)).
+* Improved overall performance of merges by reducing the overhead of scheduling steps of merges. [#68016](https://github.com/ClickHouse/ClickHouse/pull/68016) ([Anton Popov](https://github.com/CurtizJ)).
+* Speed up requests to S3 when a profile is not set, credentials are not set, and IMDS is not available (for example, when you are querying a public bucket on a machine outside of a cloud). This closes [#52771](https://github.com/ClickHouse/ClickHouse/issues/52771). [#68082](https://github.com/ClickHouse/ClickHouse/pull/68082) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* New algorithm to determine the unit of marks distribution between replicas by consistent hash. Different numbers of marks chosen for different read patterns to improve performance. [#68424](https://github.com/ClickHouse/ClickHouse/pull/68424) ([Nikita Taranov](https://github.com/nickitat)).
+* Try to devirtualize format reader in RowInputFormatWithNamesAndTypes, and see if it could bring some performance improvement. [#68437](https://github.com/ClickHouse/ClickHouse/pull/68437) ([李扬](https://github.com/taiyang-li)).
+* Add the parallel merge with key implementation to maximize the CPU utilization. [#68441](https://github.com/ClickHouse/ClickHouse/pull/68441) ([Jiebin Sun](https://github.com/jiebinn)).
+* Add settings `output_format_orc_dictionary_key_size_threshold` to allow user to enable dict encoding for string column in ORC output format. It helps reduce the output orc file size and improve reading performance significantly. [#68591](https://github.com/ClickHouse/ClickHouse/pull/68591) ([李扬](https://github.com/taiyang-li)).
+* Implemented reading of required files only during hive partitioning. [#68963](https://github.com/ClickHouse/ClickHouse/pull/68963) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Introduce new Keeper request RemoveRecursive which removes node with all it's subtree. [#69332](https://github.com/ClickHouse/ClickHouse/pull/69332) ([Mikhail Artemenko](https://github.com/Michicosun)).
+* Speedup insert performance with vector similarity index by adding data to vector index parallel. [#69493](https://github.com/ClickHouse/ClickHouse/pull/69493) ([flynn](https://github.com/ucasfl)).
+* Previously the algorithmic complexity of part deduplication logic in parallel replica announcement handling was O(n^2) which could take noticeable time for tables with many part (or partitions). This change makes the complexity O(n*log(n)). [#69596](https://github.com/ClickHouse/ClickHouse/pull/69596) ([Alexander Gololobov](https://github.com/davenger)).
+
+#### Improvement
+* Hardened parts of the codebase related to parsing of small entities. The following (minor) bugs were found and fixed: - if a `DeltaLake` table is partitioned by Bool, the partition value is always interpreted as false; - `ExternalDistributed` table was using only a single shard in the provided addresses; the value of `max_threads` setting and similar were printed as `'auto(N)'` instead of `auto(N)`. [#52503](https://github.com/ClickHouse/ClickHouse/pull/52503) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Refreshable materialized view improvements: append mode (`... REFRESH EVERY 1 MINUTE APPEND ...`) to add rows to existing table instead of overwriting the whole table, retries (disabled by default, configured in SETTINGS section of the query), `SYSTEM WAIT VIEW <name>` query that waits for the currently running refresh, some fixes. [#58934](https://github.com/ClickHouse/ClickHouse/pull/58934) ([Michael Kolupaev](https://github.com/al13n321)).
+* Use cgroup-specific metrics for CPU usage accounting instead of system-wide metrics. [#62003](https://github.com/ClickHouse/ClickHouse/pull/62003) ([Nikita Taranov](https://github.com/nickitat)).
+* IO scheduling for remote S3 disks is now done on the level of HTTP socket streams (instead of the whole S3 requests) to resolve `bandwidth_limit` throttling issues. [#65182](https://github.com/ClickHouse/ClickHouse/pull/65182) ([Sergei Trifonov](https://github.com/serxa)).
+* Allow a user to have multiple authentication methods instead of only one. Allow authentication methods to be reset to most recently added method. If you want to run instances on 24.8 and one on 24.9 for some time, it's better to set `max_authentication_methods_per_user = 1` for that period to avoid potential errors. [#65277](https://github.com/ClickHouse/ClickHouse/pull/65277) ([Arthur Passos](https://github.com/arthurpassos)).
+* Functions `upperUTF8` and `lowerUTF8` were previously only able to uppercase / lowercase Cyrillic characters. This limitation is now removed and characters in arbitrary languages are uppercased/lowercased. Example: `SELECT upperUTF8('Süden')` now returns `SÜDEN`. [#65761](https://github.com/ClickHouse/ClickHouse/pull/65761) ([李扬](https://github.com/taiyang-li)).
+* When lightweight delete happens on a table with projection(s), despite users have choices either throw an exception (by default) or drop the projection lightweight delete would happen, now the third option is to still have lightweight delete and then rebuild projection(s). [#66169](https://github.com/ClickHouse/ClickHouse/pull/66169) ([Shichao](https://github.com/jsc0218)).
+* Two options (`dns_allow_resolve_names_to_ipv4` and `dns_allow_resolve_names_to_ipv6`) have been added, to allow block connections ip family. [#66895](https://github.com/ClickHouse/ClickHouse/pull/66895) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Added `min_max`as a new type of (experimental) statistics. It supports estimating range predicates over numeric columns, e.g. `x < 100`. [#67013](https://github.com/ClickHouse/ClickHouse/pull/67013) ([JackyWoo](https://github.com/JackyWoo)).
+* Make C-z ignorance configurable (ignore_shell_suspend) in clickhouse-client. [#67134](https://github.com/ClickHouse/ClickHouse/pull/67134) ([Azat Khuzhin](https://github.com/azat)).
+* Improve castOrDefault from Variant/Dynamic columns so it works when inner types are not convertable at all. [#67150](https://github.com/ClickHouse/ClickHouse/pull/67150) ([Pavel Kruglov](https://github.com/Avogar)).
+* Improve unicode encoding in JSON output formats. Ensures that valid JSON is generated in the case of certain byte sequences in the result data. [#67938](https://github.com/ClickHouse/ClickHouse/pull/67938) ([mwoenker](https://github.com/mwoenker)).
+* Added profile events for merges and mutations for better introspection. [#68015](https://github.com/ClickHouse/ClickHouse/pull/68015) ([Anton Popov](https://github.com/CurtizJ)).
+* Odbc: get http_max_tries from server configuration. [#68128](https://github.com/ClickHouse/ClickHouse/pull/68128) ([Rodolphe Dugé de Bernonville](https://github.com/RodolpheDuge)).
+* Add wildcard support for user identification in x509 SubjectAltName extension. [#68236](https://github.com/ClickHouse/ClickHouse/pull/68236) ([Marco Vilas Boas](https://github.com/marco-vb)).
+* Improve schema inference of date times. Now DateTime64 used only when date time has fractional part, otherwise regular DateTime is used. Inference of Date/DateTime is more strict now, especially when `date_time_input_format='best_effort'` to avoid inferring date times from strings in corner cases. [#68382](https://github.com/ClickHouse/ClickHouse/pull/68382) ([Pavel Kruglov](https://github.com/Avogar)).
+* Delete old code of named collections from dictionaries and substitute it to the new, which allows to use DDL created named collections in dictionaries. Closes [#60936](https://github.com/ClickHouse/ClickHouse/issues/60936), closes [#36890](https://github.com/ClickHouse/ClickHouse/issues/36890). [#68412](https://github.com/ClickHouse/ClickHouse/pull/68412) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Use HTTP/1.1 instead of HTTP/1.0 (set by default) for external HTTP authentication. [#68456](https://github.com/ClickHouse/ClickHouse/pull/68456) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Functions `upperUTF8` and `lowerUTF8` were previously only able to uppercase / lowercase Cyrillic characters. This limitation is now removed and arbitrary characters are uppercased/lowercased. Example: `SELECT upperUTF8('Süden')` now returns `SÜDEN`. [#68523](https://github.com/ClickHouse/ClickHouse/pull/68523) ([Robert Schulze](https://github.com/rschu1ze)).
+* Added a new set of metrics for Thread Pool introspection, providing deeper insights into thread pool performance and behavior. [#68674](https://github.com/ClickHouse/ClickHouse/pull/68674) ([filimonov](https://github.com/filimonov)).
+* Support query parameters in async inserts with format `Values`. [#68741](https://github.com/ClickHouse/ClickHouse/pull/68741) ([Anton Popov](https://github.com/CurtizJ)).
+* Support Date32 on dateTrunc and toStartOfInterval. [#68874](https://github.com/ClickHouse/ClickHouse/pull/68874) ([LiuNeng](https://github.com/liuneng1994)).
+* Add a new setting: `output_format_always_quote_identifiers` to always quote identifiers if it is enabled. - Add a new setting: `output_format_identifier_quoting_style` to set the identifier quoting style. Applicable values: `'None'`, `'Backticks'` (default), `'DoubleQuotes'`, `'BackticksMySQL'`. [#68896](https://github.com/ClickHouse/ClickHouse/pull/68896) ([tuanpach](https://github.com/tuanpach)).
+* Add `plan_step_name` and `plan_step_description` columns to `system.processors_profile_log`. [#68954](https://github.com/ClickHouse/ClickHouse/pull/68954) ([Alexander Gololobov](https://github.com/davenger)).
+* Support for the Spanish language in the embedded dictionaries. [#69035](https://github.com/ClickHouse/ClickHouse/pull/69035) ([Vasily Okunev](https://github.com/VOkunev)).
+* Add CPU arch to short fault info. [#69037](https://github.com/ClickHouse/ClickHouse/pull/69037) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* CREATE TABLE AS copy PRIMARY KEY, ORDER BY, and similar clauses. Now it supports only for MergeTree family of table engines. [#69076](https://github.com/ClickHouse/ClickHouse/pull/69076) ([sakulali](https://github.com/sakulali)).
+* Replication of subset of columns is now available through MaterializedPostgreSQL. Closes [#33748](https://github.com/ClickHouse/ClickHouse/issues/33748). [#69092](https://github.com/ClickHouse/ClickHouse/pull/69092) ([Kruglov Kirill](https://github.com/1on)).
+* Add `std::string getHost(const std::string & key) const;` method to "base/poco/Util/include/Poco/Util/AbstractConfiguration.h" which does the same as `Poco::Util::AbstractConfiguration::getString` method but additionally does validity check to make sure value is a correct IP address or domain name. [#69130](https://github.com/ClickHouse/ClickHouse/pull/69130) ([Maxim](https://github.com/m4xxx1m)).
+* Do not block retries when establishing a new keeper connection. [#69148](https://github.com/ClickHouse/ClickHouse/pull/69148) ([Raúl Marín](https://github.com/Algunenano)).
+* Update DatabaseFactory so it would be possible for user defined database engines to have arguments, settings and table overrides (similar to StorageFactory). [#69201](https://github.com/ClickHouse/ClickHouse/pull/69201) ([Барыкин Никита Олегович](https://github.com/NikBarykin)).
+* Restore mode that replaces all external table engines and functions to Null (`restore_replace_external_engines_to_null`, `restore_replace_external_table_functions_to_null` settings) was failing if table had SETTINGS. Now it removes settings from table definition in this case and allows to restore such tables. [#69253](https://github.com/ClickHouse/ClickHouse/pull/69253) ([Ilya Yatsishin](https://github.com/qoega)).
+* Reduce memory usage of inserts to JSON by using adaptive write buffer size. A lot of files created by JSON column in wide part contains small amount of data and it doesn't make sense to allocate 1MB buffer for them. [#69272](https://github.com/ClickHouse/ClickHouse/pull/69272) ([Pavel Kruglov](https://github.com/Avogar)).
+* CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
+* Added user-level settings `min_free_disk_bytes_to_throw_insert` and `min_free_disk_ratio_to_throw_insert` to prevent insertions on disks that are almost full. [#69376](https://github.com/ClickHouse/ClickHouse/pull/69376) ([Marco Vilas Boas](https://github.com/marco-vb)).
+* Not retaining thread in concurrent hash join threadpool to avoid query excessively spawn threads. [#69406](https://github.com/ClickHouse/ClickHouse/pull/69406) ([Duc Canh Le](https://github.com/canhld94)).
+* Allow empty arguments for arrayZip/arrayZipUnaligned, as concat did in https://github.com/ClickHouse/ClickHouse/pull/65887. It is for spark compatiability in Gluten CH Backend. [#69576](https://github.com/ClickHouse/ClickHouse/pull/69576) ([李扬](https://github.com/taiyang-li)).
+* Support more advanced SSL options for Keeper's internal communication (e.g. private keys with passphrase). [#69582](https://github.com/ClickHouse/ClickHouse/pull/69582) ([Antonio Andelic](https://github.com/antonio2368)).
+* Sleep for 10ms before retrying to acquire a lock in `databasereplicatedddlworker::enqueuequeryimpl`. [#69588](https://github.com/ClickHouse/ClickHouse/pull/69588) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Index analysis can take noticeable time for big tables with many parts or partitions. This change should enable killing a heavy query at that stage. [#69606](https://github.com/ClickHouse/ClickHouse/pull/69606) ([Alexander Gololobov](https://github.com/davenger)).
+* Masking sensitive info in `gcs()` table function. [#69611](https://github.com/ClickHouse/ClickHouse/pull/69611) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Sync changes to `interpreterdropquery::executetodatabaseimpl` from the private fork. [#69670](https://github.com/ClickHouse/ClickHouse/pull/69670) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Backported in [#69905](https://github.com/ClickHouse/ClickHouse/issues/69905): Change the join to sort settings type to unsigned int. [#69886](https://github.com/ClickHouse/ClickHouse/pull/69886) ([kevinyhzou](https://github.com/KevinyhZou)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Rebuild projection for merges that reduce number of rows. [#62364](https://github.com/ClickHouse/ClickHouse/pull/62364) ([cangyin](https://github.com/cangyin)).
+* Fix attaching table when pg dbname contains "-" in MaterializedPostgreSQL. [#62730](https://github.com/ClickHouse/ClickHouse/pull/62730) ([takakawa](https://github.com/takakawa)).
+* Storage Join support for nullable columns in left table, close [#61247](https://github.com/ClickHouse/ClickHouse/issues/61247). [#66926](https://github.com/ClickHouse/ClickHouse/pull/66926) ([vdimir](https://github.com/vdimir)).
+* Incorrect query result with parallel replicas (distribute queries as well) when `IN` operator contains conversion to Decimal(). The bug was introduced with the new analyzer. [#67234](https://github.com/ClickHouse/ClickHouse/pull/67234) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix the problem that alter modfiy order by causes inconsistent metadata. [#67436](https://github.com/ClickHouse/ClickHouse/pull/67436) ([iceFireser](https://github.com/iceFireser)).
+* Fix the upper bound of the function `fromModifiedJulianDay`. It was supposed to be `9999-12-31` but was mistakenly set to `9999-01-01`. [#67583](https://github.com/ClickHouse/ClickHouse/pull/67583) ([PHO](https://github.com/depressed-pho)).
+* Fix when the index is not at the beginning of the tuple during `IN` query. [#67626](https://github.com/ClickHouse/ClickHouse/pull/67626) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fixed error on generated columns in MaterializedPostgreSQL when adnum ordering is broken [#63161](https://github.com/ClickHouse/ClickHouse/issues/63161). Fixed error on id column with nextval expression as default MaterializedPostgreSQL when there are generated columns in table. Fixed error on dropping publication with symbols except [a-z1-9-]. [#67664](https://github.com/ClickHouse/ClickHouse/pull/67664) ([Kruglov Kirill](https://github.com/1on)).
+* Fix expiration in `RoleCache`. [#67748](https://github.com/ClickHouse/ClickHouse/pull/67748) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix window view missing blocks due to slow flush to view. [#67983](https://github.com/ClickHouse/ClickHouse/pull/67983) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix MSAN issue caused by incorrect date format. [#68105](https://github.com/ClickHouse/ClickHouse/pull/68105) ([JackyWoo](https://github.com/JackyWoo)).
+* Fixed crash in Parquet filtering when data types in the file substantially differ from requested types (e.g. `... FROM file('a.parquet', Parquet, 'x String')`, but the file has `x Int64`). Without this fix, use `input_format_parquet_filter_push_down = 0` as a workaround. [#68131](https://github.com/ClickHouse/ClickHouse/pull/68131) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix crash in `lag`/`lead` which is introduced in [#67091](https://github.com/ClickHouse/ClickHouse/issues/67091). [#68262](https://github.com/ClickHouse/ClickHouse/pull/68262) ([lgbo](https://github.com/lgbo-ustc)).
+* Try fix postgres crash when query is cancelled. [#68288](https://github.com/ClickHouse/ClickHouse/pull/68288) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* After https://github.com/ClickHouse/ClickHouse/pull/61984 `schema_inference_make_columns_nullable=0` still can make columns `Nullable` in Parquet/Arrow formats. The change was backward incompatible and users noticed the changes in the behaviour. This PR makes `schema_inference_make_columns_nullable=0` to work as before (no Nullable columns will be inferred) and introduces new value `auto` for this setting that will make columns `Nullable` only if data has information about nullability. [#68298](https://github.com/ClickHouse/ClickHouse/pull/68298) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fixes [#50868](https://github.com/ClickHouse/ClickHouse/issues/50868). Small DateTime64 constant values returned by a nested subquery inside a distributed query were wrongly transformed to Nulls, thus causing errors and possible incorrect query results. [#68323](https://github.com/ClickHouse/ClickHouse/pull/68323) ([Shankar](https://github.com/shiyer7474)).
+* Fix missing sync replica mode in query `SYSTEM SYNC REPLICA`. [#68326](https://github.com/ClickHouse/ClickHouse/pull/68326) ([Duc Canh Le](https://github.com/canhld94)).
+* Fix bug in key condition. [#68354](https://github.com/ClickHouse/ClickHouse/pull/68354) ([Han Fei](https://github.com/hanfei1991)).
+* Fix crash on drop or rename a role that is used in LDAP external user directory. [#68355](https://github.com/ClickHouse/ClickHouse/pull/68355) ([Andrey Zvonov](https://github.com/zvonand)).
+* Fix Progress column value of system.view_refreshes greater than 1 [#68377](https://github.com/ClickHouse/ClickHouse/issues/68377). [#68378](https://github.com/ClickHouse/ClickHouse/pull/68378) ([megao](https://github.com/jetgm)).
+* Process regexp flags correctly. [#68389](https://github.com/ClickHouse/ClickHouse/pull/68389) ([Han Fei](https://github.com/hanfei1991)).
+* PostgreSQL-style cast operator (`::`) works correctly even for SQL-style hex and binary string literals (e.g., `SELECT x'414243'::String`). This closes [#68324](https://github.com/ClickHouse/ClickHouse/issues/68324). [#68482](https://github.com/ClickHouse/ClickHouse/pull/68482) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Minor patch for https://github.com/ClickHouse/ClickHouse/pull/68131. [#68494](https://github.com/ClickHouse/ClickHouse/pull/68494) ([Chang chen](https://github.com/baibaichen)).
+* Fix [#68239](https://github.com/ClickHouse/ClickHouse/issues/68239) SAMPLE n where n is an integer. [#68499](https://github.com/ClickHouse/ClickHouse/pull/68499) ([Denis Hananein](https://github.com/denis-hananein)).
+* Fix bug in mann-whitney-utest when the size of two districutions are not equal. [#68556](https://github.com/ClickHouse/ClickHouse/pull/68556) ([Han Fei](https://github.com/hanfei1991)).
+* After unexpected restart, fail to start replication of ReplicatedMergeTree due to abnormal handling of covered-by-broken part. [#68584](https://github.com/ClickHouse/ClickHouse/pull/68584) ([baolin](https://github.com/baolinhuang)).
+* Fix `LOGICAL_ERROR`s when functions `sipHash64Keyed`, `sipHash128Keyed`, or `sipHash128ReferenceKeyed` are applied to empty arrays or tuples. [#68630](https://github.com/ClickHouse/ClickHouse/pull/68630) ([Robert Schulze](https://github.com/rschu1ze)).
+* Full text index may filter out wrong columns when index multiple columns, it didn't reset row_id between different columns, the reproduce procedure is in tests/queries/0_stateless/03228_full_text_with_multi_col.sql. Without this. [#68644](https://github.com/ClickHouse/ClickHouse/pull/68644) ([siyuan](https://github.com/linkwk7)).
+* Fix invalid character '\t' and '\n' in replica_name when creating a Replicated table, which causes incorrect parsing of 'source replica' in LogEntry. Mentioned in issue [#68640](https://github.com/ClickHouse/ClickHouse/issues/68640). [#68645](https://github.com/ClickHouse/ClickHouse/pull/68645) ([Zhigao Hong](https://github.com/zghong)).
+* Added back virtual columns ` _table` and `_database` to distributed tables. They were available until version 24.3. [#68672](https://github.com/ClickHouse/ClickHouse/pull/68672) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix possible error `Size of permutation (0) is less than required (...)` during Variant column permutation. [#68681](https://github.com/ClickHouse/ClickHouse/pull/68681) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix possible error `DB::Exception: Block structure mismatch in joined block stream: different columns:` with new JSON column. [#68686](https://github.com/ClickHouse/ClickHouse/pull/68686) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix issue with materialized constant keys when hashing maps with arrays as keys in functions `sipHash(64/128)Keyed`. [#68731](https://github.com/ClickHouse/ClickHouse/pull/68731) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Make `ColumnsDescription::toString` format each column using the same `IAST::FormatState object`. This results in uniform columns metadata being written to disk and ZooKeeper. [#68733](https://github.com/ClickHouse/ClickHouse/pull/68733) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Fix merging of aggregated data for grouping sets. [#68744](https://github.com/ClickHouse/ClickHouse/pull/68744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix logical error, when we create a replicated merge tree, alter a column and then execute modify statistics. [#68820](https://github.com/ClickHouse/ClickHouse/pull/68820) ([Han Fei](https://github.com/hanfei1991)).
+* Fix resolving dynamic subcolumns from subqueries in analyzer. [#68824](https://github.com/ClickHouse/ClickHouse/pull/68824) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix complex types metadata parsing in DeltaLake. Closes [#68739](https://github.com/ClickHouse/ClickHouse/issues/68739). [#68836](https://github.com/ClickHouse/ClickHouse/pull/68836) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fixed asynchronous inserts in case when metadata of table is changed (by `ALTER ADD/MODIFY COLUMN` queries) after insert but before flush to the table. [#68837](https://github.com/ClickHouse/ClickHouse/pull/68837) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix unexpected exception when passing empty tuple in array. This fixes [#68618](https://github.com/ClickHouse/ClickHouse/issues/68618). [#68848](https://github.com/ClickHouse/ClickHouse/pull/68848) ([Amos Bird](https://github.com/amosbird)).
+* Fix parsing pure metadata mutations commands. [#68935](https://github.com/ClickHouse/ClickHouse/pull/68935) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fix possible wrong result during anyHeavy state merge. [#68950](https://github.com/ClickHouse/ClickHouse/pull/68950) ([Raúl Marín](https://github.com/Algunenano)).
+* Fixed writing to Materialized Views with enabled setting `optimize_functions_to_subcolumns`. [#68951](https://github.com/ClickHouse/ClickHouse/pull/68951) ([Anton Popov](https://github.com/CurtizJ)).
+* Don't use serializations cache in const Dynamic column methods. It could let to use-of-unitialized value or even race condition during aggregations. [#68953](https://github.com/ClickHouse/ClickHouse/pull/68953) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix parsing error when null should be inserted as default in some cases during JSON type parsing. [#68955](https://github.com/ClickHouse/ClickHouse/pull/68955) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* There were cases when path was concatenated incorrectly and had the `//` part in it, solving this problem using path normalization. [#69066](https://github.com/ClickHouse/ClickHouse/pull/69066) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix logical error when we have empty async insert. [#69080](https://github.com/ClickHouse/ClickHouse/pull/69080) ([Han Fei](https://github.com/hanfei1991)).
+* Fixed data race of progress indication in clickhouse-client during query canceling. [#69081](https://github.com/ClickHouse/ClickHouse/pull/69081) ([Sergei Trifonov](https://github.com/serxa)).
+* Fix a bug that the vector similarity index (currently experimental) was not utilized when used with cosine distance as distance function. [#69090](https://github.com/ClickHouse/ClickHouse/pull/69090) ([flynn](https://github.com/ucasfl)).
+* This change addresses an issue where attempting to create a Replicated database again after a server failure during the initial creation process could result in error. [#69102](https://github.com/ClickHouse/ClickHouse/pull/69102) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Don't infer Bool type from String in CSV when `input_format_csv_try_infer_numbers_from_strings = 1` because we don't allow reading bool values from strings. [#69109](https://github.com/ClickHouse/ClickHouse/pull/69109) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix explain ast insert queries parsing errors on client when `--multiquery` is enabled. [#69123](https://github.com/ClickHouse/ClickHouse/pull/69123) ([wxybear](https://github.com/wxybear)).
+* `UNION` clause in subqueries wasn't handled correctly in queries with parallel replicas and lead to LOGICAL_ERROR `Duplicate announcement received for replica`. [#69146](https://github.com/ClickHouse/ClickHouse/pull/69146) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix propogating structure argument in s3Cluster. Previously the `DEFAULT` expression of the column could be lost when sending the query to the replicas in s3Cluster. [#69147](https://github.com/ClickHouse/ClickHouse/pull/69147) ([Pavel Kruglov](https://github.com/Avogar)).
+* Respect format settings in Values format during conversion from expression to the destination type. [#69149](https://github.com/ClickHouse/ClickHouse/pull/69149) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix `clickhouse-client --queries-file` for readonly users (previously fails with `Cannot modify 'log_comment' setting in readonly mode`). [#69175](https://github.com/ClickHouse/ClickHouse/pull/69175) ([Azat Khuzhin](https://github.com/azat)).
+* Fix data race in clickhouse-client when it's piped to a process that terminated early. [#69186](https://github.com/ClickHouse/ClickHouse/pull/69186) ([vdimir](https://github.com/vdimir)).
+* Fix incorrect results of Fix uniq and GROUP BY for JSON/Dynamic types. [#69203](https://github.com/ClickHouse/ClickHouse/pull/69203) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix the INFILE format detection for asynchronous inserts. If the format is not explicitly defined in the FORMAT clause, it can be detected from the INFILE file extension. [#69237](https://github.com/ClickHouse/ClickHouse/pull/69237) ([Julia Kartseva](https://github.com/jkartseva)).
+* After [this issue](https://github.com/ClickHouse/ClickHouse/pull/59946#issuecomment-1943653197) there are quite a few table replicas in production such that their `metadata_version` node value is both equal to `0` and is different from the respective table's `metadata` node version. This leads to `alter` queries failing on such replicas. [#69274](https://github.com/ClickHouse/ClickHouse/pull/69274) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Backported in [#69986](https://github.com/ClickHouse/ClickHouse/issues/69986): Fix inf loop after `restore replica` in the replicated merge tree with zero copy. [#69293](https://github.com/ClickHouse/ClickHouse/pull/69293) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Mark Dynamic type as not safe primary key type to avoid issues with Fields. [#69311](https://github.com/ClickHouse/ClickHouse/pull/69311) ([Pavel Kruglov](https://github.com/Avogar)).
+* Improve restoring of access entities' dependencies. [#69346](https://github.com/ClickHouse/ClickHouse/pull/69346) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix undefined behavior when all connection attempts fail getting a connection for insertions. [#69390](https://github.com/ClickHouse/ClickHouse/pull/69390) ([Pablo Marcos](https://github.com/pamarcos)).
+* Close [#69135](https://github.com/ClickHouse/ClickHouse/issues/69135). If we try to reuse joined data for `cross` join, but this could not happen in ClickHouse at present. It's better to keep `have_compressed` in `reuseJoinedData`. [#69404](https://github.com/ClickHouse/ClickHouse/pull/69404) ([lgbo](https://github.com/lgbo-ustc)).
+* Make `materialize()` function return full column when parameter is a sparse column. [#69429](https://github.com/ClickHouse/ClickHouse/pull/69429) ([Alexander Gololobov](https://github.com/davenger)).
+* Fixed a `LOGICAL_ERROR` with function `sqidDecode` ([#69450](https://github.com/ClickHouse/ClickHouse/issues/69450)). [#69451](https://github.com/ClickHouse/ClickHouse/pull/69451) ([Robert Schulze](https://github.com/rschu1ze)).
+* Quick fix for s3queue problem on 24.6 or create query with database replicated. [#69454](https://github.com/ClickHouse/ClickHouse/pull/69454) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fixed case when memory consumption was too high because of the squashing in `INSERT INTO ... SELECT` or `CREATE TABLE AS SELECT` queries. [#69469](https://github.com/ClickHouse/ClickHouse/pull/69469) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Statements `SHOW COLUMNS` and `SHOW INDEX` now work properly if the table has dots in its name. [#69514](https://github.com/ClickHouse/ClickHouse/pull/69514) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Usage of the query cache for queries with an overflow mode != 'throw' is now disallowed. This prevents situations where potentially truncated and incorrect query results could be stored in the query cache. (issue [#67476](https://github.com/ClickHouse/ClickHouse/issues/67476)). [#69549](https://github.com/ClickHouse/ClickHouse/pull/69549) ([Robert Schulze](https://github.com/rschu1ze)).
+* Keep original order of conditions during move to prewhere. Previously the order could change and it could lead to failing queries when the order is important. [#69560](https://github.com/ClickHouse/ClickHouse/pull/69560) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix Keeper multi-request preprocessing after ZNOAUTH error. [#69627](https://github.com/ClickHouse/ClickHouse/pull/69627) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix METADATA_MISMATCH that might have happened due to TTL with a WHERE clause in DatabaseReplicated when creating a new replica. [#69736](https://github.com/ClickHouse/ClickHouse/pull/69736) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix `StorageS3(Azure)Queue` settings `tracked_file_ttl_sec`. We wrote it to keeper with key `tracked_file_ttl_sec`, but read as `tracked_files_ttl_sec`, which was a typo. [#69742](https://github.com/ClickHouse/ClickHouse/pull/69742) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Make getHyperrectangleForRowGroup not throw an exception when the data type in parquet file is not convertable into the requested data type. Solved the user's problem when the Parquet file had Decimal64 data type and the column data type was DateTime. [#69745](https://github.com/ClickHouse/ClickHouse/pull/69745) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Backported in [#70021](https://github.com/ClickHouse/ClickHouse/issues/70021): Fix analyzer default with old compatibility value. [#69895](https://github.com/ClickHouse/ClickHouse/pull/69895) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#69943](https://github.com/ClickHouse/ClickHouse/issues/69943): Don't check dependencies during CREATE OR REPLACE VIEW during DROP of old table. Previously CREATE OR REPLACE query failed when there are dependent tables of the recreated view. [#69907](https://github.com/ClickHouse/ClickHouse/pull/69907) ([Pavel Kruglov](https://github.com/Avogar)).
+* Backported in [#70003](https://github.com/ClickHouse/ClickHouse/issues/70003): Now SQL security will work with parameterized views correctly. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)).
+
+#### Build/Testing/Packaging Improvement
+* Allow to specify min and max for random settings in the test after tags in a form `setting=(min, max)`. For some tests really low value of a setting can lead to timeout (for example `index_granularity=1`) but we still want to randomize this setting in such tests. To avoid timeouts we can specify custom minimum value for this setting. [#67875](https://github.com/ClickHouse/ClickHouse/pull/67875) ([Pavel Kruglov](https://github.com/Avogar)).
+* Integration tests flaky check now runs each module(file) as a separate group. It allows to apply timeout per module. Previously slow tests could use all the time and other tests were not run in flaky check with successful result. [#68380](https://github.com/ClickHouse/ClickHouse/pull/68380) ([Ilya Yatsishin](https://github.com/qoega)).
+* Backported in [#69980](https://github.com/ClickHouse/ClickHouse/issues/69980): Makes dbms independent from clickhouse_functions. [#69914](https://github.com/ClickHouse/ClickHouse/pull/69914) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### NO CL CATEGORY
+
+* Backported in [#69995](https://github.com/ClickHouse/ClickHouse/issues/69995):. [#69989](https://github.com/ClickHouse/ClickHouse/pull/69989) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY:  'Revert "Revert "Fix AWS ECS""'. [#65362](https://github.com/ClickHouse/ClickHouse/pull/65362) ([Pavel Kruglov](https://github.com/Avogar)).
+* NO CL ENTRY:  'Revert "[RFC] Fix settings/current_database in system.processes for async BACKUP/RESTORE"'. [#68386](https://github.com/ClickHouse/ClickHouse/pull/68386) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* NO CL ENTRY:  'Revert "Improve compatibility of `upper/lowerUTF8` with Spark"'. [#68510](https://github.com/ClickHouse/ClickHouse/pull/68510) ([Robert Schulze](https://github.com/rschu1ze)).
+* NO CL ENTRY:  'Revert "Fix unexpected behavior with `FORMAT` and `SETTINGS` parsing"'. [#68608](https://github.com/ClickHouse/ClickHouse/pull/68608) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* NO CL ENTRY:  'Revert "Fix prewhere without columns and without adaptive index granularity (almost w/o anything)"'. [#68897](https://github.com/ClickHouse/ClickHouse/pull/68897) ([Alexander Gololobov](https://github.com/davenger)).
+* NO CL ENTRY:  'Revert "Speed up some Kafka tests with multiprocessing"'. [#69356](https://github.com/ClickHouse/ClickHouse/pull/69356) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* NO CL ENTRY:  'Revert "Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. V"'. [#69393](https://github.com/ClickHouse/ClickHouse/pull/69393) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* NO CL ENTRY:  'Revert "Add user-level settings min_free_diskspace_bytes_to_throw_insert and min_free_diskspace_ratio_to_throw_insert"'. [#69705](https://github.com/ClickHouse/ClickHouse/pull/69705) ([Raúl Marín](https://github.com/Algunenano)).
+* NO CL ENTRY:  'Revert "Support more oss endpoints"'. [#69779](https://github.com/ClickHouse/ClickHouse/pull/69779) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Followup for [#56996](https://github.com/ClickHouse/ClickHouse/issues/56996). [#58289](https://github.com/ClickHouse/ClickHouse/pull/58289) ([vdimir](https://github.com/vdimir)).
+* Add chunked wrapper to native protocol. [#63781](https://github.com/ClickHouse/ClickHouse/pull/63781) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Local plan for parallel replicas. [#64448](https://github.com/ClickHouse/ClickHouse/pull/64448) ([Igor Nikonov](https://github.com/devcrafter)).
+* Disallow creating refreshable MV on Linux < 3.15. [#64953](https://github.com/ClickHouse/ClickHouse/pull/64953) ([Michael Kolupaev](https://github.com/al13n321)).
+* Allow query profiler period to be longer than 4 seconds. [#65255](https://github.com/ClickHouse/ClickHouse/pull/65255) ([Michael Kolupaev](https://github.com/al13n321)).
+* Improved getting of alter conversions for queries. [#65832](https://github.com/ClickHouse/ClickHouse/pull/65832) ([Anton Popov](https://github.com/CurtizJ)).
+* Simplify some memory tracking parts. [#66648](https://github.com/ClickHouse/ClickHouse/pull/66648) ([Antonio Andelic](https://github.com/antonio2368)).
+* What if we tighten limits for functional tests?. [#66837](https://github.com/ClickHouse/ClickHouse/pull/66837) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix flaky `test_delayed_replica_failover`. [#67541](https://github.com/ClickHouse/ClickHouse/pull/67541) ([Sergei Trifonov](https://github.com/serxa)).
+* Fix(asan): access destroyed shared context from handleCrash(). [#67700](https://github.com/ClickHouse/ClickHouse/pull/67700) ([Igor Nikonov](https://github.com/devcrafter)).
+* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. II. [#67749](https://github.com/ClickHouse/ClickHouse/pull/67749) ([Robert Schulze](https://github.com/rschu1ze)).
+* Make `ColumnLowCardinality::getName()` consistent with other columns. [#67789](https://github.com/ClickHouse/ClickHouse/pull/67789) ([李扬](https://github.com/taiyang-li)).
+* Catch exception in destructor of LocalFileHolder. [#67891](https://github.com/ClickHouse/ClickHouse/pull/67891) ([Duc Canh Le](https://github.com/canhld94)).
+* Add keeper error description to the message. [#67935](https://github.com/ClickHouse/ClickHouse/pull/67935) ([Alexander Gololobov](https://github.com/davenger)).
+* Timeout handling for functional and integration tests, store artifacts and report if timed out - sets 2h default timeout for all jobs. [#67944](https://github.com/ClickHouse/ClickHouse/pull/67944) ([Max K.](https://github.com/maxknv)).
+* Add test cases to 03217_datetime64_constant_to_ast. [#67966](https://github.com/ClickHouse/ClickHouse/pull/67966) ([vdimir](https://github.com/vdimir)).
+* Remove some no-parallel tags from tests (Part 6). [#68048](https://github.com/ClickHouse/ClickHouse/pull/68048) ([Raúl Marín](https://github.com/Algunenano)).
+* Keeper improvements package. [#68108](https://github.com/ClickHouse/ClickHouse/pull/68108) ([Antonio Andelic](https://github.com/antonio2368)).
+* CI: Docker images clean from test scripts. [#68117](https://github.com/ClickHouse/ClickHouse/pull/68117) ([Max K.](https://github.com/maxknv)).
+* Output an operation error for ZK Multi request failed operation into log. [#68127](https://github.com/ClickHouse/ClickHouse/pull/68127) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Better test for Not-ready Set is passed in system.* tables. [#68139](https://github.com/ClickHouse/ClickHouse/pull/68139) ([Azat Khuzhin](https://github.com/azat)).
+* ci: add more logs in the functional tests reports. [#68153](https://github.com/ClickHouse/ClickHouse/pull/68153) ([Azat Khuzhin](https://github.com/azat)).
+* Fix fundamentally broken `test_throttling`. [#68180](https://github.com/ClickHouse/ClickHouse/pull/68180) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test `00600_replace_running_query`. [#68184](https://github.com/ClickHouse/ClickHouse/pull/68184) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix non-deterministic result order in `test_storage_mysql.test_mysql_distributed`. [#68188](https://github.com/ClickHouse/ClickHouse/pull/68188) ([Robert Schulze](https://github.com/rschu1ze)).
+* Try to unflaky `test_storage_s3_queue/test.py::test_shards_distributed`. [#68233](https://github.com/ClickHouse/ClickHouse/pull/68233) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Bump usearch to v2.13.2. [#68248](https://github.com/ClickHouse/ClickHouse/pull/68248) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix 'Refresh set entry already exists'. [#68249](https://github.com/ClickHouse/ClickHouse/pull/68249) ([Michael Kolupaev](https://github.com/al13n321)).
+* tests: make 01600_parts_states_metrics_long better. [#68265](https://github.com/ClickHouse/ClickHouse/pull/68265) ([Azat Khuzhin](https://github.com/azat)).
+* Avoid ignoring errors of execute_process() (set COMMAND_ERROR_IS_FATAL=ANY). [#68267](https://github.com/ClickHouse/ClickHouse/pull/68267) ([Azat Khuzhin](https://github.com/azat)).
+* Fix FullSortingJoinTest.AsofGreaterGeneratedTestData with empty data. [#68280](https://github.com/ClickHouse/ClickHouse/pull/68280) ([vdimir](https://github.com/vdimir)).
+* Fix min/max time columns in TimeSeries table. [#68282](https://github.com/ClickHouse/ClickHouse/pull/68282) ([Antonio Andelic](https://github.com/antonio2368)).
+* Update fuzzer dictionary. [#68284](https://github.com/ClickHouse/ClickHouse/pull/68284) ([Pablo Marcos](https://github.com/pamarcos)).
+* [Green CI] Test 00652_mergetree_mutations is flaky. [#68286](https://github.com/ClickHouse/ClickHouse/pull/68286) ([Daniil Ivanik](https://github.com/divanik)).
+* Fix test storage_join_direct_join. [#68287](https://github.com/ClickHouse/ClickHouse/pull/68287) ([Nikita Taranov](https://github.com/nickitat)).
+* Add execution status to PipelineExecutor. Avoid LOGICAL_ERROR if the pushing pipeline was canceled by timeout. Replace `Pipeline for PushingPipelineExecutor was finished before all data was inserted. (LOGICAL_ERROR)` to `QUERY_WAS_CANCELLED`. Closes [#63979](https://github.com/ClickHouse/ClickHouse/issues/63979). [#68291](https://github.com/ClickHouse/ClickHouse/pull/68291) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* CI: Create new release branch workflow updates. [#68294](https://github.com/ClickHouse/ClickHouse/pull/68294) ([Max K.](https://github.com/maxknv)).
+* Update version after release. [#68306](https://github.com/ClickHouse/ClickHouse/pull/68306) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Various vector similarity index related fixes. [#68308](https://github.com/ClickHouse/ClickHouse/pull/68308) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix: min marks to read overflow with parallel replicas. [#68309](https://github.com/ClickHouse/ClickHouse/pull/68309) ([Igor Nikonov](https://github.com/devcrafter)).
+* Apply libunwind changes needed for musl. [#68312](https://github.com/ClickHouse/ClickHouse/pull/68312) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update musl to 1.2.5 with unwind info. [#68313](https://github.com/ClickHouse/ClickHouse/pull/68313) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix data race on SampleKey. [#68321](https://github.com/ClickHouse/ClickHouse/pull/68321) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix part name in 00961_check_table. [#68325](https://github.com/ClickHouse/ClickHouse/pull/68325) ([vdimir](https://github.com/vdimir)).
+* Fix 02995_index_10 timeout. [#68329](https://github.com/ClickHouse/ClickHouse/pull/68329) ([vdimir](https://github.com/vdimir)).
+* CI: Fix for change log critical bug fix regex. [#68332](https://github.com/ClickHouse/ClickHouse/pull/68332) ([Max K.](https://github.com/maxknv)).
+* Optionally re-enable compilation with `-O0`. [#68352](https://github.com/ClickHouse/ClickHouse/pull/68352) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add debug info for `00180_no_seek_avoiding_when_reading_from_cache`. [#68353](https://github.com/ClickHouse/ClickHouse/pull/68353) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix data race in `DynamicResourceManager::updateConfiguration`. [#68356](https://github.com/ClickHouse/ClickHouse/pull/68356) ([Sergei Trifonov](https://github.com/serxa)).
+* Update version_date.tsv and changelog after v24.3.7.30-lts. [#68357](https://github.com/ClickHouse/ClickHouse/pull/68357) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* More improvements in integration tests. [#68358](https://github.com/ClickHouse/ClickHouse/pull/68358) ([Ilya Yatsishin](https://github.com/qoega)).
+* Rename: S3DiskNoKeyErrors -> DiskS3NoSuchKeyErrors. [#68361](https://github.com/ClickHouse/ClickHouse/pull/68361) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* CI: Minor fixes for changelog and release exceptions. [#68362](https://github.com/ClickHouse/ClickHouse/pull/68362) ([Max K.](https://github.com/maxknv)).
+* Check that setProcessListElement() is not called on global context. [#68364](https://github.com/ClickHouse/ClickHouse/pull/68364) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix off-by-one inline function info in stack traces. [#68365](https://github.com/ClickHouse/ClickHouse/pull/68365) ([Michael Kolupaev](https://github.com/al13n321)).
+* Check that merge entries are valid. [#68366](https://github.com/ClickHouse/ClickHouse/pull/68366) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* performance comparison test for output_format_parquet_write_page_index. [#68367](https://github.com/ClickHouse/ClickHouse/pull/68367) ([max-vostrikov](https://github.com/max-vostrikov)).
+* Add a test for [#57324](https://github.com/ClickHouse/ClickHouse/issues/57324). [#68376](https://github.com/ClickHouse/ClickHouse/pull/68376) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* CI: Auto release workflow. [#68402](https://github.com/ClickHouse/ClickHouse/pull/68402) ([Max K.](https://github.com/maxknv)).
+* Update delta lake test. [#68404](https://github.com/ClickHouse/ClickHouse/pull/68404) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add a test for [#59118](https://github.com/ClickHouse/ClickHouse/issues/59118). [#68406](https://github.com/ClickHouse/ClickHouse/pull/68406) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. IV. [#68407](https://github.com/ClickHouse/ClickHouse/pull/68407) ([Robert Schulze](https://github.com/rschu1ze)).
+* Try to fix test 03221_mutation_analyzer_skip_part. [#68409](https://github.com/ClickHouse/ClickHouse/pull/68409) ([Anton Popov](https://github.com/CurtizJ)).
+* Add `NotStarted` status to not-prepared processors. This is for better diagnostics in the case of PipelineStuck. [#68413](https://github.com/ClickHouse/ClickHouse/pull/68413) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Update log message. [#68426](https://github.com/ClickHouse/ClickHouse/pull/68426) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix 01119_session_log flakiness. [#68433](https://github.com/ClickHouse/ClickHouse/pull/68433) ([Michael Kolupaev](https://github.com/al13n321)).
+* Try to mitigate 02818_memory_profiler_sample_min_max_allocation_size flakiness. [#68434](https://github.com/ClickHouse/ClickHouse/pull/68434) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update analyzer_tech_debt.txt. [#68443](https://github.com/ClickHouse/ClickHouse/pull/68443) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove obsolete `-n` / `--multiquery` from tests. [#68447](https://github.com/ClickHouse/ClickHouse/pull/68447) ([Robert Schulze](https://github.com/rschu1ze)).
+* Check for invalid regexp in JSON SKIP REGEXP section. [#68451](https://github.com/ClickHouse/ClickHouse/pull/68451) ([Pavel Kruglov](https://github.com/Avogar)).
+* Better inference of date times 2. [#68452](https://github.com/ClickHouse/ClickHouse/pull/68452) ([Pavel Kruglov](https://github.com/Avogar)).
+* CI: Native build for package_aarch64. [#68457](https://github.com/ClickHouse/ClickHouse/pull/68457) ([Max K.](https://github.com/maxknv)).
+* Minor update in Dynamic/JSON serializations. [#68459](https://github.com/ClickHouse/ClickHouse/pull/68459) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix test `02122_join_group_by_timeout`. [#68462](https://github.com/ClickHouse/ClickHouse/pull/68462) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add check for `min_number_of_marks = 0` in parallel replicas requests. [#68476](https://github.com/ClickHouse/ClickHouse/pull/68476) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix `Broken pipe` error for `03149_numbers_max_block_size_zero.sh`. [#68478](https://github.com/ClickHouse/ClickHouse/pull/68478) ([Julia Kartseva](https://github.com/jkartseva)).
+* Use temporary tables for input and output in `clickhouse-local`. [#68483](https://github.com/ClickHouse/ClickHouse/pull/68483) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Pass-through RENAME and UUID-related operations in Overlay database to underlying databases. [#68486](https://github.com/ClickHouse/ClickHouse/pull/68486) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix output of clickhouse-test in case of tests timeouts. [#68487](https://github.com/ClickHouse/ClickHouse/pull/68487) ([Azat Khuzhin](https://github.com/azat)).
+* Part of [#68024](https://github.com/ClickHouse/ClickHouse/issues/68024). [#68488](https://github.com/ClickHouse/ClickHouse/pull/68488) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Miscellaneous changes in BaseDaemon. [#68489](https://github.com/ClickHouse/ClickHouse/pull/68489) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Miscellaneous changes from [#66999](https://github.com/ClickHouse/ClickHouse/issues/66999) (2). [#68490](https://github.com/ClickHouse/ClickHouse/pull/68490) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Miscellaneous. [#68504](https://github.com/ClickHouse/ClickHouse/pull/68504) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Miscellanous changes from [#66999](https://github.com/ClickHouse/ClickHouse/issues/66999). [#68507](https://github.com/ClickHouse/ClickHouse/pull/68507) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test `01017_uniqCombined_memory_usage`. [#68508](https://github.com/ClickHouse/ClickHouse/pull/68508) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix race condition in MergeTreeRestartingThread. [#68513](https://github.com/ClickHouse/ClickHouse/pull/68513) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test `01079_bad_alters_zookeeper_long`. [#68515](https://github.com/ClickHouse/ClickHouse/pull/68515) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad exception messages. [#68520](https://github.com/ClickHouse/ClickHouse/pull/68520) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* CI: Auto Releases in prod. [#68525](https://github.com/ClickHouse/ClickHouse/pull/68525) ([Max K.](https://github.com/maxknv)).
+* Fix build with `-DENABLE_LIBRARIES=0`. [#68527](https://github.com/ClickHouse/ClickHouse/pull/68527) ([Robert Schulze](https://github.com/rschu1ze)).
+* Removed the use of the context during function execution, as it was causing errors when the context was expired. [#68534](https://github.com/ClickHouse/ClickHouse/pull/68534) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* CI: Minor release workflow fix. [#68536](https://github.com/ClickHouse/ClickHouse/pull/68536) ([Max K.](https://github.com/maxknv)).
+* Fix after [#68291](https://github.com/ClickHouse/ClickHouse/issues/68291). [#68548](https://github.com/ClickHouse/ClickHouse/pull/68548) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Stateless tests: increase hung check timeout. [#68552](https://github.com/ClickHouse/ClickHouse/pull/68552) ([Nikita Fomichev](https://github.com/fm4v)).
+* CI: Tidy build timeout from 2h to 3h. [#68567](https://github.com/ClickHouse/ClickHouse/pull/68567) ([Max K.](https://github.com/maxknv)).
+* Reduce memory consumption in ghdata JSON tests. [#68571](https://github.com/ClickHouse/ClickHouse/pull/68571) ([Pavel Kruglov](https://github.com/Avogar)).
+* Move enabling experimental settings to a separate file. [#68577](https://github.com/ClickHouse/ClickHouse/pull/68577) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Minor logging fixes. [#68578](https://github.com/ClickHouse/ClickHouse/pull/68578) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix enumerating dynamic subcolumns. [#68582](https://github.com/ClickHouse/ClickHouse/pull/68582) ([Pavel Kruglov](https://github.com/Avogar)).
+* Update version_date.tsv and changelog after v23.8.16.16-lts. [#68593](https://github.com/ClickHouse/ClickHouse/pull/68593) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update 02995_index_7.sh. [#68594](https://github.com/ClickHouse/ClickHouse/pull/68594) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix style in Functions/printf.cpp. [#68595](https://github.com/ClickHouse/ClickHouse/pull/68595) ([vdimir](https://github.com/vdimir)).
+* Update version_date.tsv and changelog after v24.3.8.13-lts. [#68599](https://github.com/ClickHouse/ClickHouse/pull/68599) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Prioritizing of virtual columns in hive partitioning. [#68606](https://github.com/ClickHouse/ClickHouse/pull/68606) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* materialized_view_deduplication performance comparison test. [#68607](https://github.com/ClickHouse/ClickHouse/pull/68607) ([max-vostrikov](https://github.com/max-vostrikov)).
+* Update README.md. [#68610](https://github.com/ClickHouse/ClickHouse/pull/68610) ([Tyler Hannan](https://github.com/tylerhannan)).
+* patch: fix reference to sorting key in primary key docs. [#68612](https://github.com/ClickHouse/ClickHouse/pull/68612) ([Leon Kozlowski](https://github.com/leonkozlowski)).
+* Do not fuzz 02835 drop user during session. [#68620](https://github.com/ClickHouse/ClickHouse/pull/68620) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix 03221_s3_imds_decent_timeout. [#68627](https://github.com/ClickHouse/ClickHouse/pull/68627) ([vdimir](https://github.com/vdimir)).
+* Fix test `01079_bad_alters_zookeeper_long`. [#68629](https://github.com/ClickHouse/ClickHouse/pull/68629) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Disable min_bytes_to_use_direct_io in some tests with Dynamic/JSON subcolumns because it's broken. [#68632](https://github.com/ClickHouse/ClickHouse/pull/68632) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix false "Killed by signal (output files)" in stress_tests.lib. [#68638](https://github.com/ClickHouse/ClickHouse/pull/68638) ([Michael Kolupaev](https://github.com/al13n321)).
+* Remove wrong release version. [#68646](https://github.com/ClickHouse/ClickHouse/pull/68646) ([Max K.](https://github.com/maxknv)).
+* Increase connectTimeoutMs IMDS connection timeout to 50ms to avoid failures in CI. [#68653](https://github.com/ClickHouse/ClickHouse/pull/68653) ([Pavel Kruglov](https://github.com/Avogar)).
+* CI: Disable SQLLogic job. [#68654](https://github.com/ClickHouse/ClickHouse/pull/68654) ([Max K.](https://github.com/maxknv)).
+* Update version_date.tsv and changelog after v24.8.1.2684-lts. [#68664](https://github.com/ClickHouse/ClickHouse/pull/68664) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix flaky test test_distributed_replica_max_ignored_errors. [#68665](https://github.com/ClickHouse/ClickHouse/pull/68665) ([Pavel Kruglov](https://github.com/Avogar)).
+* Improve `02293_http_header_full_summary_without_progress` logging. [#68666](https://github.com/ClickHouse/ClickHouse/pull/68666) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Fix flaky check when all tests are skipped. [#68673](https://github.com/ClickHouse/ClickHouse/pull/68673) ([Pavel Kruglov](https://github.com/Avogar)).
+* Vector similarity index: make `bf16` the default quantization. [#68678](https://github.com/ClickHouse/ClickHouse/pull/68678) ([Robert Schulze](https://github.com/rschu1ze)).
+* 3h is not enough. [#68683](https://github.com/ClickHouse/ClickHouse/pull/68683) ([Max K.](https://github.com/maxknv)).
+* Un-flake 01278_random_string_utf8. [#68684](https://github.com/ClickHouse/ClickHouse/pull/68684) ([Robert Schulze](https://github.com/rschu1ze)).
+* CI: Integration tests timeout to 3h. [#68685](https://github.com/ClickHouse/ClickHouse/pull/68685) ([Max K.](https://github.com/maxknv)).
+* Fix Upgrade Check: move some settings to 24.9 section. [#68688](https://github.com/ClickHouse/ClickHouse/pull/68688) ([Pavel Kruglov](https://github.com/Avogar)).
+* Update README.md. [#68690](https://github.com/ClickHouse/ClickHouse/pull/68690) ([Tanya Bragin](https://github.com/tbragin)).
+* Use proper ErrorCodes, replace NETWORK_ERROR by HDFS_ERROR. [#68696](https://github.com/ClickHouse/ClickHouse/pull/68696) ([flynn](https://github.com/ucasfl)).
+* Fix for failing test in CI. [#68701](https://github.com/ClickHouse/ClickHouse/pull/68701) ([Pedro Ferreira](https://github.com/PedroTadim)).
+* CI: Stress test fix. [#68712](https://github.com/ClickHouse/ClickHouse/pull/68712) ([Max K.](https://github.com/maxknv)).
+* Speedup test 02150_index_hypothesis_race_long. [#68713](https://github.com/ClickHouse/ClickHouse/pull/68713) ([vdimir](https://github.com/vdimir)).
+* Turn off fault injection for insert in `01396_inactive_replica_cleanup_nodes_zookeeper`. [#68715](https://github.com/ClickHouse/ClickHouse/pull/68715) ([alesapin](https://github.com/alesapin)).
+* Update README.md - Meetups update. [#68723](https://github.com/ClickHouse/ClickHouse/pull/68723) ([Tanya Bragin](https://github.com/tbragin)).
+* Fix flaky check. [#68725](https://github.com/ClickHouse/ClickHouse/pull/68725) ([Pavel Kruglov](https://github.com/Avogar)).
+* fix shutdown for PeriodicLog. [#68728](https://github.com/ClickHouse/ClickHouse/pull/68728) ([Sema Checherinda](https://github.com/CheSema)).
+* Update version_date.tsv and changelog after v24.5.5.41-stable. [#68729](https://github.com/ClickHouse/ClickHouse/pull/68729) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Bump Replxx to support custom descriptors. [#68730](https://github.com/ClickHouse/ClickHouse/pull/68730) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Update version_date.tsv and changelog after v24.6.3.38-stable. [#68732](https://github.com/ClickHouse/ClickHouse/pull/68732) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fixes `00080_show_tables_and_system_tables` that was a bit flaky. [#68734](https://github.com/ClickHouse/ClickHouse/pull/68734) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Update version_date.tsv and changelog after v24.7.3.47-stable. [#68735](https://github.com/ClickHouse/ClickHouse/pull/68735) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix flaky test 00989_parallel_parts_loading. [#68737](https://github.com/ClickHouse/ClickHouse/pull/68737) ([alesapin](https://github.com/alesapin)).
+* Update README.md. [#68738](https://github.com/ClickHouse/ClickHouse/pull/68738) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Update version_date.tsv and changelog after v24.8.2.3-lts. [#68740](https://github.com/ClickHouse/ClickHouse/pull/68740) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelog after v24.5.5.41-stable. [#68745](https://github.com/ClickHouse/ClickHouse/pull/68745) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* To make patch release possible from every commit on release branch, package_debug build is required and must not be skipped. [#68750](https://github.com/ClickHouse/ClickHouse/pull/68750) ([Max K.](https://github.com/maxknv)).
+* Try to disable rerun check if job triggered manually. [#68751](https://github.com/ClickHouse/ClickHouse/pull/68751) ([Max K.](https://github.com/maxknv)).
+* Fix 2477 timeout. [#68752](https://github.com/ClickHouse/ClickHouse/pull/68752) ([Shichao](https://github.com/jsc0218)).
+* Update README.md. [#68764](https://github.com/ClickHouse/ClickHouse/pull/68764) ([Tanya Bragin](https://github.com/tbragin)).
+* Update version_date.tsv and changelog after v24.5.6.45-stable. [#68766](https://github.com/ClickHouse/ClickHouse/pull/68766) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelog after v24.6.4.42-stable. [#68767](https://github.com/ClickHouse/ClickHouse/pull/68767) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelog after v24.7.4.51-stable. [#68768](https://github.com/ClickHouse/ClickHouse/pull/68768) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Split test case and reduce number of random runs to reduce the time necessary to run the test. [#68772](https://github.com/ClickHouse/ClickHouse/pull/68772) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Check setting use_json_alias_for_old_object_type in runtime. [#68793](https://github.com/ClickHouse/ClickHouse/pull/68793) ([Pavel Kruglov](https://github.com/Avogar)).
+* Make dynamic structure selection more consistent. [#68802](https://github.com/ClickHouse/ClickHouse/pull/68802) ([Pavel Kruglov](https://github.com/Avogar)).
+* Fix zero copy bug with encrypted disk and UNFREEZE. [#68821](https://github.com/ClickHouse/ClickHouse/pull/68821) ([Joe Lynch](https://github.com/joelynch)).
+* Speed up functions `lowerUTF8`/`upperUTF8`. [#68822](https://github.com/ClickHouse/ClickHouse/pull/68822) ([李扬](https://github.com/taiyang-li)).
+* Convert integration test `test_incorrect_datetime_format` to stateless. [#68823](https://github.com/ClickHouse/ClickHouse/pull/68823) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix Function Typo. [#68835](https://github.com/ClickHouse/ClickHouse/pull/68835) ([Shichao](https://github.com/jsc0218)).
+* Fix test `03228_virtual_column_merge_dist`. [#68841](https://github.com/ClickHouse/ClickHouse/pull/68841) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix test `03221_mutation_analyzer_skip_part`. [#68842](https://github.com/ClickHouse/ClickHouse/pull/68842) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix flaky `02932_analyzer_rewrite_sum_column_and_constant `. [#68843](https://github.com/ClickHouse/ClickHouse/pull/68843) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Update README.md - Add Austin meetup. [#68845](https://github.com/ClickHouse/ClickHouse/pull/68845) ([Tanya Bragin](https://github.com/tbragin)).
+* Fix ssl handshake error processing. [#68866](https://github.com/ClickHouse/ClickHouse/pull/68866) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Speedup test 00653_verification_monotonic_data_load. [#68878](https://github.com/ClickHouse/ClickHouse/pull/68878) ([vdimir](https://github.com/vdimir)).
+* ci: add IPv6 support to `NetworkManager`. [#68900](https://github.com/ClickHouse/ClickHouse/pull/68900) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* ci: add checks for `iptables-nft` in integration test runner. [#68902](https://github.com/ClickHouse/ClickHouse/pull/68902) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Fix `02378_part_log_profile_events` flakiness. [#68917](https://github.com/ClickHouse/ClickHouse/pull/68917) ([Julia Kartseva](https://github.com/jkartseva)).
+* Make long_parquet* tests less long. [#68918](https://github.com/ClickHouse/ClickHouse/pull/68918) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix 01114_database_atomic flakiness. [#68930](https://github.com/ClickHouse/ClickHouse/pull/68930) ([Raúl Marín](https://github.com/Algunenano)).
+* CI: Fix job rerun check. [#68931](https://github.com/ClickHouse/ClickHouse/pull/68931) ([Max K.](https://github.com/maxknv)).
+* perf tests set cgroups_memory_usage_observer_wait_time to zero. [#68957](https://github.com/ClickHouse/ClickHouse/pull/68957) ([vdimir](https://github.com/vdimir)).
+* Revert un-quoting `auto` settings values. Fixes [#68748](https://github.com/ClickHouse/ClickHouse/issues/68748). [#68979](https://github.com/ClickHouse/ClickHouse/pull/68979) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Actually fix false "Killed by signal (output files)" in stress_tests.lib. [#68983](https://github.com/ClickHouse/ClickHouse/pull/68983) ([Michael Kolupaev](https://github.com/al13n321)).
+* no-tsan in 02735_parquet_encoder. [#68984](https://github.com/ClickHouse/ClickHouse/pull/68984) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update CHANGELOG.md. [#68994](https://github.com/ClickHouse/ClickHouse/pull/68994) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test_role & test_keeper_s3_snapshot integration tests. [#69013](https://github.com/ClickHouse/ClickHouse/pull/69013) ([Shankar](https://github.com/shiyer7474)).
+* fix false leak detect in libfiu. [#69018](https://github.com/ClickHouse/ClickHouse/pull/69018) ([Han Fei](https://github.com/hanfei1991)).
+* Update README.md. [#69041](https://github.com/ClickHouse/ClickHouse/pull/69041) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Update parametric-functions.md. [#69055](https://github.com/ClickHouse/ClickHouse/pull/69055) ([Samuel Warfield](https://github.com/Warfields)).
+* Disallow `ALTER TABLE ADD VECTOR SIMILARITY INDEX` if corresponding setting is not enabled. [#69065](https://github.com/ClickHouse/ClickHouse/pull/69065) ([flynn](https://github.com/ucasfl)).
+* Remove stale moving parts without zookeeper. [#69075](https://github.com/ClickHouse/ClickHouse/pull/69075) ([Kirill](https://github.com/kirillgarbar)).
+* Update README.md. [#69077](https://github.com/ClickHouse/ClickHouse/pull/69077) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Fix typo in ActionsDag. [#69086](https://github.com/ClickHouse/ClickHouse/pull/69086) ([LiuNeng](https://github.com/liuneng1994)).
+* Follow-up for https://github.com/ClickHouse/ClickHouse/pull/68332. [#69099](https://github.com/ClickHouse/ClickHouse/pull/69099) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fix subnet in docker_compose_net.yml. [#69121](https://github.com/ClickHouse/ClickHouse/pull/69121) ([Ilya Golshtein](https://github.com/ilejn)).
+* Fix: expression description in plan after lift up union optimization. [#69124](https://github.com/ClickHouse/ClickHouse/pull/69124) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix locking in UserDefinedSQLObjectsZooKeeperStorage.cpp. [#69132](https://github.com/ClickHouse/ClickHouse/pull/69132) ([Sergei Trifonov](https://github.com/serxa)).
+* Do not use docker pause for Kerberos KDC container in integration tests. [#69136](https://github.com/ClickHouse/ClickHouse/pull/69136) ([Ilya Golshtein](https://github.com/ilejn)).
+* Don't create Object type if use_json_alias_for_old_object_type=1 but allow_experimental_object_type=0. [#69150](https://github.com/ClickHouse/ClickHouse/pull/69150) ([Pavel Kruglov](https://github.com/Avogar)).
+* Use QueryPlan for merge. [#69167](https://github.com/ClickHouse/ClickHouse/pull/69167) ([Alexander Gololobov](https://github.com/davenger)).
+* Update version_date.tsv and changelog after v24.3.10.33-lts. [#69177](https://github.com/ClickHouse/ClickHouse/pull/69177) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* CI: Rerun check: do not check if manual rerun. [#69181](https://github.com/ClickHouse/ClickHouse/pull/69181) ([Max K.](https://github.com/maxknv)).
+* Update version_date.tsv and changelog after v24.5.7.31-stable. [#69182](https://github.com/ClickHouse/ClickHouse/pull/69182) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelog after v24.6.5.30-stable. [#69184](https://github.com/ClickHouse/ClickHouse/pull/69184) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix flaky 02915_move_partition_inactive_replica. [#69190](https://github.com/ClickHouse/ClickHouse/pull/69190) ([alesapin](https://github.com/alesapin)).
+* Update version_date.tsv and changelog after v24.8.3.59-lts. [#69191](https://github.com/ClickHouse/ClickHouse/pull/69191) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Disable memory test with sanitizer. [#69193](https://github.com/ClickHouse/ClickHouse/pull/69193) ([alesapin](https://github.com/alesapin)).
+* Disable perf-like test with sanitizers. [#69194](https://github.com/ClickHouse/ClickHouse/pull/69194) ([alesapin](https://github.com/alesapin)).
+* Fix jepsen for aarch64. [#69199](https://github.com/ClickHouse/ClickHouse/pull/69199) ([Antonio Andelic](https://github.com/antonio2368)).
+* Test for parallel replicas, reverse in order reading mode. [#69202](https://github.com/ClickHouse/ClickHouse/pull/69202) ([Igor Nikonov](https://github.com/devcrafter)).
+* Collect sanitizer report from client to `client_log`. [#69204](https://github.com/ClickHouse/ClickHouse/pull/69204) ([vdimir](https://github.com/vdimir)).
+* Fix flaky 00157_cache_dictionary. [#69205](https://github.com/ClickHouse/ClickHouse/pull/69205) ([Igor Nikonov](https://github.com/devcrafter)).
+* Hide Settings implementation. [#69213](https://github.com/ClickHouse/ClickHouse/pull/69213) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix progress bar when reading from Memory tables. [#69234](https://github.com/ClickHouse/ClickHouse/pull/69234) ([Michael Kolupaev](https://github.com/al13n321)).
+* CMake: Update ICU build description. [#69240](https://github.com/ClickHouse/ClickHouse/pull/69240) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add testcase for ANN index usage with subquery. [#69241](https://github.com/ClickHouse/ClickHouse/pull/69241) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backports should read tags from its repo only. [#69252](https://github.com/ClickHouse/ClickHouse/pull/69252) ([Raúl Marín](https://github.com/Algunenano)).
+* 01114_database_atomic: Increase time frames to reduce flakiness. [#69255](https://github.com/ClickHouse/ClickHouse/pull/69255) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix dropping of file cache in CHECK query in case of enabled transactions. [#69256](https://github.com/ClickHouse/ClickHouse/pull/69256) ([Anton Popov](https://github.com/CurtizJ)).
+* CI: Merge stress and func runners type. [#69257](https://github.com/ClickHouse/ClickHouse/pull/69257) ([Max K.](https://github.com/maxknv)).
+* Make infrastructure related scripts private. [#69260](https://github.com/ClickHouse/ClickHouse/pull/69260) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix: Not-ready Set with parallel replicas. [#69264](https://github.com/ClickHouse/ClickHouse/pull/69264) ([Igor Nikonov](https://github.com/devcrafter)).
+* Revert "CREATE TABLE AS copy PRIMARY KEY, ORDER BY, and similar clauses.". [#69268](https://github.com/ClickHouse/ClickHouse/pull/69268) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Use CANNOT_DECOMPRESS error code for the compressed external data in CompressedReadBufferBase for checksum validation and decompression. [#69269](https://github.com/ClickHouse/ClickHouse/pull/69269) ([Alexey Katsman](https://github.com/alexkats)).
+* Fix the error in loading client suggestions and select from `system.function` when `allow_experimental_nlp_functions` setting is enabled and no lemmatizers are specified in the configuration. [#69271](https://github.com/ClickHouse/ClickHouse/pull/69271) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Improve logical error trace for TryResult. [#69297](https://github.com/ClickHouse/ClickHouse/pull/69297) ([Pablo Marcos](https://github.com/pamarcos)).
+* Just a small refactoring. [#69298](https://github.com/ClickHouse/ClickHouse/pull/69298) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Don't run 01287_max_execution_speed with slow builds. [#69299](https://github.com/ClickHouse/ClickHouse/pull/69299) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Add checks against segfault in DeltaLakeMetadata. [#69305](https://github.com/ClickHouse/ClickHouse/pull/69305) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Update StackTrace.cpp. [#69306](https://github.com/ClickHouse/ClickHouse/pull/69306) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Bump libarchive from v3.7.0 to v3.7.1. [#69318](https://github.com/ClickHouse/ClickHouse/pull/69318) ([Robert Schulze](https://github.com/rschu1ze)).
+* Bump curl to v8.9.1. [#69321](https://github.com/ClickHouse/ClickHouse/pull/69321) ([Robert Schulze](https://github.com/rschu1ze)).
+* These tests take long time to execute with slow builds (either 700 or 400 seconds). [#69322](https://github.com/ClickHouse/ClickHouse/pull/69322) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Bump grpc to v1.59.5. [#69323](https://github.com/ClickHouse/ClickHouse/pull/69323) ([Robert Schulze](https://github.com/rschu1ze)).
+* Speed up some Kafka tests with multiprocessing. [#69324](https://github.com/ClickHouse/ClickHouse/pull/69324) ([Antonio Andelic](https://github.com/antonio2368)).
+* Minor follow-up for [#66933](https://github.com/ClickHouse/ClickHouse/issues/66933). [#69326](https://github.com/ClickHouse/ClickHouse/pull/69326) ([Robert Schulze](https://github.com/rschu1ze)).
+* Randomize integration tests settings. [#69328](https://github.com/ClickHouse/ClickHouse/pull/69328) ([vdimir](https://github.com/vdimir)).
+* Update version_date.tsv and changelog after v24.3.11.7-lts. [#69330](https://github.com/ClickHouse/ClickHouse/pull/69330) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelog after v24.5.8.10-stable. [#69333](https://github.com/ClickHouse/ClickHouse/pull/69333) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Add CITATION.cff. [#69334](https://github.com/ClickHouse/ClickHouse/pull/69334) ([Robert Schulze](https://github.com/rschu1ze)).
+* Update version_date.tsv and changelog after v24.6.6.6-stable. [#69335](https://github.com/ClickHouse/ClickHouse/pull/69335) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix a leak of a few bytes in function `lower/upperUTF8`. [#69336](https://github.com/ClickHouse/ClickHouse/pull/69336) ([Robert Schulze](https://github.com/rschu1ze)).
+* Update version_date.tsv and changelog after v24.7.6.8-stable. [#69337](https://github.com/ClickHouse/ClickHouse/pull/69337) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelog after v24.8.4.13-lts. [#69340](https://github.com/ClickHouse/ClickHouse/pull/69340) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Bump OpenSSL to v3.2.3. [#69341](https://github.com/ClickHouse/ClickHouse/pull/69341) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. V. [#69344](https://github.com/ClickHouse/ClickHouse/pull/69344) ([Robert Schulze](https://github.com/rschu1ze)).
+* Bump libuv to v1.48.0. [#69345](https://github.com/ClickHouse/ClickHouse/pull/69345) ([Robert Schulze](https://github.com/rschu1ze)).
+* Update README.md - Meetups. [#69359](https://github.com/ClickHouse/ClickHouse/pull/69359) ([Tanya Bragin](https://github.com/tbragin)).
+* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. VI. [#69361](https://github.com/ClickHouse/ClickHouse/pull/69361) ([Robert Schulze](https://github.com/rschu1ze)).
+* Bump libarchive to v3.7.4. [#69367](https://github.com/ClickHouse/ClickHouse/pull/69367) ([Robert Schulze](https://github.com/rschu1ze)).
+* Rename `count_min` statistics to `countmin`. [#69377](https://github.com/ClickHouse/ClickHouse/pull/69377) ([JackyWoo](https://github.com/JackyWoo)).
+* Refactor temporary file usage in MergeTask with QueryPlan. [#69383](https://github.com/ClickHouse/ClickHouse/pull/69383) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix type mismatch. [#69385](https://github.com/ClickHouse/ClickHouse/pull/69385) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix 24.8 setting compatibility `rows_before_aggregation`. [#69394](https://github.com/ClickHouse/ClickHouse/pull/69394) ([Nikita Fomichev](https://github.com/fm4v)).
+* Support more oss endpoints. [#69400](https://github.com/ClickHouse/ClickHouse/pull/69400) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix 01603_read_with_backoff_bug. [#69413](https://github.com/ClickHouse/ClickHouse/pull/69413) ([Nikita Taranov](https://github.com/nickitat)).
+* See if it would address strange issue with `Resource temporarily unavailable`. [#69416](https://github.com/ClickHouse/ClickHouse/pull/69416) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Added a server setting `database_replicated_allow_detach_permanently` that disallows `DETACH TABLE PERMANENTLY` queries in Replicated databases (allowed by default). [#69422](https://github.com/ClickHouse/ClickHouse/pull/69422) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* See https://s3.amazonaws.com/clickhouse-test-reports/69093/d9b1fb7a7cb0813dd12d8f73423662e02a458056/fast_test.html. [#69430](https://github.com/ClickHouse/ClickHouse/pull/69430) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix test_disks_app_func. [#69449](https://github.com/ClickHouse/ClickHouse/pull/69449) ([Nikita Taranov](https://github.com/nickitat)).
+* Fixed consistency for `groupConcat` with `arrayStringConcat` in case of `FixedString` as argument. [#69455](https://github.com/ClickHouse/ClickHouse/pull/69455) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* The fix from [#69416](https://github.com/ClickHouse/ClickHouse/issues/69416) was incomplete. Reduce the size of printed statement to 4k max. [#69483](https://github.com/ClickHouse/ClickHouse/pull/69483) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* CMake: Add comment about ICU data files. [#69511](https://github.com/ClickHouse/ClickHouse/pull/69511) ([Robert Schulze](https://github.com/rschu1ze)).
+* S3Queue: small refactoring. [#69513](https://github.com/ClickHouse/ClickHouse/pull/69513) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backport MergeTask changes from private to minimize merge conflicts. [#69525](https://github.com/ClickHouse/ClickHouse/pull/69525) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix getting the latest synced commit for long PRs. [#69538](https://github.com/ClickHouse/ClickHouse/pull/69538) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* When trying to ATTACH ReplicatedMergeTree tables that were previously PERMANENTLY DETACHED, we will fail due to explicitly specified parameters. Reference https://github.com/ClickHouse/ClickHouse/pull/66104. [#69539](https://github.com/ClickHouse/ClickHouse/pull/69539) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix ci upgrade check failure for `join_to_sort` settings in pr https://github.com/ClickHouse/ClickHouse/pull/60341. [#69554](https://github.com/ClickHouse/ClickHouse/pull/69554) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Add function `kill_ci_runner`. Kill runner when pre-pull failed. [#69557](https://github.com/ClickHouse/ClickHouse/pull/69557) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Replace libpq code dump by postgresql fork + bump to v14.3. [#69564](https://github.com/ClickHouse/ClickHouse/pull/69564) ([Robert Schulze](https://github.com/rschu1ze)).
+* Mask azure connection string sensitive info. [#69570](https://github.com/ClickHouse/ClickHouse/pull/69570) ([Alexey Katsman](https://github.com/alexkats)).
+* Don't leave an empty znode when replicated table is dropped. [#69574](https://github.com/ClickHouse/ClickHouse/pull/69574) ([Michael Kolupaev](https://github.com/al13n321)).
+* Enable removerecursive in ci. [#69578](https://github.com/ClickHouse/ClickHouse/pull/69578) ([Mikhail Artemenko](https://github.com/Michicosun)).
+* Bump libpqxx to v7.7.5. [#69580](https://github.com/ClickHouse/ClickHouse/pull/69580) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove superfluous `--multiquery/-n`, pt. V. [#69581](https://github.com/ClickHouse/ClickHouse/pull/69581) ([Robert Schulze](https://github.com/rschu1ze)).
+* A better fix for [#67476](https://github.com/ClickHouse/ClickHouse/issues/67476). [#69583](https://github.com/ClickHouse/ClickHouse/pull/69583) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add more contexts to the debug action and use it broadly. [#69599](https://github.com/ClickHouse/ClickHouse/pull/69599) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Try to fix data race in `WriteBufferFromHTTPServerResponse`. [#69601](https://github.com/ClickHouse/ClickHouse/pull/69601) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Remove explicit announce from local replica in ReadFromMergeTree. [#69602](https://github.com/ClickHouse/ClickHouse/pull/69602) ([Nikita Taranov](https://github.com/nickitat)).
+* Unification of FunctionSecretArgumentsFinder. [#69615](https://github.com/ClickHouse/ClickHouse/pull/69615) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Reorder some tests (follow-up to [#69514](https://github.com/ClickHouse/ClickHouse/issues/69514)). [#69626](https://github.com/ClickHouse/ClickHouse/pull/69626) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix invalid `clickhouse-format` invocation in tests. [#69642](https://github.com/ClickHouse/ClickHouse/pull/69642) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Try fix `02447_drop_database_replica`. [#69655](https://github.com/ClickHouse/ClickHouse/pull/69655) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* S3Queue small refactoring. [#69672](https://github.com/ClickHouse/ClickHouse/pull/69672) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Update assert. [#69673](https://github.com/ClickHouse/ClickHouse/pull/69673) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Bump libpq from v14.3 to v15.8. [#69674](https://github.com/ClickHouse/ClickHouse/pull/69674) ([Robert Schulze](https://github.com/rschu1ze)).
+* Try fix asserts failure in `HashJoin`. [#69682](https://github.com/ClickHouse/ClickHouse/pull/69682) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Prohibit `ALTER TABLE ... ADD INDEX ... TYPE` inverted if setting = 0. [#69684](https://github.com/ClickHouse/ClickHouse/pull/69684) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fixed 2 bugs in Remove Recursive implementation:. [#69690](https://github.com/ClickHouse/ClickHouse/pull/69690) ([Mikhail Artemenko](https://github.com/Michicosun)).
+* New profile events for parallel replicas. [#69706](https://github.com/ClickHouse/ClickHouse/pull/69706) ([Nikita Taranov](https://github.com/nickitat)).
+* Update README.md - Meetups. [#69714](https://github.com/ClickHouse/ClickHouse/pull/69714) ([Tanya Bragin](https://github.com/tbragin)).
+* added some edge cases for printf tests. [#69737](https://github.com/ClickHouse/ClickHouse/pull/69737) ([max-vostrikov](https://github.com/max-vostrikov)).
+* Bump libpq to v16.4. [#69741](https://github.com/ClickHouse/ClickHouse/pull/69741) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix parallel replicas protocol after [#68424](https://github.com/ClickHouse/ClickHouse/issues/68424). [#69744](https://github.com/ClickHouse/ClickHouse/pull/69744) ([Nikita Taranov](https://github.com/nickitat)).
+* CI: Include PostgreSQL in sparse checkout script. [#69746](https://github.com/ClickHouse/ClickHouse/pull/69746) ([Robert Schulze](https://github.com/rschu1ze)).
+* Get rid of the import cycle when env_helper can't be imported from `ci_*.py` files. Now, `env_helper` won't import download helper or basically anything from the `ci` module. The report is the best place for the GH job info since we use it mostly there. [#69750](https://github.com/ClickHouse/ClickHouse/pull/69750) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Save CREATE QUERY with the KeeperMap engine with evaluated parameters. It is important because, for example, we can use the `currentDatabase()` function in the KeeperMap `zk_root_path` argument, and we want it to point to the same path even if we move this table to another database. [#69751](https://github.com/ClickHouse/ClickHouse/pull/69751) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix native macOS build. [#69767](https://github.com/ClickHouse/ClickHouse/pull/69767) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#69974](https://github.com/ClickHouse/ClickHouse/issues/69974): S3Queue: support having deprecated settings to not fail server startup. [#69769](https://github.com/ClickHouse/ClickHouse/pull/69769) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Limit fast tests to 30 seconds. [#69781](https://github.com/ClickHouse/ClickHouse/pull/69781) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#69824](https://github.com/ClickHouse/ClickHouse/issues/69824): Allow cyrillic characters in generated contributor names. [#69820](https://github.com/ClickHouse/ClickHouse/pull/69820) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#69912](https://github.com/ClickHouse/ClickHouse/issues/69912): Add Proj Obsolete Setting. [#69883](https://github.com/ClickHouse/ClickHouse/pull/69883) ([Shichao](https://github.com/jsc0218)).
+* Backported in [#69899](https://github.com/ClickHouse/ClickHouse/issues/69899): Revert "Merge pull request [#69032](https://github.com/ClickHouse/ClickHouse/issues/69032) from alexon1234/include_real_time_execution_in_http_header". [#69885](https://github.com/ClickHouse/ClickHouse/pull/69885) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#69931](https://github.com/ClickHouse/ClickHouse/issues/69931): RIPE is an acronym and thus should be capital. RIPE stands for **R**ACE **I**ntegrity **P**rimitives **E**valuation and RACE stands for **R**esearch and Development in **A**dvanced **C**ommunications **T**echnologies in **E**urope. [#69901](https://github.com/ClickHouse/ClickHouse/pull/69901) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Backported in [#70034](https://github.com/ClickHouse/ClickHouse/issues/70034): Revert "Add RIPEMD160 function". [#70005](https://github.com/ClickHouse/ClickHouse/pull/70005) ([Robert Schulze](https://github.com/rschu1ze)).
+

docs/en/development/developer-instruction.md

@@ -232,7 +232,7 @@ Below you can find some quick links which may be useful when writing code for Cl
 - [The code style guide](https://clickhouse.com/docs/en/development/style/).
 - [Adding third-party libraries](https://clickhouse.com/docs/en/development/contrib/#adding-third-party-libraries)
 - [Writing tests](https://clickhouse.com/docs/en/development/tests/)
-- [List of open issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3Ahacktoberfest)
+- [List of open issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3A%22easy+task%22)
 
 ## Writing Documentation {#writing-documentation}
 

docs/en/getting-started/example-datasets/stackoverflow.md

@@ -7,7 +7,7 @@ description: Analyzing Stack Overflow data with ClickHouse
 
 # Analyzing Stack Overflow data with ClickHouse
 
-This dataset contains every `Post`, `User`, `Vote`, `Comment`, `Badge, `PostHistory`, and `PostLink` that has occurred on Stack Overflow.
+This dataset contains every `Posts`, `Users`, `Votes`, `Comments`, `Badges`, `PostHistory`, and `PostLinks` that has occurred on Stack Overflow.
 
 Users can either download pre-prepared Parquet versions of the data, containing every post up to April 2024, or download the latest data in XML format and load this. Stack Overflow provide updates to this data periodically - historically every 3 months.
 
@@ -159,7 +159,7 @@ INSERT INTO stackoverflow.badges SELECT * FROM s3('https://datasets-documentatio
 0 rows in set. Elapsed: 6.635 sec. Processed 51.29 million rows, 797.05 MB (7.73 million rows/s., 120.13 MB/s.)
 ```
 
-### `PostLinks`
+### PostLinks
 
 ```sql
 CREATE TABLE stackoverflow.postlinks
@@ -178,7 +178,7 @@ INSERT INTO stackoverflow.postlinks SELECT * FROM s3('https://datasets-documenta
 0 rows in set. Elapsed: 1.534 sec. Processed 6.55 million rows, 129.70 MB (4.27 million rows/s., 84.57 MB/s.)
 ```
 
-### `PostHistory`
+### PostHistory
 
 ```sql
 CREATE TABLE stackoverflow.posthistory

docs/en/operations/settings/merge-tree-settings.md

@@ -1064,4 +1064,32 @@ Possible values:
 
 - throw, drop, rebuild
 
-Default value: throw
+Default value: throw
+
+## min_free_disk_bytes_to_perform_insert
+
+The minimum number of bytes that should be free in disk space in order to insert data. If the number of available free bytes is less than `min_free_disk_bytes_to_throw_insert` then an exception is thrown and the insert is not executed. Note that this setting:
+- takes into account the `keep_free_space_bytes` setting.
+- does not take into account the amount of data that will be written by the `INSERT` operation.
+- is only checked if a positive (non-zero) number of bytes is specified
+
+Possible values:
+
+- Any positive integer.
+
+Default value: 0 bytes.
+
+## min_free_disk_ratio_to_perform_insert 
+
+The minimum free to total disk space ratio to perform an `INSERT`. Must be a floating point value between 0 and 1. Note that this setting:
+- takes into account the `keep_free_space_bytes` setting.
+- does not take into account the amount of data that will be written by the `INSERT` operation.
+- is only checked if a positive (non-zero) ratio is specified
+
+Possible values:
+
+- Float, 0.0 - 1.0
+
+Default value: 0.0
+
+Note that if both `min_free_disk_ratio_to_perform_insert` and `min_free_disk_bytes_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory.

docs/en/operations/settings/settings.md

@@ -2045,7 +2045,7 @@ Possible values:
 - 0 - Disabled.
 - 1 - Enabled.
 
-Default value: `0`.
+Default value: `1`.
 
 ### async_insert_busy_timeout_min_ms {#async-insert-busy-timeout-min-ms}
 
@@ -5683,6 +5683,30 @@ Enable `IF NOT EXISTS` for `CREATE` statement by default. If either this setting
 
 Default value: `false`.
 
+## enable_secure_identifiers
+
+If enabled, only allow secure identifiers which contain only underscore and alphanumeric characters
+
+Default value: `false`.
+
+## show_create_query_identifier_quoting_rule
+
+Define identifier quoting behavior of the show create query result:
+- `when_necessary`: When the identifiers is one of `{"distinct", "all", "table"}`, or it can cause ambiguity: column names, dictionary attribute names.
+- `always`: Always quote identifiers.
+- `user_display`: When the identifiers is a keyword.
+
+Default value: `when_necessary`.
+
+## show_create_query_identifier_quoting_style
+
+Define identifier quoting style of the show create query result:
+- `Backticks`: \`clickhouse\` style.
+- `DoubleQuotes`: "postgres" style
+- `BackticksMySQL`: \`mysql\` style, most same as `Backticks`, but it uses '``' to escape '`'
+
+Default value: `Backticks`.
+
 ## mongodb_throw_on_unsupported_query
 
 If enabled, MongoDB tables will return an error when a MongoDB query can't be built.

docs/en/sql-reference/aggregate-functions/reference/avgweighted.md

@@ -19,9 +19,7 @@ avgWeighted(x, weight)
 - `weight` — Weights of the values.
 
 `x` and `weight` must both be
-[Integer](../../../sql-reference/data-types/int-uint.md),
-[floating-point](../../../sql-reference/data-types/float.md), or
-[Decimal](../../../sql-reference/data-types/decimal.md),
+[Integer](../../../sql-reference/data-types/int-uint.md) or [floating-point](../../../sql-reference/data-types/float.md),
 but may have different types.
 
 **Returned value**

docs/en/sql-reference/functions/date-time-functions.md

@@ -1623,12 +1623,22 @@ If unit `WEEK` was specified, `toStartOfInterval` assumes that weeks start on Mo
 toStartOfInterval(value, INTERVAL x unit[, time_zone])
 toStartOfInterval(value, INTERVAL x unit[, origin[, time_zone]])
 ```
+Aliases: `time_bucket`, `date_bin`.
 
 The second overload emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function, e.g.
 
 ``` SQL
 SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));
 ```
+
+Result:
+
+``` reference
+┌───toStartOfInterval(...)─┐
+│      2023-01-01 14:44:30 │
+└──────────────────────────┘
+```
+
 **See Also**
 - [date_trunc](#date_trunc)
 
@@ -1964,6 +1974,38 @@ Result:
 
 Converts a date, or date with time, to a UInt16 number containing the ISO Year number.
 
+**Syntax**
+
+```sql
+toISOYear(value)
+```
+
+**Arguments**
+
+- `value` — The value with date or date with time.
+
+**Returned value**
+
+- `value` converted to the current ISO year number. [UInt16](../data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+  toISOYear(toDate('2024/10/02')) as year1,
+  toISOYear(toDateTime('2024/10/02 01:30:00')) as year2
+```
+
+Result:
+
+```response
+┌─year1─┬─year2─┐
+│  2024 │  2024 │
+└───────┴───────┘
+```
+
 ## toISOWeek
 
 Converts a date, or date with time, to a UInt8 number containing the ISO Week number.

docs/en/sql-reference/functions/hash-functions.md

@@ -51,6 +51,40 @@ Calculates the MD5 from a string and returns the resulting set of bytes as Fixed
 If you do not need MD5 in particular, but you need a decent cryptographic 128-bit hash, use the ‘sipHash128’ function instead.
 If you want to get the same result as output by the md5sum utility, use lower(hex(MD5(s))).
 
+## RIPEMD160
+
+Produces [RIPEMD-160](https://en.wikipedia.org/wiki/RIPEMD) hash value.
+
+**Syntax**
+
+```sql
+RIPEMD160(input)
+```
+
+**Parameters**
+
+- `input`: Input string. [String](../data-types/string.md)
+
+**Returned value**
+
+- A 160-bit `RIPEMD-160` hash value of type [FixedString(20)](../data-types/fixedstring.md).
+
+**Example**
+
+Use the [hex](../functions/encoding-functions.md/#hex) function to represent the result as a hex-encoded string.
+
+Query:
+
+```sql
+SELECT HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'));
+```
+
+```response
+┌─HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'))─┐
+│ 37F332F68DB77BD9D7EDD4969571AD671CF9DD3B                      │
+└───────────────────────────────────────────────────────────────┘
+```
+
 ## sipHash64
 
 Produces a 64-bit [SipHash](https://en.wikipedia.org/wiki/SipHash) hash value.

docs/en/sql-reference/functions/ip-address-functions.md

@@ -280,6 +280,38 @@ SELECT
 
 Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns `0.0.0.0` (0 IPv4).
 
+**Syntax**
+
+```sql
+toIPv4OrDefault(value)
+```
+
+**Arguments**
+
+- `value` — The value with IPv4 address.
+
+**Returned value**
+
+- `value` converted to the current IPv4 address. [String](../data-types/string.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+  toIPv4OrDefault('192.168.0.1') AS s1,
+  toIPv4OrDefault('192.168.0') AS s2
+```
+
+Result:
+
+```response
+┌─s1──────────┬─s2──────┐
+│ 192.168.0.1 │ 0.0.0.0 │
+└─────────────┴─────────┘
+```
+
 ## toIPv4OrNull(string)
 
 Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns null.

docs/en/sql-reference/functions/other-functions.md

@@ -207,7 +207,31 @@ If `NULL` is passed, then the function returns type `Nullable(Nothing)`, which c
 **Syntax**
 
 ```sql
-toTypeName(x)
+toTypeName(value)
+```
+
+**Arguments**
+
+- `value` — The value with any arbitrary.
+
+**Returned value**
+
+- `value` converted to the current data type name. [String](../data-types/string.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT toTypeName(123);
+```
+
+Result:
+
+```response
+┌─toTypeName(123)─┐
+│ UInt8           │
+└─────────────────┘
 ```
 
 ## blockSize {#blockSize}
@@ -500,6 +524,30 @@ Useful in table engine parameters of `CREATE TABLE` queries where you need to sp
 currentDatabase()
 ```
 
+**Arguments**
+
+None.
+
+**Returned value**
+
+- `value` returns the current database name. [String](../data-types/string.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT currentDatabase()
+```
+
+Result:
+
+```response
+┌─currentDatabase()─┐
+│ default           │
+└───────────────────┘
+```
+
 ## currentUser {#currentUser}
 
 Returns the name of the current user. In case of a distributed query, the name of the user who initiated the query is returned.
@@ -2789,6 +2837,45 @@ Result:
 
 - [Custom Settings](../../operations/settings/index.md#custom_settings)
 
+## getSettingOrDefault
+
+Returns the current value of a [custom setting](../../operations/settings/index.md#custom_settings) or returns the default value specified in the 2nd argument if the custom setting is not set in the current profile.
+
+**Syntax**
+
+```sql
+getSettingOrDefault('custom_setting', default_value);
+```
+
+**Parameter**
+
+- `custom_setting` — The setting name. [String](../data-types/string.md).
+- `default_value` — Value to return if custom_setting is not set. Value may be of any data type or Null.
+
+**Returned value**
+
+- The setting's current value or default_value if setting is not set.
+
+**Example**
+
+```sql
+SELECT getSettingOrDefault('custom_undef1', 'my_value');
+SELECT getSettingOrDefault('custom_undef2', 100);
+SELECT getSettingOrDefault('custom_undef3', NULL);
+```
+
+Result:
+
+```
+my_value
+100
+NULL
+```
+
+**See Also**
+
+- [Custom Settings](../../operations/settings/index.md#custom_settings)
+
 ## isDecimalOverflow
 
 Checks whether the [Decimal](../data-types/decimal.md) value is outside its precision or outside the specified precision.

docs/en/sql-reference/functions/rounding-functions.md

@@ -288,7 +288,7 @@ roundToExp2(num)
 
 **Parameters**
 
-- `num`: A number representing an age in years. [UInt](../data-types/int-uint.md)/[Float](../data-types/float.md).
+- `num`: A number to round. [UInt](../data-types/int-uint.md)/[Float](../data-types/float.md).
 
 **Returned value**
 

docs/en/sql-reference/statements/alter/column.md

@@ -272,8 +272,7 @@ ALTER TABLE table_name MODIFY COLUMN column_name RESET SETTING max_compress_bloc
 
 ## MATERIALIZE COLUMN
 
-Materializes a column with a `DEFAULT` or `MATERIALIZED` value expression.
-This statement can be used to rewrite existing column data after a `DEFAULT` or `MATERIALIZED` expression has been added or updated (which only updates the metadata but does not change existing data).
+Materializes a column with a `DEFAULT` or `MATERIALIZED` value expression. When adding a materialized column using `ALTER TABLE table_name ADD COLUMN column_name MATERIALIZED`, existing rows without materialized values are not automatically filled. `MATERIALIZE COLUMN` statement can be used to rewrite existing column data after a `DEFAULT` or `MATERIALIZED` expression has been added or updated (which only updates the metadata but does not change existing data).
 Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
 
 For columns with a new or updated `MATERIALIZED` value expression, all existing rows are rewritten.

docs/en/sql-reference/statements/alter/user.md

@@ -12,10 +12,10 @@ Syntax:
 ``` sql
 ALTER USER [IF EXISTS] name1 [RENAME TO new_name |, name2 [,...]] 
     [ON CLUSTER cluster_name]
-    [NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}]
+    [NOT IDENTIFIED | RESET AUTHENTICATION METHODS TO NEW | {IDENTIFIED | ADD IDENTIFIED} {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']} 
+    [, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]]
     [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
     [VALID UNTIL datetime]
-    [RESET AUTHENTICATION METHODS TO NEW]
     [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
     [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]

docs/en/sql-reference/statements/create/table.md

@@ -43,6 +43,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine]
 
 Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table.
 
+### With a Schema and Data Cloned from Another Table 
+
+``` sql
+CREATE TABLE [IF NOT EXISTS] [db.]table_name CLONE AS [db2.]name2 [ENGINE = engine]
+```
+
+Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table. After the new table is created, all partitions from `db2.name2` are attached to it. In other words, the data of `db2.name2` is cloned into `db.table_name` upon creation. This query is equivalent to the following:
+
+``` sql
+CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine];
+ALTER TABLE [db.]table_name ATTACH PARTITION ALL FROM [db2].name2;
+```
+
 ### From a Table Function
 
 ``` sql

docs/en/sql-reference/statements/create/user.md

@@ -11,10 +11,10 @@ Syntax:
 
 ``` sql
 CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [, name2 [,...]] [ON CLUSTER cluster_name]
-    [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}]
+    [NOT IDENTIFIED | IDENTIFIED {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']} 
+    [, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]]
     [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
     [VALID UNTIL datetime]
-    [RESET AUTHENTICATION METHODS TO NEW]
     [IN access_storage_type]
     [DEFAULT ROLE role [,...]]
     [DEFAULT DATABASE database | NONE]

docs/en/sql-reference/statements/exists.md

@@ -7,7 +7,7 @@ sidebar_label: EXISTS
 # EXISTS Statement
 
 ``` sql
-EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format]
+EXISTS [TEMPORARY] [TABLE|DICTIONARY|DATABASE] [db.]name [INTO OUTFILE filename] [FORMAT format]
 ```
 
 Returns a single `UInt8`-type column, which contains the single value `0` if the table or database does not exist, or `1` if the table exists in the specified database.

docs/ru/sql-reference/functions/date-time-functions.md

@@ -468,6 +468,52 @@ WITH toDateTime64('2020-01-01 10:20:30.999', 3) AS dt64 SELECT toStartOfSecond(d
 `toStartOfInterval(t, INTERVAL 1 day)` возвращает то же самое, что и `toStartOfDay(t)`,
 `toStartOfInterval(t, INTERVAL 15 minute)` возвращает то же самое, что и `toStartOfFifteenMinutes(t)`, и т.п.
 
+Вычисление происхожит относительно конкретного времени:
+
+| Interval    | Start                  |
+|-------------|------------------------|
+| YEAR        | year 0                 |
+| QUARTER     | 1900 Q1                |
+| MONTH       | 1900 January           |
+| WEEK        | 1970, 1st week (01-05) |
+| DAY         | 1970-01-01             |
+| HOUR        | (*)                    |
+| MINUTE      | 1970-01-01 00:00:00    |
+| SECOND      | 1970-01-01 00:00:00    |
+| MILLISECOND | 1970-01-01 00:00:00    |
+| MICROSECOND | 1970-01-01 00:00:00    |
+| NANOSECOND  | 1970-01-01 00:00:00    |
+
+(*) часовые интервалы особенные: Вычисления всегда происходят относительно 00:00:00 (полночь) текущего дня. В результате, только
+    часовые значения полезны только между 1 и 23.
+
+Если была указана единица измерения `WEEK`, то `toStartOfInterval` предполагает, что неделя начинается в понедельник. Важно, что это поведение отдичается от поведения функции `toStartOfWeek` в которой неделя по умолчанию начинается у воскресенье.
+
+**Синтаксис**
+
+```sql
+toStartOfInterval(value, INTERVAL x unit[, time_zone])
+toStartOfInterval(value, INTERVAL x unit[, origin[, time_zone]])
+```
+Синонимы: `time_bucket`, `date_bin`.
+
+Вторая перегрузка эмулирует функцию `time_bucket()` из TimescaleDB, и функцию `date_bin()` из PostgreSQL, например:
+
+``` SQL
+SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));
+```
+
+Результат:
+
+``` reference
+┌───toStartOfInterval(...)─┐
+│      2023-01-01 14:44:30 │
+└──────────────────────────┘
+```
+
+**См. также**
+- [date_trunc](#date_trunc)
+
 ## toTime {#totime}
 
 Переводит дату-с-временем на некоторую фиксированную дату, сохраняя при этом время.

docs/ru/sql-reference/statements/alter/user.md

@@ -13,8 +13,10 @@ sidebar_label: USER
 ``` sql
 ALTER USER [IF EXISTS] name1 [RENAME TO new_name |, name2 [,...]] 
     [ON CLUSTER cluster_name]
-    [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}]
+    [NOT IDENTIFIED | RESET AUTHENTICATION METHODS TO NEW | {IDENTIFIED | ADD IDENTIFIED} {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']} 
+    [, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]]
     [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
+    [VALID UNTIL datetime]
     [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
     [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]

docs/ru/sql-reference/statements/create/user.md

@@ -12,8 +12,11 @@ sidebar_label: "Пользователь"
 
 ``` sql
 CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [, name2 [,...]] [ON CLUSTER cluster_name]
-    [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'}]
+    [NOT IDENTIFIED | IDENTIFIED {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']} 
+    [, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]]
     [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
+    [VALID UNTIL datetime]
+    [IN access_storage_type]
     [DEFAULT ROLE role [,...]]
     [DEFAULT DATABASE database | NONE]
     [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]

docs/zh/sql-reference/aggregate-functions/reference/avgweighted.md

@@ -21,8 +21,7 @@ avgWeighted(x, weight)
 
 `x` 和 `weight` 的类型必须是
 [整数](../../../sql-reference/data-types/int-uint.md), 或
-[浮点数](../../../sql-reference/data-types/float.md), 或
-[定点数](../../../sql-reference/data-types/decimal.md),
+[浮点数](../../../sql-reference/data-types/float.md),
 但是可以不一样。
 
 **返回值**

programs/disks/CommandRead.cpp

@@ -26,7 +26,7 @@ public:
         String path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
         std::optional<String> path_to = getValueFromCommandLineOptionsWithOptional<String>(options, "path-to");
 
-        auto in = disk.getDisk()->readFile(path_from);
+        auto in = disk.getDisk()->readFile(path_from, getReadSettings());
         std::unique_ptr<WriteBufferFromFileBase> out = {};
         if (path_to.has_value())
         {

programs/disks/CommandWrite.cpp

@@ -39,7 +39,7 @@ public:
             else
             {
                 String relative_path_from = disk.getRelativeFromRoot(path_from.value());
-                return disk.getDisk()->readFile(relative_path_from);
+                return disk.getDisk()->readFile(relative_path_from, getReadSettings());
             }
         }();
 

programs/odbc-bridge/MainHandler.cpp

@@ -164,7 +164,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
             std::string table_name = params.get("table_name");
             LOG_TRACE(log, "DB name: '{}', table name: '{}'", db_name, table_name);
 
-            auto quoting_style = IdentifierQuotingStyle::None;
+            auto quoting_style = IdentifierQuotingStyle::Backticks;
 #if USE_ODBC
             quoting_style = getQuotingStyle(connection_handler);
 #endif

programs/odbc-bridge/getIdentifierQuote.cpp

@@ -38,7 +38,7 @@ IdentifierQuotingStyle getQuotingStyle(nanodbc::ConnectionHolderPtr connection)
 {
     auto identifier_quote = getIdentifierQuote(connection);
     if (identifier_quote.empty())
-        return IdentifierQuotingStyle::None;
+        return IdentifierQuotingStyle::Backticks;
     else if (identifier_quote[0] == '`')
         return IdentifierQuotingStyle::Backticks;
     else if (identifier_quote[0] == '"')

programs/server/Server.cpp

@@ -36,7 +36,7 @@
 #include <Common/ZooKeeper/ZooKeeperNodeCache.h>
 #include <Common/formatReadable.h>
 #include <Common/getMultipleKeysFromConfig.h>
-#include <Common/getNumberOfPhysicalCPUCores.h>
+#include <Common/getNumberOfCPUCoresToUse.h>
 #include <Common/getExecutablePath.h>
 #include <Common/ProfileEvents.h>
 #include <Common/Scheduler/IResourceManager.h>
@@ -833,11 +833,13 @@ try
 
     const size_t physical_server_memory = getMemoryAmount();
 
-    LOG_INFO(log, "Available RAM: {}; logical cores: {}; used cores: {}.",
+    LOG_INFO(
+        log,
+        "Available RAM: {}; logical cores: {}; used cores: {}.",
         formatReadableSizeWithBinarySuffix(physical_server_memory),
         std::thread::hardware_concurrency(),
-        getNumberOfPhysicalCPUCores()  // on ARM processors it can show only enabled at current moment cores
-        );
+        getNumberOfCPUCoresToUse() // on ARM processors it can show only enabled at current moment cores
+    );
 
 #if defined(__x86_64__)
     String cpu_info;
@@ -1060,8 +1062,9 @@ try
         0, // We don't need any threads one all the parts will be deleted
         server_settings.max_parts_cleaning_thread_pool_size);
 
-    auto max_database_replicated_create_table_thread_pool_size = server_settings.max_database_replicated_create_table_thread_pool_size ?
-        server_settings.max_database_replicated_create_table_thread_pool_size : getNumberOfPhysicalCPUCores();
+    auto max_database_replicated_create_table_thread_pool_size = server_settings.max_database_replicated_create_table_thread_pool_size
+        ? server_settings.max_database_replicated_create_table_thread_pool_size
+        : getNumberOfCPUCoresToUse();
     getDatabaseReplicatedCreateTablesThreadPool().initialize(
         max_database_replicated_create_table_thread_pool_size,
         0, // We don't need any threads once all the tables will be created
@@ -1638,7 +1641,7 @@ try
                 concurrent_threads_soft_limit = new_server_settings.concurrent_threads_soft_limit_num;
             if (new_server_settings.concurrent_threads_soft_limit_ratio_to_cores > 0)
             {
-                auto value = new_server_settings.concurrent_threads_soft_limit_ratio_to_cores * getNumberOfPhysicalCPUCores();
+                auto value = new_server_settings.concurrent_threads_soft_limit_ratio_to_cores * getNumberOfCPUCoresToUse();
                 if (value > 0 && value < concurrent_threads_soft_limit)
                     concurrent_threads_soft_limit = value;
             }

pyproject.toml

@@ -7,42 +7,34 @@ max-branches=50
 max-nested-blocks=10
 max-statements=200
 
-[tool.isort]
-profile = "black"
-src_paths = ["src", "tests/ci", "tests/sqllogic"]
-
 [tool.pylint.FORMAT]
 #ignore-long-lines = (# )?<?https?://\S+>?$
 
 [tool.pylint.'MESSAGES CONTROL']
           # pytest.mark.parametrize is not callable (not-callable)
 disable = '''
-          pointless-string-statement,
-          line-too-long,
-          missing-docstring,
-          too-few-public-methods,
-          invalid-name,
-          too-many-arguments,
-          keyword-arg-before-vararg,
-          too-many-locals,
-          too-many-instance-attributes,
+          bare-except,
+          broad-except,
           cell-var-from-loop,
           fixme,
+          invalid-name,
+          missing-docstring,
+          redefined-outer-name,
+          too-few-public-methods,
+          too-many-arguments,
+          too-many-instance-attributes,
+          too-many-locals,
           too-many-public-methods,
           wildcard-import,
-          unused-wildcard-import,
-          singleton-comparison,
-          not-callable,
-          redefined-outer-name,
-          broad-except,
-          bare-except,
-          no-else-return,
-          global-statement,
-          f-string-without-interpolation,
-          consider-using-with,
-          use-maxsplit-arg,
           '''
 
+[tool.isort]
+profile = "black"
+src_paths = ["src", "tests/ci", "tests/sqllogic", "tests/queries", "tests/integration"]
+
+[tool.black]
+required-version = 24
+
 [tool.pylint.SIMILARITIES]
 # due to SQL
 min-similarity-lines=1000

src/Access/AccessBackup.cpp

@@ -30,6 +30,7 @@ namespace ErrorCodes
 {
     extern const int CANNOT_RESTORE_TABLE;
     extern const int ACCESS_ENTITY_ALREADY_EXISTS;
+    extern const int ACCESS_ENTITY_NOT_FOUND;
     extern const int LOGICAL_ERROR;
 }
 
@@ -41,6 +42,7 @@ namespace
     {
         std::unordered_map<UUID, AccessEntityPtr> entities;
         std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
+        std::unordered_map<UUID, AccessEntityPtr> dependents;
 
         BackupEntryPtr toBackupEntry() const
         {
@@ -72,6 +74,24 @@ namespace
                 }
             }
 
+            if (!dependents.empty())
+            {
+                if (!dependencies.empty())
+                    writeText("\n", buf);
+                writeText("DEPENDENTS\n", buf);
+                for (const auto & [id, entity] : dependents)
+                {
+                    writeText(id, buf);
+                    writeChar('\t', buf);
+                    writeText(entity->getTypeInfo().name, buf);
+                    writeChar('\t', buf);
+                    writeText(entity->getName(), buf);
+                    writeChar('\n', buf);
+                    writeText(serializeAccessEntity(*entity), buf);
+                    writeChar('\n', buf);
+                }
+            }
+
             return std::make_shared<BackupEntryFromMemory>(buf.str());
         }
 
@@ -81,59 +101,71 @@ namespace
             {
                 AccessEntitiesInBackup res;
 
-                bool dependencies_found = false;
+                bool reading_dependencies = false;
+                bool reading_dependents = false;
 
                 while (!buf->eof())
                 {
                     String line;
                     readStringUntilNewlineInto(line, *buf);
                     buf->ignore();
+
                     if (line == "DEPENDENCIES")
                     {
-                        dependencies_found = true;
-                        break;
+                        reading_dependencies = true;
+                        reading_dependents = false;
+                        continue;
+                    }
+                    else if (line == "DEPENDENTS")
+                    {
+                        reading_dependents = true;
+                        reading_dependencies = false;
+                        continue;
+                    }
+                    else if (line.empty())
+                    {
+                        continue;
                     }
 
-                    UUID id = parse<UUID>(line.substr(0, line.find('\t')));
-                    line.clear();
+                    size_t separator1 = line.find('\t');
+                    size_t separator2 = line.find('\t', separator1 + 1);
+                    if ((separator1 == String::npos) || (separator2 == String::npos))
+                        throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Separators not found in line {}", line);
 
-                    String queries;
-                    while (!buf->eof())
+                    UUID id = parse<UUID>(line.substr(0, separator1));
+                    AccessEntityType type = AccessEntityTypeInfo::parseType(line.substr(separator1 + 1, separator2 - separator1 - 1));
+                    String name = line.substr(separator2 + 1);
+
+                    if (reading_dependencies)
                     {
-                        String query;
-                        readStringUntilNewlineInto(query, *buf);
-                        buf->ignore();
-                        if (query.empty())
-                            break;
-                        if (!queries.empty())
-                            queries.append("\n");
-                        queries.append(query);
+                       res.dependencies.emplace(id, std::pair{name, type});
                     }
-
-                    AccessEntityPtr entity = deserializeAccessEntity(queries);
-                    res.entities.emplace(id, entity);
-                }
-
-                if (dependencies_found)
-                {
-                    while (!buf->eof())
+                    else
                     {
-                        String id_as_string;
-                        readStringInto(id_as_string, *buf);
-                        buf->ignore();
-                        UUID id = parse<UUID>(id_as_string);
+                        String queries;
+                        while (!buf->eof())
+                        {
+                            String query;
+                            readStringUntilNewlineInto(query, *buf);
+                            buf->ignore();
+                            if (query.empty())
+                                break;
+                            if (!queries.empty())
+                                queries.append("\n");
+                            queries.append(query);
+                        }
 
-                        String type_as_string;
-                        readStringInto(type_as_string, *buf);
-                        buf->ignore();
-                        AccessEntityType type = AccessEntityTypeInfo::parseType(type_as_string);
+                        AccessEntityPtr entity = deserializeAccessEntity(queries);
 
-                        String name;
-                        readStringInto(name, *buf);
-                        buf->ignore();
+                        if (name != entity->getName())
+                            throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Unexpected name {} is specified for {}", name, entity->formatTypeWithName());
+                        if (type != entity->getType())
+                            throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Unexpected type {} is specified for {}", AccessEntityTypeInfo::get(type).name, entity->formatTypeWithName());
 
-                        if (!res.entities.contains(id))
-                            res.dependencies.emplace(id, std::pair{name, type});
+                        if (reading_dependents)
+                            res.dependents.emplace(id, entity);
+                        else
+                            res.entities.emplace(id, entity);
                     }
                 }
 
@@ -146,190 +178,59 @@ namespace
             }
         }
     };
-
-    std::vector<UUID> findDependencies(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities)
-    {
-        std::vector<UUID> res;
-        for (const auto & entity : entities | boost::adaptors::map_values)
-            insertAtEnd(res, entity->findDependencies());
-
-        /// Remove duplicates in the list of dependencies (some entities can refer to other entities).
-        ::sort(res.begin(), res.end());
-        res.erase(std::unique(res.begin(), res.end()), res.end());
-        for (const auto & id : entities | boost::adaptors::map_keys)
-        {
-            auto it = std::lower_bound(res.begin(), res.end(), id);
-            if ((it != res.end()) && (*it == id))
-                res.erase(it);
-        }
-        return res;
-    }
-
-    std::unordered_map<UUID, std::pair<String, AccessEntityType>> readDependenciesNamesAndTypes(const std::vector<UUID> & dependencies, const AccessControl & access_control)
-    {
-        std::unordered_map<UUID, std::pair<String, AccessEntityType>> res;
-        for (const auto & id : dependencies)
-        {
-            if (auto name_and_type = access_control.tryReadNameWithType(id))
-                res.emplace(id, name_and_type.value());
-        }
-        return res;
-    }
-
-    /// Checks if new entities (which we're going to restore) already exist,
-    /// and either skips them or throws an exception depending on the restore settings.
-    void checkExistingEntities(std::vector<std::pair<UUID, AccessEntityPtr>> & entities,
-                               std::unordered_map<UUID, UUID> & old_to_new_id,
-                               const AccessControl & access_control,
-                               RestoreAccessCreationMode creation_mode)
-    {
-        if (creation_mode == RestoreAccessCreationMode::kReplace)
-            return;
-
-        auto should_skip = [&](const std::pair<UUID, AccessEntityPtr> & id_and_entity)
-        {
-            const auto & id = id_and_entity.first;
-            const auto & entity = *id_and_entity.second;
-            auto existing_id = access_control.find(entity.getType(), entity.getName());
-            if (!existing_id)
-            {
-                return false;
-            }
-            else if (creation_mode == RestoreAccessCreationMode::kCreateIfNotExists)
-            {
-                old_to_new_id[id] = *existing_id;
-                return true;
-            }
-            else
-            {
-                throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Cannot restore {} because it already exists", entity.formatTypeWithName());
-            }
-        };
-
-        std::erase_if(entities, should_skip);
-    }
-
-    /// If new entities (which we're going to restore) depend on other entities which are not going to be restored or not present in the backup
-    /// then we should try to replace those dependencies with already existing entities.
-    void resolveDependencies(const std::unordered_map<UUID, std::pair<String, AccessEntityType>> & dependencies,
-                             std::unordered_map<UUID, UUID> & old_to_new_ids,
-                             const AccessControl & access_control,
-                             bool allow_unresolved_dependencies)
-    {
-        for (const auto & [id, name_and_type] : dependencies)
-        {
-            std::optional<UUID> new_id;
-            if (allow_unresolved_dependencies)
-                new_id = access_control.find(name_and_type.second, name_and_type.first);
-            else
-                new_id = access_control.getID(name_and_type.second, name_and_type.first);
-            if (new_id)
-                old_to_new_ids.emplace(id, *new_id);
-        }
-    }
-
-    /// Generates random IDs for the new entities.
-    void generateRandomIDs(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, std::unordered_map<UUID, UUID> & old_to_new_ids)
-    {
-        Poco::UUIDGenerator generator;
-        for (auto & [id, entity] : entities)
-        {
-            UUID new_id;
-            generator.createRandom().copyTo(reinterpret_cast<char *>(&new_id));
-            old_to_new_ids.emplace(id, new_id);
-            id = new_id;
-        }
-    }
-
-    /// Updates dependencies of the new entities using a specified map.
-    void replaceDependencies(std::vector<std::pair<UUID, AccessEntityPtr>> & entities,
-                             const std::unordered_map<UUID, UUID> & old_to_new_ids)
-    {
-        for (auto & entity : entities | boost::adaptors::map_values)
-            IAccessEntity::replaceDependencies(entity, old_to_new_ids);
-    }
-
-    AccessRightsElements getRequiredAccessToRestore(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities)
-    {
-        AccessRightsElements res;
-        for (const auto & entity : entities | boost::adaptors::map_values)
-        {
-            auto entity_type = entity->getType();
-            switch (entity_type)
-            {
-                case User::TYPE:
-                {
-                    const auto & user = typeid_cast<const User &>(*entity);
-                    res.emplace_back(AccessType::CREATE_USER);
-                    auto elements = user.access.getElements();
-                    for (auto & element : elements)
-                    {
-                        if (element.is_partial_revoke)
-                            continue;
-                        element.grant_option = true;
-                        res.emplace_back(element);
-                    }
-                    if (!user.granted_roles.isEmpty())
-                        res.emplace_back(AccessType::ROLE_ADMIN);
-                    break;
-                }
-
-                case Role::TYPE:
-                {
-                    const auto & role = typeid_cast<const Role &>(*entity);
-                    res.emplace_back(AccessType::CREATE_ROLE);
-                    auto elements = role.access.getElements();
-                    for (auto & element : elements)
-                    {
-                        if (element.is_partial_revoke)
-                            continue;
-                        element.grant_option = true;
-                        res.emplace_back(element);
-                    }
-                    if (!role.granted_roles.isEmpty())
-                        res.emplace_back(AccessType::ROLE_ADMIN);
-                    break;
-                }
-
-                case SettingsProfile::TYPE:
-                {
-                    res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE);
-                    break;
-                }
-
-                case RowPolicy::TYPE:
-                {
-                    const auto & policy = typeid_cast<const RowPolicy &>(*entity);
-                    res.emplace_back(AccessType::CREATE_ROW_POLICY, policy.getDatabase(), policy.getTableName());
-                    break;
-                }
-
-                case Quota::TYPE:
-                {
-                    res.emplace_back(AccessType::CREATE_QUOTA);
-                    break;
-                }
-
-                default:
-                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown type: {}", toString(entity_type));
-            }
-        }
-        return res;
-    }
 }
 
 
-std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
-    const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
-    const String & data_path_in_backup,
-    size_t counter,
-    const AccessControl & access_control)
+std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
+    const std::vector<UUID> & entities_ids,
+    const std::unordered_map<UUID, AccessEntityPtr> & all_entities,
+    bool write_dependents,
+    const String & data_path_in_backup)
 {
-    auto dependencies = readDependenciesNamesAndTypes(findDependencies(access_entities), access_control);
     AccessEntitiesInBackup ab;
-    boost::range::copy(access_entities, std::inserter(ab.entities, ab.entities.end()));
-    ab.dependencies = std::move(dependencies);
-    String filename = fmt::format("access{:02}.txt", counter + 1); /// access01.txt, access02.txt, ...
+
+    std::unordered_set<UUID> entities_ids_set;
+    for (const auto & id : entities_ids)
+        entities_ids_set.emplace(id);
+
+    for (const auto & id : entities_ids)
+    {
+        auto it = all_entities.find(id);
+        if (it != all_entities.end())
+        {
+            AccessEntityPtr entity = it->second;
+            ab.entities.emplace(id, entity);
+
+            auto dependencies = entity->findDependencies();
+            for (const auto & dependency_id : dependencies)
+            {
+                if (!entities_ids_set.contains(dependency_id))
+                {
+                    auto it_dependency = all_entities.find(dependency_id);
+                    if (it_dependency != all_entities.end())
+                    {
+                        auto dependency_entity = it_dependency->second;
+                        ab.dependencies.emplace(dependency_id, std::make_pair(dependency_entity->getName(), dependency_entity->getType()));
+                    }
+                }
+            }
+        }
+    }
+
+    if (write_dependents)
+    {
+        for (const auto & [id, possible_dependent] : all_entities)
+        {
+            if (!entities_ids_set.contains(id) && possible_dependent->hasDependencies(entities_ids_set))
+            {
+                auto dependent = possible_dependent->clone();
+                dependent->clearAllExceptDependencies();
+                ab.dependents.emplace(id, dependent);
+            }
+        }
+    }
+
+    String filename = fmt::format("access-{}.txt", UUIDHelpers::generateV4());
     String file_path_in_backup = fs::path{data_path_in_backup} / filename;
     return {file_path_in_backup, ab.toBackupEntry()};
 }
@@ -339,61 +240,411 @@ AccessRestorerFromBackup::AccessRestorerFromBackup(
     const BackupPtr & backup_, const RestoreSettings & restore_settings_)
     : backup(backup_)
     , creation_mode(restore_settings_.create_access)
-    , allow_unresolved_dependencies(restore_settings_.allow_unresolved_access_dependencies)
+    , skip_unresolved_dependencies(restore_settings_.skip_unresolved_access_dependencies)
+    , update_dependents(restore_settings_.update_access_entities_dependents)
+    , log(getLogger("AccessRestorerFromBackup"))
 {
 }
 
 AccessRestorerFromBackup::~AccessRestorerFromBackup() = default;
 
-void AccessRestorerFromBackup::addDataPath(const String & data_path)
+
+void AccessRestorerFromBackup::addDataPath(const String & data_path_in_backup)
 {
-    if (!data_paths.emplace(data_path).second)
-        return;
+    if (loaded)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities already loaded");
 
-    fs::path data_path_in_backup_fs = data_path;
-    Strings filenames = backup->listFiles(data_path, /*recursive*/ false);
-    if (filenames.empty())
-        return;
-
-    for (const String & filename : filenames)
-    {
-        if (!filename.starts_with("access") || !filename.ends_with(".txt"))
-            throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File name {} doesn't match the wildcard \"access*.txt\"",
-                            String{data_path_in_backup_fs / filename});
-    }
-
-    ::sort(filenames.begin(), filenames.end());
-
-    for (const String & filename : filenames)
-    {
-        String filepath_in_backup = data_path_in_backup_fs / filename;
-        auto read_buffer_from_backup = backup->readFile(filepath_in_backup);
-        auto ab = AccessEntitiesInBackup::fromBackupEntry(std::move(read_buffer_from_backup), filepath_in_backup);
-
-        boost::range::copy(ab.entities, std::back_inserter(entities));
-        boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end()));
-    }
-
-    for (const auto & id : entities | boost::adaptors::map_keys)
-        dependencies.erase(id);
+    if (std::find(data_paths_in_backup.begin(), data_paths_in_backup.end(), data_path_in_backup) == data_paths_in_backup.end())
+        data_paths_in_backup.emplace_back(data_path_in_backup);
 }
 
+
+void AccessRestorerFromBackup::loadFromBackup()
+{
+    if (loaded)
+        return;
+
+    /// Parse files "access*.txt" found in the added data paths in the backup.
+    for (size_t data_path_index = 0; data_path_index != data_paths_in_backup.size(); ++data_path_index)
+    {
+        const String & data_path_in_backup = data_paths_in_backup[data_path_index];
+
+        fs::path data_path_in_backup_fs = data_path_in_backup;
+        Strings filenames = backup->listFiles(data_path_in_backup_fs, /*recursive*/ false);
+        if (filenames.empty())
+            continue;
+
+        for (const String & filename : filenames)
+        {
+            if (!filename.starts_with("access") || !filename.ends_with(".txt"))
+                throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File name {} doesn't match the wildcard \"access*.txt\"",
+                                String{data_path_in_backup_fs / filename});
+        }
+
+        for (const String & filename : filenames)
+        {
+            String filepath_in_backup = data_path_in_backup_fs / filename;
+            AccessEntitiesInBackup ab;
+
+            try
+            {
+                auto read_buffer_from_backup = backup->readFile(filepath_in_backup);
+                ab = AccessEntitiesInBackup::fromBackupEntry(std::move(read_buffer_from_backup), filepath_in_backup);
+            }
+            catch (Exception & e)
+            {
+                e.addMessage("While reading access entities from {} in backup", filepath_in_backup);
+                throw;
+            }
+
+            for (const auto & [id, entity] : ab.entities)
+            {
+                auto it = entity_infos.find(id);
+                if (it == entity_infos.end())
+                {
+                    it = entity_infos.emplace(id, EntityInfo{.id = id, .name = entity->getName(), .type = entity->getType()}).first;
+                }
+                EntityInfo & entity_info = it->second;
+                entity_info.entity = entity;
+                entity_info.restore = true;
+                entity_info.data_path_index = data_path_index;
+            }
+
+            for (const auto & [id, name_and_type] : ab.dependencies)
+            {
+                auto it = entity_infos.find(id);
+                if (it == entity_infos.end())
+                {
+                    it = entity_infos.emplace(id, EntityInfo{.id = id, .name = name_and_type.first, .type = name_and_type.second}).first;
+                }
+                EntityInfo & entity_info = it->second;
+                entity_info.is_dependency = true;
+            }
+
+            for (const auto & [id, entity] : ab.dependents)
+            {
+                auto it = entity_infos.find(id);
+                if (it == entity_infos.end())
+                {
+                    it = entity_infos.emplace(id, EntityInfo{.id = id, .name = entity->getName(), .type = entity->getType()}).first;
+                }
+                EntityInfo & entity_info = it->second;
+                if (!entity_info.restore)
+                    entity_info.entity = entity;
+            }
+        }
+    }
+
+    loaded = true;
+}
+
+
 AccessRightsElements AccessRestorerFromBackup::getRequiredAccess() const
 {
-    return getRequiredAccessToRestore(entities);
+    if (!loaded)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities not loaded");
+
+    AccessRightsElements res;
+    for (const auto & [id, entity_info] : entity_infos)
+    {
+        if (!entity_info.restore)
+            continue;
+        const auto & entity = entity_info.entity;
+        auto entity_type = entity->getType();
+        switch (entity_type)
+        {
+            case User::TYPE:
+            {
+                const auto & user = typeid_cast<const User &>(*entity);
+                res.emplace_back(AccessType::CREATE_USER);
+                auto elements = user.access.getElements();
+                for (auto & element : elements)
+                {
+                    if (element.is_partial_revoke)
+                        continue;
+                    element.grant_option = true;
+                    res.emplace_back(element);
+                }
+                if (!user.granted_roles.isEmpty())
+                    res.emplace_back(AccessType::ROLE_ADMIN);
+                break;
+            }
+
+            case Role::TYPE:
+            {
+                const auto & role = typeid_cast<const Role &>(*entity);
+                res.emplace_back(AccessType::CREATE_ROLE);
+                auto elements = role.access.getElements();
+                for (auto & element : elements)
+                {
+                    if (element.is_partial_revoke)
+                        continue;
+                    element.grant_option = true;
+                    res.emplace_back(element);
+                }
+                if (!role.granted_roles.isEmpty())
+                    res.emplace_back(AccessType::ROLE_ADMIN);
+                break;
+            }
+
+            case SettingsProfile::TYPE:
+            {
+                res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE);
+                break;
+            }
+
+            case RowPolicy::TYPE:
+            {
+                const auto & policy = typeid_cast<const RowPolicy &>(*entity);
+                res.emplace_back(AccessType::CREATE_ROW_POLICY, policy.getDatabase(), policy.getTableName());
+                break;
+            }
+
+            case Quota::TYPE:
+            {
+                res.emplace_back(AccessType::CREATE_QUOTA);
+                break;
+            }
+
+            default:
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown type: {}", toString(entity_type));
+        }
+    }
+    return res;
 }
 
-std::vector<std::pair<UUID, AccessEntityPtr>> AccessRestorerFromBackup::getAccessEntities(const AccessControl & access_control) const
+
+void AccessRestorerFromBackup::generateRandomIDsAndResolveDependencies(const AccessControl & access_control)
 {
-    auto new_entities = entities;
+    if (ids_assigned)
+        return;
 
+    if (!loaded)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities not loaded");
+
+    /// Calculate `new_id` for each entity info.
+    /// Check which ones of the loaded access entities already exist.
+    /// Generate random UUIDs for access entities which we're going to restore if they don't exist.
+    for (auto & [id, entity_info] : entity_infos)
+    {
+        const String & name = entity_info.name;
+        auto type = entity_info.type;
+
+        if (entity_info.restore && (creation_mode == RestoreAccessCreationMode::kReplace))
+        {
+            entity_info.new_id = UUIDHelpers::generateV4();
+            LOG_TRACE(log, "{}: Generated new UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *entity_info.new_id);
+            continue;
+        }
+
+        if (auto existing_id = access_control.find(type, name))
+        {
+            if (entity_info.restore && (creation_mode == RestoreAccessCreationMode::kCreate))
+            {
+                throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Cannot restore {} because it already exists",
+                                AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
+            }
+            bool was_going_to_restore = entity_info.restore;
+            entity_info.new_id = *existing_id;
+            entity_info.restore = false;
+            LOG_TRACE(log, "{}: Found with UUID {}{}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *existing_id,
+                      (was_going_to_restore ? ", will not restore" : ""));
+        }
+        else
+        {
+            if (entity_info.restore)
+            {
+                entity_info.new_id = UUIDHelpers::generateV4();
+                LOG_TRACE(log, "{}: Generated new UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *entity_info.new_id);
+            }
+            else if (skip_unresolved_dependencies)
+            {
+                LOG_TRACE(log, "{}: Not found, ignoring", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
+            }
+            else
+            {
+                throw Exception(ErrorCodes::ACCESS_ENTITY_NOT_FOUND, "Cannot resolve {} while restoring from backup",
+                                AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
+            }
+        }
+    }
+
+    /// Prepare map from old UUIDs to new UUIDs.
+    std::unordered_set<UUID> ids_to_restore;
     std::unordered_map<UUID, UUID> old_to_new_ids;
-    checkExistingEntities(new_entities, old_to_new_ids, access_control, creation_mode);
-    resolveDependencies(dependencies, old_to_new_ids, access_control, allow_unresolved_dependencies);
-    generateRandomIDs(new_entities, old_to_new_ids);
-    replaceDependencies(new_entities, old_to_new_ids);
+    std::unordered_set<UUID> unresolved_ids;
 
-    return new_entities;
+    for (const auto & [id, entity_info] : entity_infos)
+    {
+        if (entity_info.restore)
+            ids_to_restore.insert(id);
+
+        if (entity_info.new_id)
+            old_to_new_ids[id] = *entity_info.new_id;
+        else
+            unresolved_ids.insert(id);
+    }
+
+    /// Calculate `is_dependent` for each entity info.
+    if (update_dependents)
+    {
+        for (auto & [id, entity_info] : entity_infos)
+        {
+            if (!entity_info.restore && entity_info.new_id && entity_info.entity && entity_info.entity->hasDependencies(ids_to_restore))
+                entity_info.is_dependent = true;
+        }
+    }
+
+    /// Remap the UUIDs of dependencies in the access entities we're going to restore.
+    for (auto & [id, entity_info] : entity_infos)
+    {
+        if (entity_info.restore || entity_info.is_dependent)
+        {
+            auto new_entity = entity_info.entity->clone();
+            new_entity->replaceDependencies(old_to_new_ids);
+            new_entity->removeDependencies(unresolved_ids);
+            entity_info.entity = new_entity;
+        }
+
+        if (entity_info.restore && data_path_with_entities_to_restore.empty())
+            data_path_with_entities_to_restore = data_paths_in_backup[entity_info.data_path_index];
+    }
+
+    ids_assigned = true;
+}
+
+
+AccessEntitiesToRestore AccessRestorerFromBackup::getEntitiesToRestore(const String & data_path_in_backup) const
+{
+    if (!ids_assigned)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "IDs not assigned");
+
+    if (data_path_in_backup != data_path_with_entities_to_restore)
+        return {};
+
+    AccessEntitiesToRestore res;
+    res.new_entities.reserve(entity_infos.size());
+    res.dependents.reserve(entity_infos.size());
+
+    for (const auto & [id, entity_info] : entity_infos)
+    {
+        if (entity_info.restore)
+            res.new_entities.emplace_back(*entity_info.new_id, entity_info.entity);
+
+        if (entity_info.is_dependent)
+            res.dependents.emplace_back(AccessEntitiesToRestore::Dependent{*entity_info.new_id, entity_info.entity});
+    }
+
+    return res;
+}
+
+
+void restoreAccessEntitiesFromBackup(
+    IAccessStorage & destination_access_storage,
+    const AccessEntitiesToRestore & entities_to_restore,
+    const RestoreSettings & restore_settings)
+{
+    if (entities_to_restore.new_entities.empty())
+        return; /// Nothing to restore.
+
+    auto log = getLogger("AccessRestorerFromBackup");
+
+    bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace);
+    bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate);
+    bool update_dependents = restore_settings.update_access_entities_dependents;
+
+    std::unordered_set<UUID> restored_ids;
+    std::unordered_map<UUID, UUID> new_to_existing_ids;
+    AccessEntitiesToRestore::Dependents additional_dependents;
+    additional_dependents.reserve(entities_to_restore.new_entities.size());
+
+    for (const auto & [id, entity] : entities_to_restore.new_entities)
+    {
+        const String & name = entity->getName();
+        auto type = entity->getType();
+        LOG_TRACE(log, "{}: Adding with UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), id);
+
+        UUID existing_id;
+        if (destination_access_storage.insert(id, entity, replace_if_exists, throw_if_exists, &existing_id))
+        {
+            LOG_TRACE(log, "{}: Added successfully", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
+            restored_ids.emplace(id);
+        }
+        else
+        {
+            /// Couldn't insert `entity` because there is an existing entity with the same name.
+            LOG_TRACE(log, "{}: Not added because already exists with UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), existing_id);
+            new_to_existing_ids[id] = existing_id;
+            if (update_dependents)
+                additional_dependents.emplace_back(AccessEntitiesToRestore::Dependent{existing_id, entity});
+        }
+    }
+
+    if (!new_to_existing_ids.empty())
+    {
+        std::vector<UUID> ids_to_update;
+        ids_to_update.reserve(restored_ids.size());
+        boost::copy(restored_ids, std::inserter(ids_to_update, ids_to_update.end()));
+
+        std::unordered_set<UUID> new_ids;
+        boost::copy(new_to_existing_ids | boost::adaptors::map_keys, std::inserter(new_ids, new_ids.end()));
+
+        /// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed,
+        /// then we should correct those dependencies.
+        auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
+        {
+            if (!entity->hasDependencies(new_ids))
+                return entity;
+            LOG_TRACE(log, "{}: Updating dependencies", entity->formatTypeWithName());
+            auto res = entity->clone();
+            res->replaceDependencies(new_to_existing_ids);
+            return res;
+        };
+
+        /// It's totally ok if some UUIDs from `ids_to_update` don't exist anymore, that's why we use tryUpdate() here.
+        destination_access_storage.tryUpdate(ids_to_update, update_func);
+    }
+
+    auto do_update_dependents = [&](const AccessEntitiesToRestore::Dependents & dependents)
+    {
+        if (dependents.empty())
+            return;
+
+        std::vector<UUID> ids_to_update;
+        ids_to_update.reserve(dependents.size());
+        std::unordered_map<UUID, AccessEntityPtr> id_to_source;
+
+        for (const auto & dependent : dependents)
+        {
+            const UUID & id = dependent.existing_id;
+            if (!destination_access_storage.isReadOnly(id))
+            {
+                ids_to_update.emplace_back(id);
+                auto modified_source = dependent.source->clone();
+                modified_source->replaceDependencies(new_to_existing_ids);
+                id_to_source[id] = modified_source;
+            }
+        }
+
+        /// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed,
+        /// then we should correct those dependencies.
+        auto update_func = [&](const AccessEntityPtr & entity, const UUID & existing_id) -> AccessEntityPtr
+        {
+            const auto & source = *id_to_source.at(existing_id);
+            if (!source.hasDependencies(restored_ids))
+                return entity;
+            LOG_TRACE(log, "{}: Updating dependent", entity->formatTypeWithName());
+            auto res = entity->clone();
+            res->copyDependenciesFrom(source, restored_ids);
+            return res;
+        };
+
+        /// It's totally ok if some UUIDs from `ids_to_update` don't exist anymore, that's why we use tryUpdate() here.
+        destination_access_storage.tryUpdate(ids_to_update, update_func);
+    };
+
+    do_update_dependents(entities_to_restore.dependents);
+    do_update_dependents(additional_dependents);
 }
 
 }

src/Access/AccessBackup.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include <Common/Logger.h>
 #include <Core/UUID.h>
 #include <unordered_map>
-#include <unordered_set>
 
 
 namespace DB
@@ -12,6 +12,7 @@ enum class AccessEntityType : uint8_t;
 struct IAccessEntity;
 using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
 class AccessRightsElements;
+class IAccessStorage;
 class IBackup;
 using BackupPtr = std::shared_ptr<const IBackup>;
 class IBackupEntry;
@@ -20,15 +21,42 @@ struct RestoreSettings;
 enum class RestoreAccessCreationMode : uint8_t;
 
 
-/// Makes a backup of access entities of a specified type.
-std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
-    const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
-    const String & data_path_in_backup,
-    size_t counter,
-    const AccessControl & access_control);
+/// Makes a backup entry for of a set of access entities.
+std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
+    const std::vector<UUID> & entities_ids,
+    const std::unordered_map<UUID, AccessEntityPtr> & all_entities,
+    bool write_dependents,
+    const String & data_path_in_backup);
 
+struct AccessEntitiesToRestore
+{
+    /// Access entities loaded from backup with new randomly generated UUIDs.
+    std::vector<std::pair<UUID /* new_id */, AccessEntityPtr /* new_entity */>> new_entities;
+
+    /// Dependents are access entities which exist already and they should be updated after restoring.
+    /// For example, if there were a role granted to a user: `CREATE USER user1; CREATE ROLE role1; GRANT role1 TO user1`,
+    /// and we're restoring only role `role1` because user `user1` already exists,
+    /// then user `user1` should be modified after restoring role `role1` to add this grant `GRANT role1 TO user1`.
+    struct Dependent
+    {
+        /// UUID of an existing access entities.
+        UUID existing_id;
+
+        /// Source access entity from backup to copy dependencies from.
+        AccessEntityPtr source;
+    };
+    using Dependents = std::vector<Dependent>;
+    Dependents dependents;
+};
 
 /// Restores access entities from a backup.
+void restoreAccessEntitiesFromBackup(
+    IAccessStorage & access_storage,
+    const AccessEntitiesToRestore & entities_to_restore,
+    const RestoreSettings & restore_settings);
+
+
+/// Loads access entities from a backup and prepares them for insertion into an access storage.
 class AccessRestorerFromBackup
 {
 public:
@@ -36,21 +64,75 @@ public:
     ~AccessRestorerFromBackup();
 
     /// Adds a data path to loads access entities from.
-    void addDataPath(const String & data_path);
+    void addDataPath(const String & data_path_in_backup);
+
+    /// Loads access entities from the backup.
+    void loadFromBackup();
 
     /// Checks that the current user can do restoring.
+    /// Function loadFromBackup() must be called before that.
     AccessRightsElements getRequiredAccess() const;
 
-    /// Inserts all access entities loaded from all the paths added by addDataPath().
-    std::vector<std::pair<UUID, AccessEntityPtr>> getAccessEntities(const AccessControl & access_control) const;
+    /// Generates random IDs for access entities we're restoring to insert them into an access storage;
+    /// and finds IDs of existing access entities which are used as dependencies.
+    void generateRandomIDsAndResolveDependencies(const AccessControl & access_control);
+
+    /// Returns access entities loaded from backup and prepared for insertion into an access storage.
+    /// Both functions loadFromBackup() and generateRandomIDsAndResolveDependencies() must be called before that.
+    AccessEntitiesToRestore getEntitiesToRestore(const String & data_path_in_backup) const;
 
 private:
-    BackupPtr backup;
-    RestoreAccessCreationMode creation_mode;
-    bool allow_unresolved_dependencies = false;
-    std::vector<std::pair<UUID, AccessEntityPtr>> entities;
-    std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
-    std::unordered_set<String> data_paths;
+    const BackupPtr backup;
+    const RestoreAccessCreationMode creation_mode;
+    const bool skip_unresolved_dependencies;
+    const bool update_dependents;
+    const LoggerPtr log;
+
+    /// Whether loadFromBackup() finished.
+    bool loaded = false;
+
+    /// Whether generateRandomIDsAndResolveDependencies() finished.
+    bool ids_assigned = false;
+
+    Strings data_paths_in_backup;
+    String data_path_with_entities_to_restore;
+
+    /// Information about an access entity loaded from the backup.
+    struct EntityInfo
+    {
+        UUID id;
+        String name;
+        AccessEntityType type;
+
+        AccessEntityPtr entity = nullptr; /// Can be nullptr if `restore=false`.
+
+        /// Index in `data_paths_in_backup`.
+        size_t data_path_index = 0;
+
+        /// Whether we're going to restore this entity.
+        /// For example,
+        /// in case of `RESTORE TABLE system.roles` this flag is true for all the roles loaded from the backup, and
+        /// in case of `RESTORE ALL` this flag is always true.
+        bool restore = false;
+
+        /// Whether this entity info was added as a dependency of another entity which we're going to restore.
+        /// For example, if we're going to restore the following user: `CREATE USER user1 DEFAULT ROLE role1, role2 SETTINGS PROFILE profile1, profile2`
+        /// then `restore=true` for `user1` and `is_dependency=true` for `role1`, `role2`, `profile1`, `profile2`.
+        /// Flags `restore` and `is_dependency` both can be set at the same time.
+        bool is_dependency = false;
+
+        /// Whether this entity info is a dependent of another entity which we're going to restore.
+        /// For example, if we're going to restore role `role1` and there is also the following user stored in the backup:
+        /// `CREATE USER user1 DEFAULT ROLE role1`, then `is_dependent=true` for `user1`.
+        /// This flags is set by generateRandomIDsAndResolveDependencies().
+        bool is_dependent = false;
+
+        /// New UUID for this entity - either randomly generated or copied from an existing entity.
+        /// This UUID is assigned by generateRandomIDsAndResolveDependencies().
+        std::optional<UUID> new_id = std::nullopt;
+    };
+
+    std::unordered_map<UUID, EntityInfo> entity_infos;
 };
 
 }

src/Access/AccessControl.cpp

@@ -623,15 +623,15 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po
         /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons,
         /// only the log will show the exact reason.
         throw Exception(PreformattedMessage{message.str(),
-                                            "{}: Authentication failed: password is incorrect, or there is no user with such name.{}",
+                                            "{}: Authentication failed: password is incorrect, or there is no user with such name",
                                             std::vector<std::string>{credentials.getUserName()}},
                         ErrorCodes::AUTHENTICATION_FAILED);
     }
 }
 
-void AccessControl::restoreFromBackup(RestorerFromBackup & restorer)
+void AccessControl::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup)
 {
-    MultipleAccessStorage::restoreFromBackup(restorer);
+    MultipleAccessStorage::restoreFromBackup(restorer, data_path_in_backup);
     changes_notifier->sendNotifications();
 }
 

src/Access/AccessControl.h

@@ -124,7 +124,7 @@ public:
     AuthResult authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const String & forwarded_address) const;
 
     /// Makes a backup of access entities.
-    void restoreFromBackup(RestorerFromBackup & restorer) override;
+    void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) override;
 
     void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);
 

src/Access/AccessRights.cpp

@@ -594,6 +594,14 @@ public:
         optimizeTree();
     }
 
+    void makeDifference(const Node & other)
+    {
+        Node rhs = other;
+        makeDifferenceRec(*this, rhs);
+        flags -= other.flags;
+        optimizeTree();
+    }
+
     ProtoElements getElements() const
     {
         ProtoElements res;
@@ -1109,6 +1117,32 @@ private:
         result.wildcard_grant = wildcard_grant || rhs.wildcard_grant;
     }
 
+    void makeDifferenceRec(Node & result, Node & rhs)
+    {
+        if (rhs.children)
+        {
+            for (auto & rhs_child : *rhs.children)
+            {
+                auto & result_child = result.getLeaf(rhs_child.node_name, rhs_child.level, !rhs_child.isLeaf());
+                auto & lhs_child = getLeaf(rhs_child.node_name, rhs_child.level, !rhs_child.isLeaf());
+                lhs_child.makeDifferenceRec(result_child, rhs_child);
+            }
+        }
+
+        if (children)
+        {
+            for (auto & lhs_child : *children)
+            {
+                auto & result_child = result.getLeaf(lhs_child.node_name, lhs_child.level, !lhs_child.isLeaf());
+                auto & rhs_child = rhs.getLeaf(lhs_child.node_name, lhs_child.level, !lhs_child.isLeaf());
+                lhs_child.makeDifferenceRec(result_child, rhs_child);
+            }
+        }
+
+        result.flags = flags - rhs.flags;
+        result.wildcard_grant = wildcard_grant || rhs.wildcard_grant;
+    }
+
     void modifyFlagsRec(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed)
     {
         if (children)
@@ -1581,6 +1615,22 @@ void AccessRights::makeIntersection(const AccessRights & other)
 }
 
 
+void AccessRights::makeDifference(const AccessRights & other)
+{
+    auto helper = [](std::unique_ptr<Node> & root_node, const std::unique_ptr<Node> & other_root_node)
+    {
+        if (!root_node || !other_root_node)
+            return;
+
+        root_node->makeDifference(*other_root_node);
+        if (!root_node->flags && !root_node->children)
+            root_node = nullptr;
+    };
+    helper(root, other.root);
+    helper(root_with_grant_option, other.root_with_grant_option);
+}
+
+
 void AccessRights::modifyFlags(const ModifyFlagsFunction & function)
 {
     if (!root)

src/Access/AccessRights.h

@@ -150,6 +150,9 @@ public:
     /// Makes an intersection of access rights.
     void makeIntersection(const AccessRights & other);
 
+    /// Makes a difference (relative complement) of access rights.
+    void makeDifference(const AccessRights & other);
+
     /// Traverse the tree and modify each access flags.
     using ModifyFlagsFunction = std::function<AccessFlags(
         const AccessFlags & flags,

src/Access/Common/AccessRightsElement.cpp

@@ -233,11 +233,6 @@ void AccessRightsElement::replaceEmptyDatabase(const String & current_database)
 
 String AccessRightsElement::toString() const { return toStringImpl(*this, true); }
 String AccessRightsElement::toStringWithoutOptions() const { return toStringImpl(*this, false); }
-String AccessRightsElement::toStringForAccessTypeSource() const
-{
-    String result{access_flags.toKeywords().front()};
-    return result + " ON *.*";
-}
 
 bool AccessRightsElements::empty() const { return std::all_of(begin(), end(), [](const AccessRightsElement & e) { return e.empty(); }); }
 

src/Access/Common/AccessRightsElement.h

@@ -92,7 +92,6 @@ struct AccessRightsElement
     /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table".
     String toString() const;
     String toStringWithoutOptions() const;
-    String toStringForAccessTypeSource() const;
 
     void formatColumnNames(WriteBuffer & buffer) const;
     void formatONClause(WriteBuffer & buffer, bool hilite = false) const;

src/Access/ContextAccess.cpp

@@ -645,9 +645,25 @@ bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlag
                     AccessRightsElement{access_flags, fmt_args...}.toStringWithoutOptions());
             }
 
+            AccessRights difference;
+            difference.grant(flags, fmt_args...);
+            AccessRights original_rights = difference;
+            difference.makeDifference(*getAccessRights());
+
+            if (difference == original_rights)
+            {
+                return access_denied(ErrorCodes::ACCESS_DENIED,
+                    "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}",
+                    AccessRightsElement{access_flags, fmt_args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""));
+            }
+
+
             return access_denied(ErrorCodes::ACCESS_DENIED,
-                "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}",
-                AccessRightsElement{access_flags, fmt_args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""));
+                "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}. "
+                "(Missing permissions: {}){}",
+                AccessRightsElement{access_flags, fmt_args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""),
+                difference.getElements().toStringWithoutOptions(),
+                grant_option ? ". You can try to use the `GRANT CURRENT GRANTS(...)` statement" : "");
         };
 
         /// As we check the SOURCES from the Table Engine logic, direct prompt about Table Engine would be misleading
@@ -671,18 +687,7 @@ bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlag
             if (new_flags.isEmpty())
                 return access_denied_no_grant(flags, args...);
 
-            if (grant_option && acs->isGranted(flags, args...))
-            {
-                return access_denied(ErrorCodes::ACCESS_DENIED,
-                    "{}: Not enough privileges. "
-                    "The required privileges have been granted, but without grant option. "
-                    "To execute this query, it's necessary to have the grant {} WITH GRANT OPTION",
-                    AccessRightsElement{new_flags}.toStringForAccessTypeSource());
-            }
-
-            return access_denied(ErrorCodes::ACCESS_DENIED,
-                "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}",
-                AccessRightsElement{new_flags}.toStringForAccessTypeSource() + (grant_option ? " WITH GRANT OPTION" : ""));
+            return access_denied_no_grant(new_flags);
         }
 
         return access_denied_no_grant(flags, args...);

src/Access/DiskAccessStorage.cpp

@@ -676,7 +676,7 @@ bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_
     if (!entry.entity)
         entry.entity = readAccessEntityFromDisk(id);
     auto old_entity = entry.entity;
-    auto new_entity = update_func(old_entity);
+    auto new_entity = update_func(old_entity, id);
 
     if (!new_entity->isTypeOf(old_entity->getType()))
         throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());

src/Access/GrantedRoles.cpp

@@ -176,6 +176,16 @@ std::vector<UUID> GrantedRoles::findDependencies() const
     return res;
 }
 
+bool GrantedRoles::hasDependencies(const std::unordered_set<UUID> & ids) const
+{
+    for (const auto & role_id : roles)
+    {
+        if (ids.contains(role_id))
+            return true;
+    }
+    return false;
+}
+
 void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
 {
     std::vector<UUID> new_ids;
@@ -221,4 +231,56 @@ void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & ol
     }
 }
 
+void GrantedRoles::copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set<UUID> & ids)
+{
+    bool found = false;
+
+    for (const auto & role_id : src.roles)
+    {
+        if (ids.contains(role_id))
+        {
+            roles.emplace(role_id);
+            found = true;
+        }
+    }
+
+    if (found)
+    {
+        for (const auto & role_id : src.roles_with_admin_option)
+        {
+            if (ids.contains(role_id))
+                roles_with_admin_option.emplace(role_id);
+        }
+    }
+}
+
+void GrantedRoles::removeDependencies(const std::unordered_set<UUID> & ids)
+{
+    bool found = false;
+
+    for (auto it = roles.begin(); it != roles.end();)
+    {
+        if (ids.contains(*it))
+        {
+            it = roles.erase(it);
+            found = true;
+        }
+        else
+        {
+            ++it;
+        }
+    }
+
+    if (found)
+    {
+        for (auto it = roles_with_admin_option.begin(); it != roles_with_admin_option.end();)
+        {
+            if (ids.contains(*it))
+                it = roles_with_admin_option.erase(it);
+            else
+                ++it;
+        }
+    }
+}
+
 }

src/Access/GrantedRoles.h

@@ -58,7 +58,10 @@ public:
     friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); }
 
     std::vector<UUID> findDependencies() const;
+    bool hasDependencies(const std::unordered_set<UUID> & ids) const;
     void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
+    void copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set<UUID> & ids);
+    void removeDependencies(const std::unordered_set<UUID> & ids);
 
 private:
     boost::container::flat_set<UUID> roles;

src/Access/IAccessEntity.cpp

@@ -9,28 +9,4 @@ bool IAccessEntity::equal(const IAccessEntity & other) const
     return (name == other.name) && (getType() == other.getType());
 }
 
-void IAccessEntity::replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids)
-{
-    if (old_to_new_ids.empty())
-        return;
-
-    bool need_replace_dependencies = false;
-    auto dependencies = entity->findDependencies();
-    for (const auto & dependency : dependencies)
-    {
-        if (old_to_new_ids.contains(dependency))
-        {
-            need_replace_dependencies = true;
-            break;
-        }
-    }
-
-    if (!need_replace_dependencies)
-        return;
-
-    auto new_entity = entity->clone();
-    new_entity->replaceDependencies(old_to_new_ids);
-    entity = new_entity;
-}
-
 }

src/Access/IAccessEntity.h

@@ -48,10 +48,13 @@ struct IAccessEntity
 
     /// Finds all dependencies.
     virtual std::vector<UUID> findDependencies() const { return {}; }
+    virtual bool hasDependencies(const std::unordered_set<UUID> & /* ids */) const { return false; }
 
     /// Replaces dependencies according to a specified map.
-    void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) { doReplaceDependencies(old_to_new_ids); }
-    static void replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids);
+    virtual void replaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
+    virtual void copyDependenciesFrom(const IAccessEntity & /* src */, const std::unordered_set<UUID> & /* ids */) {}
+    virtual void removeDependencies(const std::unordered_set<UUID> & /* ids */) {}
+    virtual void clearAllExceptDependencies() {}
 
     /// Whether this access entity should be written to a backup.
     virtual bool isBackupAllowed() const { return false; }
@@ -67,8 +70,6 @@ protected:
     {
         return std::make_shared<EntityClassT>(typeid_cast<const EntityClassT &>(*this));
     }
-
-    virtual void doReplaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
 };
 
 using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;

src/Access/IAccessStorage.cpp

@@ -4,8 +4,10 @@
 #include <Access/User.h>
 #include <Access/AccessBackup.h>
 #include <Backups/BackupEntriesCollector.h>
-#include <Backups/RestorerFromBackup.h>
+#include <Backups/IBackupCoordination.h>
+#include <Backups/IRestoreCoordination.h>
 #include <Backups/RestoreSettings.h>
+#include <Backups/RestorerFromBackup.h>
 #include <Common/Exception.h>
 #include <Common/quoteString.h>
 #include <Common/callOnce.h>
@@ -14,6 +16,7 @@
 #include <Poco/UUIDGenerator.h>
 #include <Poco/Logger.h>
 #include <base/FnTraits.h>
+#include <base/range.h>
 #include <boost/algorithm/string/join.hpp>
 #include <boost/algorithm/string/replace.hpp>
 #include <boost/range/adaptor/map.hpp>
@@ -71,6 +74,18 @@ std::vector<UUID> IAccessStorage::find(AccessEntityType type, const Strings & na
 }
 
 
+std::vector<UUID> IAccessStorage::findAllImpl() const
+{
+    std::vector<UUID> res;
+    for (auto type : collections::range(AccessEntityType::MAX))
+    {
+        auto ids = findAllImpl(type);
+        res.insert(res.end(), ids.begin(), ids.end());
+    }
+    return res;
+}
+
+
 UUID IAccessStorage::getID(AccessEntityType type, const String & name) const
 {
     auto id = findImpl(type, name);
@@ -598,67 +613,59 @@ void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, c
     if (!isBackupAllowed())
         throwBackupNotAllowed();
 
-    auto entities = readAllWithIDs(type);
-    std::erase_if(entities, [](const std::pair<UUID, AccessEntityPtr> & x) { return !x.second->isBackupAllowed(); });
-
-    if (entities.empty())
+    auto entities_ids = findAll(type);
+    if (entities_ids.empty())
         return;
 
-    auto backup_entry = makeBackupEntryForAccess(
-        entities,
-        data_path_in_backup,
-        backup_entries_collector.getAccessCounter(type),
-        backup_entries_collector.getContext()->getAccessControl());
+    auto backup_entry_with_path = makeBackupEntryForAccessEntities(
+        entities_ids,
+        backup_entries_collector.getAllAccessEntities(),
+        backup_entries_collector.getBackupSettings().write_access_entities_dependents,
+        data_path_in_backup);
 
-    backup_entries_collector.addBackupEntry(backup_entry);
+    if (isReplicated())
+    {
+        auto backup_coordination = backup_entries_collector.getBackupCoordination();
+        auto replication_id = getReplicationID();
+        backup_coordination->addReplicatedAccessFilePath(replication_id, type, backup_entry_with_path.first);
+
+        backup_entries_collector.addPostTask(
+            [backup_entry = backup_entry_with_path.second,
+            replication_id,
+            type,
+            &backup_entries_collector,
+            backup_coordination]
+            {
+                for (const String & path : backup_coordination->getReplicatedAccessFilePaths(replication_id, type))
+                    backup_entries_collector.addBackupEntry(path, backup_entry);
+            });
+    }
+    else
+    {
+        backup_entries_collector.addBackupEntry(backup_entry_with_path);
+    }
 }
 
 
-void IAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
+void IAccessStorage::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup)
 {
     if (!isRestoreAllowed())
         throwRestoreNotAllowed();
 
-    if (isReplicated() && !acquireReplicatedRestore(restorer))
-        return;
-
-    auto entities = restorer.getAccessEntitiesToRestore();
-    if (entities.empty())
-        return;
-
-    auto create_access = restorer.getRestoreSettings().create_access;
-    bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
-    bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
-
-    restorer.addDataRestoreTask([this, entities_to_restore = std::move(entities), replace_if_exists, throw_if_exists] mutable
+    if (isReplicated())
     {
-        std::unordered_map<UUID, UUID> new_to_existing_ids;
-        for (auto & [id, entity] : entities_to_restore)
-        {
-            UUID existing_entity_id;
-            if (!insert(id, entity, replace_if_exists, throw_if_exists, &existing_entity_id))
-            {
-                /// Couldn't insert `entity` because there is an existing entity with the same name.
-                new_to_existing_ids[id] = existing_entity_id;
-            }
-        }
+        auto restore_coordination = restorer.getRestoreCoordination();
+        if (!restore_coordination->acquireReplicatedAccessStorage(getReplicationID()))
+            return;
+    }
 
-        if (!new_to_existing_ids.empty())
+    restorer.addDataRestoreTask(
+        [this, &restorer, data_path_in_backup]
         {
-            /// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed,
-            /// then we should correct those dependencies.
-            auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
-            {
-                auto res = entity;
-                IAccessEntity::replaceDependencies(res, new_to_existing_ids);
-                return res;
-            };
-            std::vector<UUID> ids;
-            ids.reserve(entities_to_restore.size());
-            boost::copy(entities_to_restore | boost::adaptors::map_keys, std::back_inserter(ids));
-            tryUpdate(ids, update_func);
-        }
-    });
+            auto entities_to_restore = restorer.getAccessEntitiesToRestore(data_path_in_backup);
+            const auto & restore_settings = restorer.getRestoreSettings();
+            restoreAccessEntitiesFromBackup(*this, entities_to_restore, restore_settings);
+        });
 }
 
 

src/Access/IAccessStorage.h

@@ -66,6 +66,7 @@ public:
 
     /// Returns true if this storage is replicated.
     virtual bool isReplicated() const { return false; }
+    virtual String getReplicationID() const { return ""; }
 
     /// Starts periodic reloading and updating of entities in this storage.
     virtual void startPeriodicReloading() {}
@@ -90,8 +91,9 @@ public:
     /// Returns the identifiers of all the entities of a specified type contained in the storage.
     std::vector<UUID> findAll(AccessEntityType type) const;
 
-    template <typename EntityClassT>
-    std::vector<UUID> findAll() const { return findAll(EntityClassT::TYPE); }
+    /// Returns the identifiers of all the entities in the storage.
+    template <typename EntityClassT = IAccessEntity>
+    std::vector<UUID> findAll() const;
 
     /// Searches for an entity with specified type and name. Returns std::nullopt if not found.
     std::optional<UUID> find(AccessEntityType type, const String & name) const;
@@ -148,7 +150,7 @@ public:
     std::optional<std::pair<String, AccessEntityType>> tryReadNameWithType(const UUID & id) const;
 
     /// Reads all entities and returns them with their IDs.
-    template <typename EntityClassT>
+    template <typename EntityClassT = IAccessEntity>
     std::vector<std::pair<UUID, std::shared_ptr<const EntityClassT>>> readAllWithIDs() const;
 
     std::vector<std::pair<UUID, AccessEntityPtr>> readAllWithIDs(AccessEntityType type) const;
@@ -180,7 +182,7 @@ public:
     /// Removes multiple entities from the storage. Returns the list of successfully dropped.
     std::vector<UUID> tryRemove(const std::vector<UUID> & ids);
 
-    using UpdateFunc = std::function<AccessEntityPtr(const AccessEntityPtr &)>;
+    using UpdateFunc = std::function<AccessEntityPtr(const AccessEntityPtr &, const UUID &)>;
 
     /// Updates an entity stored in the storage. Throws an exception if couldn't update.
     bool update(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists = true);
@@ -214,11 +216,12 @@ public:
 
     /// Makes a backup of this access storage.
     virtual void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const;
-    virtual void restoreFromBackup(RestorerFromBackup & restorer);
+    virtual void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup);
 
 protected:
     virtual std::optional<UUID> findImpl(AccessEntityType type, const String & name) const = 0;
     virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
+    virtual std::vector<UUID> findAllImpl() const;
     virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
     virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const;
     virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id);
@@ -267,6 +270,16 @@ private:
 };
 
 
+template <typename EntityClassT>
+std::vector<UUID> IAccessStorage::findAll() const
+{
+    if constexpr (std::is_same_v<EntityClassT, IAccessEntity>)
+        return findAllImpl();
+    else
+        return findAllImpl(EntityClassT::TYPE);
+}
+
+
 template <typename EntityClassT>
 std::shared_ptr<const EntityClassT> IAccessStorage::read(const UUID & id, bool throw_if_not_exists) const
 {

src/Access/LDAPAccessStorage.cpp

@@ -163,7 +163,7 @@ void LDAPAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id,
     // Update the granted roles of the relevant users.
     if (!user_ids.empty())
     {
-        auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_) -> AccessEntityPtr
+        auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr
         {
             if (auto user = typeid_cast<std::shared_ptr<const User>>(entity_))
             {
@@ -301,7 +301,7 @@ void LDAPAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String
     if (it != external_role_hashes.end() && it->second == external_roles_hash)
         return;
 
-    auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_) -> AccessEntityPtr
+    auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr
     {
         if (auto user = typeid_cast<std::shared_ptr<const User>>(entity_))
         {

src/Access/MemoryAccessStorage.cpp

@@ -204,7 +204,7 @@ bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat
 
     Entry & entry = it->second;
     auto old_entity = entry.entity;
-    auto new_entity = update_func(old_entity);
+    auto new_entity = update_func(old_entity, id);
 
     if (!new_entity->isTypeOf(old_entity->getType()))
         throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());

src/Access/MultipleAccessStorage.cpp

@@ -416,7 +416,7 @@ bool MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & updat
     {
         if (auto old_entity = storage_for_updating->tryRead(id))
         {
-            auto new_entity = update_func(old_entity);
+            auto new_entity = update_func(old_entity, id);
             if (new_entity->getName() != old_entity->getName())
             {
                 for (const auto & storage : *storages)
@@ -508,7 +508,7 @@ void MultipleAccessStorage::backup(BackupEntriesCollector & backup_entries_colle
         throwBackupNotAllowed();
 }
 
-void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
+void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup)
 {
     auto storages = getStoragesInternal();
 
@@ -516,7 +516,7 @@ void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
     {
         if (storage->isRestoreAllowed())
         {
-            storage->restoreFromBackup(restorer);
+            storage->restoreFromBackup(restorer, data_path_in_backup);
             return;
         }
     }

src/Access/MultipleAccessStorage.h

@@ -59,7 +59,7 @@ public:
     bool isBackupAllowed() const override;
     bool isRestoreAllowed() const override;
     void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override;
-    void restoreFromBackup(RestorerFromBackup & restorer) override;
+    void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) override;
     bool containsStorage(std::string_view storage_type) const;
 
 protected:

src/Access/Quota.cpp

@@ -24,9 +24,33 @@ std::vector<UUID> Quota::findDependencies() const
     return to_roles.findDependencies();
 }
 
-void Quota::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
+bool Quota::hasDependencies(const std::unordered_set<UUID> & ids) const
+{
+    return to_roles.hasDependencies(ids);
+}
+
+void Quota::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
 {
     to_roles.replaceDependencies(old_to_new_ids);
 }
 
+void Quota::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
+{
+    if (getType() != src.getType())
+        return;
+    const auto & src_quota = typeid_cast<const Quota &>(src);
+    to_roles.copyDependenciesFrom(src_quota.to_roles, ids);
+}
+
+void Quota::removeDependencies(const std::unordered_set<UUID> & ids)
+{
+    to_roles.removeDependencies(ids);
+}
+
+void Quota::clearAllExceptDependencies()
+{
+    all_limits.clear();
+    key_type = QuotaKeyType::NONE;
+}
+
 }

src/Access/Quota.h

@@ -47,7 +47,12 @@ struct Quota : public IAccessEntity
     AccessEntityType getType() const override { return TYPE; }
 
     std::vector<UUID> findDependencies() const override;
-    void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
+    void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
+    void removeDependencies(const std::unordered_set<UUID> & ids) override;
+    void clearAllExceptDependencies() override;
+
     bool isBackupAllowed() const override { return true; }
 };
 

src/Access/ReplicatedAccessStorage.cpp

@@ -4,10 +4,6 @@
 #include <Access/ReplicatedAccessStorage.h>
 #include <Access/AccessChangesNotifier.h>
 #include <Access/AccessBackup.h>
-#include <Backups/BackupEntriesCollector.h>
-#include <Backups/IBackupCoordination.h>
-#include <Backups/IRestoreCoordination.h>
-#include <Backups/RestorerFromBackup.h>
 #include <IO/ReadHelpers.h>
 #include <Interpreters/Context.h>
 #include <Common/ZooKeeper/KeeperException.h>
@@ -359,7 +355,7 @@ bool ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zooke
     }
 
     const AccessEntityPtr old_entity = deserializeAccessEntity(old_entity_definition, entity_path);
-    const AccessEntityPtr new_entity = update_func(old_entity);
+    const AccessEntityPtr new_entity = update_func(old_entity, id);
 
     if (!new_entity->isTypeOf(old_entity->getType()))
         throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());
@@ -684,44 +680,4 @@ AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if
     return memory_storage.read(id, throw_if_not_exists);
 }
 
-
-void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const
-{
-    if (!isBackupAllowed())
-        throwBackupNotAllowed();
-
-    auto entities = readAllWithIDs(type);
-    std::erase_if(entities, [](const std::pair<UUID, AccessEntityPtr> & x) { return !x.second->isBackupAllowed(); });
-
-    if (entities.empty())
-        return;
-
-    auto backup_entry_with_path = makeBackupEntryForAccess(
-        entities,
-        data_path_in_backup,
-        backup_entries_collector.getAccessCounter(type),
-        backup_entries_collector.getContext()->getAccessControl());
-
-    auto backup_coordination = backup_entries_collector.getBackupCoordination();
-    backup_coordination->addReplicatedAccessFilePath(zookeeper_path, type, backup_entry_with_path.first);
-
-    backup_entries_collector.addPostTask(
-        [backup_entry = backup_entry_with_path.second,
-         my_zookeeper_path = zookeeper_path,
-         type,
-         &backup_entries_collector,
-         backup_coordination]
-        {
-            for (const String & path : backup_coordination->getReplicatedAccessFilePaths(my_zookeeper_path, type))
-                backup_entries_collector.addBackupEntry(path, backup_entry);
-        });
-}
-
-
-bool ReplicatedAccessStorage::acquireReplicatedRestore(RestorerFromBackup & restorer) const
-{
-    auto restore_coordination = restorer.getRestoreCoordination();
-    return restore_coordination->acquireReplicatedAccessStorage(zookeeper_path);
-}
-
 }

src/Access/ReplicatedAccessStorage.h

@@ -26,7 +26,9 @@ public:
     void shutdown() override;
 
     const char * getStorageType() const override { return STORAGE_TYPE; }
+
     bool isReplicated() const override { return true; }
+    String getReplicationID() const override { return zookeeper_path; }
 
     void startPeriodicReloading() override { startWatchingThread(); }
     void stopPeriodicReloading() override { stopWatchingThread(); }
@@ -35,7 +37,6 @@ public:
     bool exists(const UUID & id) const override;
 
     bool isBackupAllowed() const override { return backup_allowed; }
-    void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override;
 
 private:
     String zookeeper_path;
@@ -80,7 +81,6 @@ private:
     std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
     std::vector<UUID> findAllImpl(AccessEntityType type) const override;
     AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
-    bool acquireReplicatedRestore(RestorerFromBackup & restorer) const override;
 
     mutable std::mutex mutex;
     MemoryAccessStorage memory_storage TSA_GUARDED_BY(mutex);

src/Access/Role.cpp

@@ -21,10 +21,36 @@ std::vector<UUID> Role::findDependencies() const
     return res;
 }
 
-void Role::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
+bool Role::hasDependencies(const std::unordered_set<UUID> & ids) const
+{
+    return granted_roles.hasDependencies(ids) || settings.hasDependencies(ids);
+}
+
+void Role::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
 {
     granted_roles.replaceDependencies(old_to_new_ids);
     settings.replaceDependencies(old_to_new_ids);
 }
 
+void Role::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
+{
+    if (getType() != src.getType())
+        return;
+    const auto & src_role = typeid_cast<const Role &>(src);
+    granted_roles.copyDependenciesFrom(src_role.granted_roles, ids);
+    settings.copyDependenciesFrom(src_role.settings, ids);
+}
+
+void Role::removeDependencies(const std::unordered_set<UUID> & ids)
+{
+    granted_roles.removeDependencies(ids);
+    settings.removeDependencies(ids);
+}
+
+void Role::clearAllExceptDependencies()
+{
+    access = {};
+    settings.removeSettingsKeepProfiles();
+}
+
 }

src/Access/Role.h

@@ -21,7 +21,12 @@ struct Role : public IAccessEntity
     AccessEntityType getType() const override { return TYPE; }
 
     std::vector<UUID> findDependencies() const override;
-    void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
+    void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
+    void removeDependencies(const std::unordered_set<UUID> & ids) override;
+    void clearAllExceptDependencies() override;
+
     bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
 };
 

src/Access/RolesOrUsersSet.cpp

@@ -295,6 +295,23 @@ std::vector<UUID> RolesOrUsersSet::findDependencies() const
     return res;
 }
 
+bool RolesOrUsersSet::hasDependencies(const std::unordered_set<UUID> & dependencies_ids) const
+{
+    for (const auto & id : ids)
+    {
+        if (dependencies_ids.contains(id))
+            return true;
+    }
+
+    for (const auto & id : except_ids)
+    {
+        if (dependencies_ids.contains(id))
+            return true;
+    }
+
+    return false;
+}
+
 void RolesOrUsersSet::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
 {
     std::vector<UUID> new_ids;
@@ -337,4 +354,41 @@ void RolesOrUsersSet::replaceDependencies(const std::unordered_map<UUID, UUID> &
     boost::range::copy(new_ids, std::inserter(except_ids, except_ids.end()));
 }
 
+void RolesOrUsersSet::copyDependenciesFrom(const RolesOrUsersSet & src, const std::unordered_set<UUID> & dependencies_ids)
+{
+    if (all != src.all)
+        return;
+
+    for (const auto & id : src.ids)
+    {
+        if (dependencies_ids.contains(id))
+            ids.emplace(id);
+    }
+
+    for (const auto & id : src.except_ids)
+    {
+        if (dependencies_ids.contains(id))
+            except_ids.emplace(id);
+    }
+}
+
+void RolesOrUsersSet::removeDependencies(const std::unordered_set<UUID> & dependencies_ids)
+{
+    for (auto it = ids.begin(); it != ids.end();)
+    {
+        if (dependencies_ids.contains(*it))
+            it = ids.erase(it);
+        else
+            ++it;
+    }
+
+    for (auto it = except_ids.begin(); it != except_ids.end();)
+    {
+        if (dependencies_ids.contains(*it))
+            except_ids.erase(it);
+        else
+            ++it;
+    }
+}
+
 }

src/Access/RolesOrUsersSet.h

@@ -64,7 +64,10 @@ struct RolesOrUsersSet
     friend bool operator !=(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs) { return !(lhs == rhs); }
 
     std::vector<UUID> findDependencies() const;
+    bool hasDependencies(const std::unordered_set<UUID> & dependencies_ids) const;
     void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
+    void copyDependenciesFrom(const RolesOrUsersSet & src, const std::unordered_set<UUID> & dependencies_ids);
+    void removeDependencies(const std::unordered_set<UUID> & dependencies_ids);
 
     bool all = false;
     boost::container::flat_set<UUID> ids;

src/Access/RowPolicy.cpp

@@ -63,9 +63,33 @@ std::vector<UUID> RowPolicy::findDependencies() const
     return to_roles.findDependencies();
 }
 
-void RowPolicy::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
+bool RowPolicy::hasDependencies(const std::unordered_set<UUID> & ids) const
+{
+    return to_roles.hasDependencies(ids);
+}
+
+void RowPolicy::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
 {
     to_roles.replaceDependencies(old_to_new_ids);
 }
 
+void RowPolicy::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
+{
+    if (getType() != src.getType())
+        return;
+    const auto & src_policy = typeid_cast<const RowPolicy &>(src);
+    to_roles.copyDependenciesFrom(src_policy.to_roles, ids);
+}
+
+void RowPolicy::removeDependencies(const std::unordered_set<UUID> & ids)
+{
+    to_roles.removeDependencies(ids);
+}
+
+void RowPolicy::clearAllExceptDependencies()
+{
+    for (auto & filter : filters)
+        filter = {};
+}
+
 }

src/Access/RowPolicy.h

@@ -50,7 +50,12 @@ struct RowPolicy : public IAccessEntity
     AccessEntityType getType() const override { return TYPE; }
 
     std::vector<UUID> findDependencies() const override;
-    void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
+    void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
+    void removeDependencies(const std::unordered_set<UUID> & ids) override;
+    void clearAllExceptDependencies() override;
+
     bool isBackupAllowed() const override { return true; }
 
     /// Which roles or users should use this row policy.

src/Access/SettingsProfile.cpp

@@ -21,10 +21,35 @@ std::vector<UUID> SettingsProfile::findDependencies() const
     return res;
 }
 
-void SettingsProfile::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
+bool SettingsProfile::hasDependencies(const std::unordered_set<UUID> & ids) const
+{
+    return elements.hasDependencies(ids) || to_roles.hasDependencies(ids);
+}
+
+void SettingsProfile::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
 {
     elements.replaceDependencies(old_to_new_ids);
     to_roles.replaceDependencies(old_to_new_ids);
 }
 
+void SettingsProfile::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
+{
+    if (getType() != src.getType())
+        return;
+    const auto & src_profile = typeid_cast<const SettingsProfile &>(src);
+    elements.copyDependenciesFrom(src_profile.elements, ids);
+    to_roles.copyDependenciesFrom(src_profile.to_roles, ids);
+}
+
+void SettingsProfile::removeDependencies(const std::unordered_set<UUID> & ids)
+{
+    elements.removeDependencies(ids);
+    to_roles.removeDependencies(ids);
+}
+
+void SettingsProfile::clearAllExceptDependencies()
+{
+    elements.removeSettingsKeepProfiles();
+}
+
 }

src/Access/SettingsProfile.h

@@ -22,7 +22,12 @@ struct SettingsProfile : public IAccessEntity
     AccessEntityType getType() const override { return TYPE; }
 
     std::vector<UUID> findDependencies() const override;
-    void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
+    void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
+    void removeDependencies(const std::unordered_set<UUID> & ids) override;
+    void clearAllExceptDependencies() override;
+
     bool isBackupAllowed() const override { return elements.isBackupAllowed(); }
 };
 

src/Access/SettingsProfileElement.cpp

@@ -158,6 +158,18 @@ std::vector<UUID> SettingsProfileElements::findDependencies() const
 }
 
 
+bool SettingsProfileElements::hasDependencies(const std::unordered_set<UUID> & ids) const
+{
+    std::vector<UUID> res;
+    for (const auto & element : *this)
+    {
+        if (element.parent_profile && ids.contains(*element.parent_profile))
+            return true;
+    }
+    return false;
+}
+
+
 void SettingsProfileElements::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
 {
     for (auto & element : *this)
@@ -176,6 +188,38 @@ void SettingsProfileElements::replaceDependencies(const std::unordered_map<UUID,
 }
 
 
+void SettingsProfileElements::copyDependenciesFrom(const SettingsProfileElements & src, const std::unordered_set<UUID> & ids)
+{
+    SettingsProfileElements new_elements;
+    for (const auto & element : src)
+    {
+        if (element.parent_profile && ids.contains(*element.parent_profile))
+        {
+            SettingsProfileElement new_element;
+            new_element.parent_profile = *element.parent_profile;
+            new_elements.emplace_back(new_element);
+        }
+    }
+    insert(begin(), new_elements.begin(), new_elements.end());
+}
+
+
+void SettingsProfileElements::removeDependencies(const std::unordered_set<UUID> & ids)
+{
+    std::erase_if(
+        *this, [&](const SettingsProfileElement & element) { return element.parent_profile && ids.contains(*element.parent_profile); });
+}
+
+
+void SettingsProfileElements::removeSettingsKeepProfiles()
+{
+    for (auto & element : *this)
+        element.setting_name.clear();
+
+    std::erase_if(*this, [&](const SettingsProfileElement & element) { return element.setting_name.empty() && !element.parent_profile; });
+}
+
+
 void SettingsProfileElements::merge(const SettingsProfileElements & other)
 {
     insert(end(), other.begin(), other.end());

src/Access/SettingsProfileElement.h

@@ -63,7 +63,12 @@ public:
     std::shared_ptr<ASTSettingsProfileElements> toASTWithNames(const AccessControl & access_control) const;
 
     std::vector<UUID> findDependencies() const;
+    bool hasDependencies(const std::unordered_set<UUID> & ids) const;
     void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
+    void copyDependenciesFrom(const SettingsProfileElements & src, const std::unordered_set<UUID> & ids);
+    void removeDependencies(const std::unordered_set<UUID> & ids);
+
+    void removeSettingsKeepProfiles();
 
     void merge(const SettingsProfileElements & other);
 

src/Access/User.cpp

@@ -49,7 +49,12 @@ std::vector<UUID> User::findDependencies() const
     return res;
 }
 
-void User::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
+bool User::hasDependencies(const std::unordered_set<UUID> & ids) const
+{
+    return default_roles.hasDependencies(ids) || granted_roles.hasDependencies(ids) || grantees.hasDependencies(ids) || settings.hasDependencies(ids);
+}
+
+void User::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
 {
     default_roles.replaceDependencies(old_to_new_ids);
     granted_roles.replaceDependencies(old_to_new_ids);
@@ -57,4 +62,33 @@ void User::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_n
     settings.replaceDependencies(old_to_new_ids);
 }
 
+void User::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
+{
+    if (getType() != src.getType())
+        return;
+    const auto & src_user = typeid_cast<const User &>(src);
+    default_roles.copyDependenciesFrom(src_user.default_roles, ids);
+    granted_roles.copyDependenciesFrom(src_user.granted_roles, ids);
+    grantees.copyDependenciesFrom(src_user.grantees, ids);
+    settings.copyDependenciesFrom(src_user.settings, ids);
+}
+
+void User::removeDependencies(const std::unordered_set<UUID> & ids)
+{
+    default_roles.removeDependencies(ids);
+    granted_roles.removeDependencies(ids);
+    grantees.removeDependencies(ids);
+    settings.removeDependencies(ids);
+}
+
+void User::clearAllExceptDependencies()
+{
+    authentication_methods.clear();
+    allowed_client_hosts = AllowedClientHosts::AnyHostTag{};
+    access = {};
+    settings.removeSettingsKeepProfiles();
+    default_database = {};
+    valid_until = 0;
+}
+
 }

src/Access/User.h

@@ -32,7 +32,12 @@ struct User : public IAccessEntity
     void setName(const String & name_) override;
 
     std::vector<UUID> findDependencies() const override;
-    void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
+    void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
+    void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
+    void removeDependencies(const std::unordered_set<UUID> & ids) override;
+    void clearAllExceptDependencies() override;
+
     bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
 };