Merge branch 'master' into default_enable_job_stack_trace

2024-11-21 23:21:59 +00:00 · 2024-10-28 10:14:49 -04:00 · 2024-10-28 10:14:49 -04:00 · 91d9878eb5
commit 91d9878eb5
parent 8f9d577c45 3e6b62f420
263 changed files with 3620 additions and 1510 deletions
--- a/ci/docker/fasttest/Dockerfile
+++ b/ci/docker/fasttest/Dockerfile
@ -33,6 +33,8 @@ RUN apt-get update \
 # moreutils - provides ts fo FT
 # expect, bzip2 - requried by FT
 # bsdmainutils - provides hexdump for FT
+# nasm - nasm copiler for one of submodules, required from normal build
+# yasm - asssembler for libhdfs3, required from normal build

 RUN apt-get update \
    && apt-get install \
@ -53,6 +55,8 @@ RUN apt-get update \
        pv \
        jq \
        bzip2 \
+        nasm \
+        yasm \
        --yes --no-install-recommends \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
--- a/ci/jobs/build_clickhouse.py
+++ b/ci/jobs/build_clickhouse.py
@ -0,0 +1,102 @@
+import argparse
+
+from praktika.result import Result
+from praktika.settings import Settings
+from praktika.utils import MetaClasses, Shell, Utils
+
+
+class JobStages(metaclass=MetaClasses.WithIter):
+    CHECKOUT_SUBMODULES = "checkout"
+    CMAKE = "cmake"
+    BUILD = "build"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="ClickHouse Build Job")
+    parser.add_argument("BUILD_TYPE", help="Type: <amd|arm_debug|release_sanitizer>")
+    parser.add_argument("--param", help="Optional custom job start stage", default=None)
+    return parser.parse_args()
+
+
+def main():
+
+    args = parse_args()
+
+    stop_watch = Utils.Stopwatch()
+
+    stages = list(JobStages)
+    stage = args.param or JobStages.CHECKOUT_SUBMODULES
+    if stage:
+        assert stage in JobStages, f"--param must be one of [{list(JobStages)}]"
+        print(f"Job will start from stage [{stage}]")
+        while stage in stages:
+            stages.pop(0)
+        stages.insert(0, stage)
+
+    cmake_build_type = "Release"
+    sanitizer = ""
+
+    if "debug" in args.BUILD_TYPE.lower():
+        print("Build type set: debug")
+        cmake_build_type = "Debug"
+
+    if "asan" in args.BUILD_TYPE.lower():
+        print("Sanitizer set: address")
+        sanitizer = "address"
+
+    # if Environment.is_local_run():
+    #     build_cache_type = "disabled"
+    # else:
+    build_cache_type = "sccache"
+
+    current_directory = Utils.cwd()
+    build_dir = f"{Settings.TEMP_DIR}/build"
+
+    res = True
+    results = []
+
+    if res and JobStages.CHECKOUT_SUBMODULES in stages:
+        Shell.check(f"rm -rf {build_dir} && mkdir -p {build_dir}")
+        results.append(
+            Result.create_from_command_execution(
+                name="Checkout Submodules",
+                command=f"git submodule sync --recursive && git submodule init && git submodule update --depth 1 --recursive --jobs {min([Utils.cpu_count(), 20])}",
+            )
+        )
+        res = results[-1].is_ok()
+
+    if res and JobStages.CMAKE in stages:
+        results.append(
+            Result.create_from_command_execution(
+                name="Cmake configuration",
+                command=f"cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE={cmake_build_type} \
+                 -DSANITIZE={sanitizer} -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DENABLE_TESTS=0 \
+                 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr \
+                 -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \
+                 -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 -DCOMPILER_CACHE={build_cache_type} -DENABLE_TESTS=1 \
+                 -DENABLE_BUILD_PROFILING=1 {current_directory}",
+                workdir=build_dir,
+                with_log=True,
+            )
+        )
+        res = results[-1].is_ok()
+
+    if res and JobStages.BUILD in stages:
+        Shell.check("sccache --show-stats")
+        results.append(
+            Result.create_from_command_execution(
+                name="Build ClickHouse",
+                command="ninja clickhouse-bundle clickhouse-odbc-bridge clickhouse-library-bridge",
+                workdir=build_dir,
+                with_log=True,
+            )
+        )
+        Shell.check("sccache --show-stats")
+        Shell.check(f"ls -l {build_dir}/programs/")
+        res = results[-1].is_ok()
+
+    Result.create_from(results=results, stopwatch=stop_watch).finish_job_accordingly()
+
+
+if __name__ == "__main__":
+    main()
--- a/ci/jobs/check_style.py
+++ b/ci/jobs/check_style.py
@ -68,7 +68,7 @@ def check_duplicate_includes(file_path):
 def check_whitespaces(file_paths):
    for file in file_paths:
        exit_code, out, err = Shell.get_res_stdout_stderr(
-            f'./ci_v2/jobs/scripts/check_style/double_whitespaces.pl "{file}"',
+            f'./ci/jobs/scripts/check_style/double_whitespaces.pl "{file}"',
            verbose=False,
        )
        if out or err:
@ -174,7 +174,7 @@ def check_broken_links(path, exclude_paths):

 def check_cpp_code():
    res, out, err = Shell.get_res_stdout_stderr(
-        "./ci_v2/jobs/scripts/check_style/check_cpp.sh"
+        "./ci/jobs/scripts/check_style/check_cpp.sh"
    )
    if err:
        out += err
@ -183,7 +183,7 @@ def check_cpp_code():

 def check_repo_submodules():
    res, out, err = Shell.get_res_stdout_stderr(
-        "./ci_v2/jobs/scripts/check_style/check_submodules.sh"
+        "./ci/jobs/scripts/check_style/check_submodules.sh"
    )
    if err:
        out += err
@ -192,7 +192,7 @@ def check_repo_submodules():

 def check_other():
    res, out, err = Shell.get_res_stdout_stderr(
-        "./ci_v2/jobs/scripts/check_style/checks_to_refactor.sh"
+        "./ci/jobs/scripts/check_style/checks_to_refactor.sh"
    )
    if err:
        out += err
@ -201,7 +201,7 @@ def check_other():

 def check_codespell():
    res, out, err = Shell.get_res_stdout_stderr(
-        "./ci_v2/jobs/scripts/check_style/check_typos.sh"
+        "./ci/jobs/scripts/check_style/check_typos.sh"
    )
    if err:
        out += err
@ -210,7 +210,7 @@ def check_codespell():

 def check_aspell():
    res, out, err = Shell.get_res_stdout_stderr(
-        "./ci_v2/jobs/scripts/check_style/check_aspell.sh"
+        "./ci/jobs/scripts/check_style/check_aspell.sh"
    )
    if err:
        out += err
@ -219,7 +219,7 @@ def check_aspell():

 def check_mypy():
    res, out, err = Shell.get_res_stdout_stderr(
-        "./ci_v2/jobs/scripts/check_style/check-mypy"
+        "./ci/jobs/scripts/check_style/check-mypy"
    )
    if err:
        out += err
@ -228,7 +228,7 @@ def check_mypy():

 def check_pylint():
    res, out, err = Shell.get_res_stdout_stderr(
-        "./ci_v2/jobs/scripts/check_style/check-pylint"
+        "./ci/jobs/scripts/check_style/check-pylint"
    )
    if err:
        out += err
--- a/ci/jobs/fast_test.py
+++ b/ci/jobs/fast_test.py
@ -1,12 +1,13 @@
+import argparse
 import threading
 from pathlib import Path

-from ci_v2.jobs.scripts.functional_tests_results import FTResultsProcessor
-from praktika.environment import Environment
 from praktika.result import Result
 from praktika.settings import Settings
 from praktika.utils import MetaClasses, Shell, Utils

+from ci.jobs.scripts.functional_tests_results import FTResultsProcessor
+

 class ClickHouseProc:
    def __init__(self):
@ -208,11 +209,18 @@ class JobStages(metaclass=MetaClasses.WithIter):
    TEST = "test"


+def parse_args():
+    parser = argparse.ArgumentParser(description="ClickHouse Fast Test Job")
+    parser.add_argument("--param", help="Optional custom job start stage", default=None)
+    return parser.parse_args()
+
+
 def main():
+    args = parse_args()
    stop_watch = Utils.Stopwatch()

    stages = list(JobStages)
-    stage = Environment.LOCAL_RUN_PARAM or JobStages.CHECKOUT_SUBMODULES
+    stage = args.param or JobStages.CHECKOUT_SUBMODULES
    if stage:
        assert stage in JobStages, f"--param must be one of [{list(JobStages)}]"
        print(f"Job will start from stage [{stage}]")
--- a/ci/jobs/scripts/check_style/check_cpp.sh
+++ b/ci/jobs/scripts/check_style/check_cpp.sh
@ -52,26 +52,6 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/n
 # Broken symlinks
 find -L $ROOT_PATH -type l 2>/dev/null | grep -v contrib && echo "^ Broken symlinks found"

-# Duplicated or incorrect setting declarations
-SETTINGS_FILE=$(mktemp)
-ALL_DECLARATION_FILES="
-  $ROOT_PATH/src/Core/Settings.cpp
-  $ROOT_PATH/src/Storages/MergeTree/MergeTreeSettings.cpp
-  $ROOT_PATH/src/Core/FormatFactorySettingsDeclaration.h"
-
-cat $ROOT_PATH/src/Core/Settings.cpp $ROOT_PATH/src/Core/FormatFactorySettingsDeclaration.h | grep "M(" | awk '{print substr($2, 0, length($2) - 1) " Settings" substr($1, 3, length($1) - 3) " SettingsDeclaration" }' | sort | uniq > ${SETTINGS_FILE}
-cat $ROOT_PATH/src/Storages/MergeTree/MergeTreeSettings.cpp | grep "M(" | awk '{print substr($2, 0, length($2) - 1) " MergeTreeSettings" substr($1, 3, length($1) - 3) " SettingsDeclaration" }' | sort | uniq >> ${SETTINGS_FILE}
-
-# Check that if there are duplicated settings (declared in different objects) they all have the same type (it's simpler to validate style with that assert)
-for setting in $(awk '{print $1 " " $2}' ${SETTINGS_FILE} | sed -e 's/MergeTreeSettings//g' -e 's/Settings//g' | sort | uniq | awk '{ print $1 }' | uniq -d);
-do
-    echo "# Found multiple definitions of setting ${setting} with different types: "
-    grep --line-number " ${setting}," ${ALL_DECLARATION_FILES} | awk '{print "    > " $0 }'
-done
-
-# We append all uses of extern found in implementation files to validate them in a single pass and avoid reading the same files over and over
-find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -e "^\s*extern const Settings" -e "^\s**extern const MergeTreeSettings" -T | awk '{print substr($5, 0, length($5) -1) " " $4 " " substr($1, 0, length($1) - 1)}' >> ${SETTINGS_FILE}
-
 # Duplicated or incorrect setting declarations
 bash $ROOT_PATH/utils/check-style/check-settings-style

--- a/ci/praktika/_environment.py
+++ b/ci/praktika/_environment.py
@ -29,9 +29,9 @@ class _Environment(MetaClasses.Serializable):
    INSTANCE_TYPE: str
    INSTANCE_ID: str
    INSTANCE_LIFE_CYCLE: str
+    LOCAL_RUN: bool = False
    PARAMETER: Any = None
    REPORT_INFO: List[str] = dataclasses.field(default_factory=list)
-    LOCAL_RUN_PARAM: str = ""
    name = "environment"

    @classmethod
@ -185,6 +185,9 @@ class _Environment(MetaClasses.Serializable):
        REPORT_URL = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}&name_1={urllib.parse.quote(self.JOB_NAME, safe='')}"
        return REPORT_URL

+    def is_local_run(self):
+        return self.LOCAL_RUN
+

 def _to_object(data):
    if isinstance(data, dict):
--- a/ci/praktika/_settings.py
+++ b/ci/praktika/_settings.py
@ -8,11 +8,7 @@ class _Settings:
    ######################################
    #    Pipeline generation settings    #
    ######################################
-    if Path("./ci_v2").is_dir():
-        # TODO: hack for CH, remove
-        CI_PATH = "./ci_v2"
-    else:
-        CI_PATH = "./ci"
+    CI_PATH = "./ci"
    WORKFLOW_PATH_PREFIX: str = "./.github/workflows"
    WORKFLOWS_DIRECTORY: str = f"{CI_PATH}/workflows"
    SETTINGS_DIRECTORY: str = f"{CI_PATH}/settings"
--- a/ci/praktika/digest.py
+++ b/ci/praktika/digest.py
@ -1,6 +1,8 @@
 import dataclasses
 import hashlib
+import os
 from hashlib import md5
+from pathlib import Path
 from typing import List

 from praktika import Job
@ -37,7 +39,9 @@ class Digest:
            sorted=True,
        )

-        print(f"calc digest: hash_key [{cache_key}], include [{included_files}] files")
+        print(
+            f"calc digest for job [{job_config.name}]: hash_key [{cache_key}], include [{len(included_files)}] files"
+        )
        # Sort files to ensure consistent hash calculation
        included_files.sort()

@ -91,10 +95,18 @@ class Digest:

    @staticmethod
    def _calc_file_digest(file_path, hash_md5):
-        # Calculate MD5 hash
-        with open(file_path, "rb") as f:
+        # Resolve file path if it's a symbolic link
+        resolved_path = file_path
+        if Path(file_path).is_symlink():
+            resolved_path = os.path.realpath(file_path)
+            if not Path(resolved_path).is_file():
+                print(
+                    f"WARNING: No valid file resolved by link {file_path} -> {resolved_path} - skipping digest calculation"
+                )
+                return hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]
+
+        with open(resolved_path, "rb") as f:
            for chunk in iter(lambda: f.read(4096), b""):
                hash_md5.update(chunk)

-        res = hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]
-        return res
+        return hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]
--- a/ci/praktika/hook_html.py
+++ b/ci/praktika/hook_html.py
@ -1,5 +1,8 @@
+import dataclasses
+import json
 import urllib.parse
 from pathlib import Path
+from typing import List

 from praktika._environment import _Environment
 from praktika.gh import GH
@ -8,12 +11,50 @@ from praktika.result import Result, ResultInfo
 from praktika.runtime import RunConfig
 from praktika.s3 import S3
 from praktika.settings import Settings
-from praktika.utils import Utils
+from praktika.utils import Shell, Utils
+
+
+@dataclasses.dataclass
+class GitCommit:
+    date: str
+    message: str
+    sha: str
+
+    @staticmethod
+    def from_json(json_data: str) -> List["GitCommit"]:
+        commits = []
+        try:
+            data = json.loads(json_data)
+
+            commits = [
+                GitCommit(
+                    message=commit["messageHeadline"],
+                    sha=commit["oid"],
+                    date=commit["committedDate"],
+                )
+                for commit in data.get("commits", [])
+            ]
+        except Exception as e:
+            print(
+                f"ERROR: Failed to deserialize commit's data: [{json_data}], ex: [{e}]"
+            )
+
+        return commits


 class HtmlRunnerHooks:
    @classmethod
    def configure(cls, _workflow):
+
+        def _get_pr_commits(pr_number):
+            res = []
+            if not pr_number:
+                return res
+            output = Shell.get_output(f"gh pr view {pr_number}  --json commits")
+            if output:
+                res = GitCommit.from_json(output)
+            return res
+
        # generate pending Results for all jobs in the workflow
        if _workflow.enable_cache:
            skip_jobs = RunConfig.from_fs(_workflow.name).cache_success
@ -62,10 +103,14 @@ class HtmlRunnerHooks:
            or_update_comment_with_substring=f"Workflow [",
        )
        if not (res1 or res2):
-            print(
-                "ERROR: Failed to set both GH commit status and PR comment with Workflow Status, cannot proceed"
+            Utils.raise_with_error(
+                "Failed to set both GH commit status and PR comment with Workflow Status, cannot proceed"
            )
-            raise
+
+        if env.PR_NUMBER:
+            commits = _get_pr_commits(env.PR_NUMBER)
+            # TODO: upload commits data to s3 to visualise it on a report page
+            print(commits)

    @classmethod
    def pre_run(cls, _workflow, _job):
--- a/ci/praktika/json.html
+++ b/ci/praktika/json.html
@ -24,13 +24,15 @@
            margin: 0;
            display: flex;
            flex-direction: column;
-            font-family: monospace, sans-serif;
+            font-family: 'IBM Plex Mono Condensed', monospace, sans-serif;
+            --header-background-color: #f4f4f4;
        }

        body.night-theme {
            --background-color: #1F1F1C;
            --text-color: #fff;
            --tile-background: black;
+            --header-background-color: #1F1F1C;
        }

        #info-container {
@ -50,27 +52,41 @@
            background-color: var(--tile-background);
            padding: 20px;
            box-sizing: border-box;
-            text-align: left;
            font-size: 18px;
+            margin: 0;
+        }
+
+        #status-container a {
+            color: #007bff;
+            text-decoration: underline;
            font-weight: bold;
-            margin: 0; /* Remove margin */
-        }
-
-        #status-container button {
-            display: block; /* Stack buttons vertically */
-            width: 100%; /* Full width of container */
-            padding: 10px;
-            margin-bottom: 10px; /* Space between buttons */
-            background-color: #4CAF50; /* Green background color */
-            color: white;
-            border: none;
-            border-radius: 5px;
-            font-size: 16px;
            cursor: pointer;
+            display: inline-block;
+            margin-top: 5px;
+            margin-left: 20px;
+            padding: 2px 0;
+            font-size: 0.8em;
        }

-        #status-container button:hover {
-            background-color: #45a049; /* Darker green on hover */
+        #status-container a:hover {
+            color: #0056b3;
+            text-decoration: none;
+        }
+
+        .key-value-pair {
+            display: flex;               /* Enable Flexbox for alignment */
+            justify-content: space-between; /* Distribute space between key and value */
+            margin-bottom: 20px;         /* Add space between each pair */
+        }
+
+        .json-key {
+            font-weight: bold;
+        }
+
+        .json-value {
+            font-weight: normal;
+            font-family: 'Source Code Pro', monospace, sans-serif;
+            letter-spacing: -0.5px;
        }

        #result-container {
@ -203,7 +219,7 @@
        }

        th {
-            background-color: #f4f4f4;
+            background-color: var(--header-background-color);
        }

        .status-success {
@ -240,23 +256,6 @@
            color: grey;
            font-weight: bold;
        }
-
-        .json-key {
-            font-weight: bold;
-            margin-top: 10px;
-        }
-
-        .json-value {
-            margin-left: 20px;
-        }
-
-        .json-value a {
-            color: #007bff;
-        }
-
-        .json-value a:hover {
-            text-decoration: underline;
-        }
    </style>
 </head>
 <body>
@ -286,7 +285,6 @@
    // Attach the toggle function to the click event of the icon
    document.getElementById('theme-toggle').addEventListener('click', toggleTheme);

-    // Function to format timestamp to "DD-mmm-YYYY HH:MM:SS.MM"
    function formatTimestamp(timestamp, showDate = true) {
        const date = new Date(timestamp * 1000);
        const day = String(date.getDate()).padStart(2, '0');
@ -304,6 +302,38 @@
            : `${hours}:${minutes}:${seconds}`;
    }

+    function formatDuration(durationInSeconds, detailed = false) {
+        // Check if the duration is empty, null, or not a number
+        if (!durationInSeconds || isNaN(durationInSeconds)) {
+            return '';
+        }
+
+        // Ensure duration is a floating-point number
+        const duration = parseFloat(durationInSeconds);
+
+        if (detailed) {
+            // Format in the detailed format with hours, minutes, and seconds
+            const hours = Math.floor(duration / 3600);
+            const minutes = Math.floor((duration % 3600) / 60);
+            const seconds = Math.floor(duration % 60);
+
+            const formattedHours = hours > 0 ? `${hours}h ` : '';
+            const formattedMinutes = minutes > 0 ? `${minutes}m ` : '';
+            const formattedSeconds = `${String(seconds).padStart(2, '0')}s`;
+
+            return `${formattedHours}${formattedMinutes}${formattedSeconds}`.trim();
+        } else {
+            // Format in the default format with seconds and milliseconds
+            const seconds = Math.floor(duration);
+            const milliseconds = Math.floor((duration % 1) * 1000);
+
+            const formattedSeconds = String(seconds);
+            const formattedMilliseconds = String(milliseconds).padStart(3, '0');
+
+            return `${formattedSeconds}.${formattedMilliseconds}`;
+        }
+    }
+
    // Function to determine status class based on value
    function getStatusClass(status) {
        const lowerStatus = status.toLowerCase();
@ -316,32 +346,13 @@
        return 'status-other';
    }

-    // Function to format duration from seconds to "HH:MM:SS"
-    function formatDuration(durationInSeconds) {
-        // Check if the duration is empty, null, or not a number
-        if (!durationInSeconds || isNaN(durationInSeconds)) {
-            return '';
-        }
-
-        // Ensure duration is a floating-point number
-        const duration = parseFloat(durationInSeconds);
-
-        // Calculate seconds and milliseconds
-        const seconds = Math.floor(duration); // Whole seconds
-        const milliseconds = Math.floor((duration % 1) * 1000); // Convert fraction to milliseconds
-
-        // Format seconds and milliseconds with leading zeros where needed
-        const formattedSeconds = String(seconds);
-        const formattedMilliseconds = String(milliseconds).padStart(3, '0');
-
-        // Return the formatted duration as seconds.milliseconds
-        return `${formattedSeconds}.${formattedMilliseconds}`;
-    }
-
    function addKeyValueToStatus(key, value) {

        const statusContainer = document.getElementById('status-container');

+        let keyValuePair = document.createElement('div');
+        keyValuePair.className = 'key-value-pair';
+
        const keyElement = document.createElement('div');
        keyElement.className = 'json-key';
        keyElement.textContent = key + ':';
@ -350,8 +361,9 @@
        valueElement.className = 'json-value';
        valueElement.textContent = value;

-        statusContainer.appendChild(keyElement);
-        statusContainer.appendChild(valueElement);
+        keyValuePair.appendChild(keyElement)
+        keyValuePair.appendChild(valueElement)
+        statusContainer.appendChild(keyValuePair);
    }

    function addFileButtonToStatus(key, links) {
@ -364,64 +376,68 @@

        const keyElement = document.createElement('div');
        keyElement.className = 'json-key';
-        keyElement.textContent = key + ':';
+        keyElement.textContent = columnSymbols[key] + ':' || key;
        statusContainer.appendChild(keyElement);

        if (Array.isArray(links) && links.length > 0) {
            links.forEach(link => {
-                // const a = document.createElement('a');
-                // a.href = link;
-                // a.textContent = link.split('/').pop();
-                // a.target = '_blank';
-                // statusContainer.appendChild(a);
-                const button = document.createElement('button');
-                button.textContent = link.split('/').pop();
-                button.addEventListener('click', function () {
-                    window.location.href = link;
-                });
-                statusContainer.appendChild(button);
+                const textLink = document.createElement('a');
+                textLink.href = link;
+                textLink.textContent = link.split('/').pop();
+                textLink.target = '_blank';
+                statusContainer.appendChild(textLink);
+                statusContainer.appendChild(document.createElement('br'));
            });
        }
    }

    function addStatusToStatus(status, start_time, duration) {
-        const statusContainer = document.getElementById('status-container');
+        const statusContainer = document.getElementById('status-container')

+        let keyValuePair = document.createElement('div');
+        keyValuePair.className = 'key-value-pair';
        let keyElement = document.createElement('div');
        let valueElement = document.createElement('div');
        keyElement.className = 'json-key';
        valueElement.className = 'json-value';
-        keyElement.textContent = 'status:';
+        keyElement.textContent = columnSymbols['status'] + ':' || 'status:';
        valueElement.classList.add('status-value');
        valueElement.classList.add(getStatusClass(status));
        valueElement.textContent = status;
-        statusContainer.appendChild(keyElement);
-        statusContainer.appendChild(valueElement);
+        keyValuePair.appendChild(keyElement);
+        keyValuePair.appendChild(valueElement);
+        statusContainer.appendChild(keyValuePair);

+        keyValuePair = document.createElement('div');
+        keyValuePair.className = 'key-value-pair';
        keyElement = document.createElement('div');
        valueElement = document.createElement('div');
        keyElement.className = 'json-key';
        valueElement.className = 'json-value';
-        keyElement.textContent = 'start_time:';
+        keyElement.textContent = columnSymbols['start_time'] + ':' || 'start_time:';
        valueElement.textContent = formatTimestamp(start_time);
-        statusContainer.appendChild(keyElement);
-        statusContainer.appendChild(valueElement);
+        keyValuePair.appendChild(keyElement);
+        keyValuePair.appendChild(valueElement);
+        statusContainer.appendChild(keyValuePair);

+        keyValuePair = document.createElement('div');
+        keyValuePair.className = 'key-value-pair';
        keyElement = document.createElement('div');
        valueElement = document.createElement('div');
        keyElement.className = 'json-key';
        valueElement.className = 'json-value';
-        keyElement.textContent = 'duration:';
+        keyElement.textContent = columnSymbols['duration'] + ':' || 'duration:';
        if (duration === null) {
            // Set initial value to 0 and add a unique ID or data attribute to identify the duration element
            valueElement.textContent = '00:00:00';
            valueElement.setAttribute('id', 'duration-value');
        } else {
            // Format the duration if it's a valid number
-            valueElement.textContent = formatDuration(duration);
+            valueElement.textContent = formatDuration(duration, true);
        }
-        statusContainer.appendChild(keyElement);
-        statusContainer.appendChild(valueElement);
+        keyValuePair.appendChild(keyElement);
+        keyValuePair.appendChild(valueElement);
+        statusContainer.appendChild(keyValuePair);
    }

    function navigatePath(jsonObj, nameArray) {
@ -470,11 +486,12 @@
    const columns = ['name', 'status', 'start_time', 'duration', 'info'];

    const columnSymbols = {
-        name: '👤',
+        name: '📂',
        status: '✔️',
        start_time: '🕒',
        duration: '⏳',
-        info: '⚠️'
+        info: 'ℹ️',
+        files: '📄'
    };

    function createResultsTable(results, nest_level) {
@ -626,6 +643,7 @@
                            footerRight.appendChild(a);
                        });
                    }
+
                    addStatusToStatus(targetData.status, targetData.start_time, targetData.duration)

                    // Handle links
@ -639,7 +657,7 @@

                        const intervalId = setInterval(() => {
                            duration++;
-                            durationElement.textContent = formatDuration(duration);
+                            durationElement.textContent = formatDuration(duration, true);
                        }, 1000);
                    }

--- a/ci/praktika/runner.py
+++ b/ci/praktika/runner.py
@ -42,6 +42,7 @@ class Runner:
            INSTANCE_ID="",
            INSTANCE_TYPE="",
            INSTANCE_LIFE_CYCLE="",
+            LOCAL_RUN=True,
        ).dump()
        workflow_config = RunConfig(
            name=workflow.name,
@ -76,9 +77,6 @@ class Runner:
            os.environ[key] = value
            print(f"Set environment variable {key}.")

-        # TODO: remove
-        os.environ["PYTHONPATH"] = os.getcwd()
-
        print("Read GH Environment")
        env = _Environment.from_env()
        env.JOB_NAME = job.name
@ -132,9 +130,7 @@ class Runner:
                    f"Custom param for local tests must be of type str, got [{type(param)}]"
                )
            env = _Environment.get()
-            env.LOCAL_RUN_PARAM = param
            env.dump()
-            print(f"Custom param for local tests [{param}] dumped into Environment")

        if job.run_in_docker and not no_docker:
            # TODO: add support for any image, including not from ci config (e.g. ubuntu:latest)
@ -142,9 +138,13 @@ class Runner:
                job.run_in_docker
            ]
            docker = docker or f"{job.run_in_docker}:{docker_tag}"
-            cmd = f"docker run --rm --user \"$(id -u):$(id -g)\" -e PYTHONPATH='{Settings.DOCKER_WD}' --volume ./:{Settings.DOCKER_WD} --volume {Settings.TEMP_DIR}:{Settings.TEMP_DIR} --workdir={Settings.DOCKER_WD} {docker} {job.command}"
+            cmd = f"docker run --rm --user \"$(id -u):$(id -g)\" -e PYTHONPATH='{Settings.DOCKER_WD}:{Settings.DOCKER_WD}/ci' --volume ./:{Settings.DOCKER_WD} --volume {Settings.TEMP_DIR}:{Settings.TEMP_DIR} --workdir={Settings.DOCKER_WD} {docker} {job.command}"
        else:
            cmd = job.command
+
+        if param:
+            print(f"Custom --param [{param}] will be passed to job's script")
+            cmd += f" --param {param}"
        print(f"--- Run command [{cmd}]")

        with TeePopen(cmd, timeout=job.timeout) as process:
--- a/ci/praktika/utils.py
+++ b/ci/praktika/utils.py
@ -348,9 +348,9 @@ class Utils:
        return multiprocessing.cpu_count()

    @staticmethod
-    def raise_with_error(error_message, stdout="", stderr=""):
+    def raise_with_error(error_message, stdout="", stderr="", ex=None):
        Utils.print_formatted_error(error_message, stdout, stderr)
-        raise
+        raise ex or RuntimeError()

    @staticmethod
    def timestamp():
--- a/ci/praktika/yaml_generator.py
+++ b/ci/praktika/yaml_generator.py
@ -83,8 +83,8 @@ jobs:
 {JOB_ADDONS}
      - name: Prepare env script
        run: |
-          export PYTHONPATH=.:$PYTHONPATH
          cat > {ENV_SETUP_SCRIPT} << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.
 {SETUP_ENVS}
          cat > {WORKFLOW_CONFIG_FILE} << 'EOF'
          ${{{{ needs.{WORKFLOW_CONFIG_JOB_NAME}.outputs.data }}}}
@ -100,6 +100,7 @@ jobs:
      - name: Run
        id: run
        run: |
+          . /tmp/praktika_setup_env.sh
          set -o pipefail
          {PYTHON} -m praktika run --job '''{JOB_NAME}''' --workflow "{WORKFLOW_NAME}" --ci |& tee {RUN_LOG}
 {UPLOADS_GITHUB}\
--- a/ci/settings/definitions.py
+++ b/ci/settings/definitions.py
@ -30,133 +30,133 @@ SECRETS = [
 DOCKERS = [
    # Docker.Config(
    #     name="clickhouse/binary-builder",
-    #     path="./ci_v2/docker/packager/binary-builder",
+    #     path="./ci/docker/packager/binary-builder",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=[],
    # ),
    # Docker.Config(
    #     name="clickhouse/cctools",
-    #     path="./ci_v2/docker/packager/cctools",
+    #     path="./ci/docker/packager/cctools",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=[],
    # ),
    # Docker.Config(
    #     name="clickhouse/test-old-centos",
-    #     path="./ci_v2/docker/test/compatibility/centos",
+    #     path="./ci/docker/test/compatibility/centos",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=[],
    # ),
    # Docker.Config(
    #     name="clickhouse/test-old-ubuntu",
-    #     path="./ci_v2/docker/test/compatibility/ubuntu",
+    #     path="./ci/docker/test/compatibility/ubuntu",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=[],
    # ),
    # Docker.Config(
    #     name="clickhouse/test-util",
-    #     path="./ci_v2/docker/test/util",
+    #     path="./ci/docker/test/util",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=[],
    # ),
    # Docker.Config(
    #     name="clickhouse/integration-test",
-    #     path="./ci_v2/docker/test/integration/base",
+    #     path="./ci/docker/test/integration/base",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/fuzzer",
-    #     path="./ci_v2/docker/test/fuzzer",
+    #     path="./ci/docker/test/fuzzer",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/performance-comparison",
-    #     path="./ci_v2/docker/test/performance-comparison",
+    #     path="./ci/docker/test/performance-comparison",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=[],
    # ),
    Docker.Config(
        name="clickhouse/fasttest",
-        path="./ci_v2/docker/fasttest",
+        path="./ci/docker/fasttest",
        platforms=Docker.Platforms.arm_amd,
        depends_on=[],
    ),
    # Docker.Config(
    #     name="clickhouse/test-base",
-    #     path="./ci_v2/docker/test/base",
+    #     path="./ci/docker/test/base",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-util"],
    # ),
    # Docker.Config(
    #     name="clickhouse/clickbench",
-    #     path="./ci_v2/docker/test/clickbench",
+    #     path="./ci/docker/test/clickbench",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/keeper-jepsen-test",
-    #     path="./ci_v2/docker/test/keeper-jepsen",
+    #     path="./ci/docker/test/keeper-jepsen",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/server-jepsen-test",
-    #     path="./ci_v2/docker/test/server-jepsen",
+    #     path="./ci/docker/test/server-jepsen",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/sqllogic-test",
-    #     path="./ci_v2/docker/test/sqllogic",
+    #     path="./ci/docker/test/sqllogic",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/sqltest",
-    #     path="./ci_v2/docker/test/sqltest",
+    #     path="./ci/docker/test/sqltest",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/stateless-test",
-    #     path="./ci_v2/docker/test/stateless",
+    #     path="./ci/docker/test/stateless",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/stateful-test",
-    #     path="./ci_v2/docker/test/stateful",
+    #     path="./ci/docker/test/stateful",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/stateless-test"],
    # ),
    # Docker.Config(
    #     name="clickhouse/stress-test",
-    #     path="./ci_v2/docker/test/stress",
+    #     path="./ci/docker/test/stress",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/stateful-test"],
    # ),
    # Docker.Config(
    #     name="clickhouse/unit-test",
-    #     path="./ci_v2/docker/test/unit",
+    #     path="./ci/docker/test/unit",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    # Docker.Config(
    #     name="clickhouse/integration-tests-runner",
-    #     path="./ci_v2/docker/test/integration/runner",
+    #     path="./ci/docker/test/integration/runner",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
    Docker.Config(
        name="clickhouse/style-test",
-        path="./ci_v2/docker/style-test",
+        path="./ci/docker/style-test",
        platforms=Docker.Platforms.arm_amd,
        depends_on=[],
    ),
    # Docker.Config(
    #     name="clickhouse/docs-builder",
-    #     path="./ci_v2/docker/docs/builder",
+    #     path="./ci/docker/docs/builder",
    #     platforms=Docker.Platforms.arm_amd,
    #     depends_on=["clickhouse/test-base"],
    # ),
@ -230,3 +230,4 @@ DOCKERS = [
 class JobNames:
    STYLE_CHECK = "Style Check"
    FAST_TEST = "Fast test"
+    BUILD_AMD_DEBUG = "Build amd64 debug"
--- a/ci/settings/settings.py
+++ b/ci/settings/settings.py
@ -1,4 +1,4 @@
-from ci_v2.settings.definitions import (
+from ci.settings.definitions import (
    S3_BUCKET_HTTP_ENDPOINT,
    S3_BUCKET_NAME,
    RunnerLabels,
--- a/ci/workflows/pull_request.py
+++ b/ci/workflows/pull_request.py
@ -1,26 +1,62 @@
 from typing import List

-from ci_v2.settings.definitions import (
+from praktika import Artifact, Job, Workflow
+from praktika.settings import Settings
+
+from ci.settings.definitions import (
    BASE_BRANCH,
    DOCKERS,
    SECRETS,
    JobNames,
    RunnerLabels,
 )
-from praktika import Job, Workflow
+
+
+class ArtifactNames:
+    ch_debug_binary = "clickhouse_debug_binary"
+

 style_check_job = Job.Config(
    name=JobNames.STYLE_CHECK,
    runs_on=[RunnerLabels.CI_SERVICES],
-    command="python3 ./ci_v2/jobs/check_style.py",
+    command="python3 ./ci/jobs/check_style.py",
    run_in_docker="clickhouse/style-test",
 )

 fast_test_job = Job.Config(
    name=JobNames.FAST_TEST,
    runs_on=[RunnerLabels.BUILDER],
-    command="python3 ./ci_v2/jobs/fast_test.py",
+    command="python3 ./ci/jobs/fast_test.py",
    run_in_docker="clickhouse/fasttest",
+    digest_config=Job.CacheDigestConfig(
+        include_paths=[
+            "./ci/jobs/fast_test.py",
+            "./tests/queries/0_stateless/",
+            "./src",
+        ],
+    ),
+)
+
+job_build_amd_debug = Job.Config(
+    name=JobNames.BUILD_AMD_DEBUG,
+    runs_on=[RunnerLabels.BUILDER],
+    command="python3 ./ci/jobs/build_clickhouse.py amd_debug",
+    run_in_docker="clickhouse/fasttest",
+    digest_config=Job.CacheDigestConfig(
+        include_paths=[
+            "./src",
+            "./contrib/",
+            "./CMakeLists.txt",
+            "./PreLoad.cmake",
+            "./cmake",
+            "./base",
+            "./programs",
+            "./docker/packager/packager",
+            "./rust",
+            "./tests/ci/version_helper.py",
+        ],
+    ),
+    provides=[ArtifactNames.ch_debug_binary],
 )

 workflow = Workflow.Config(
@ -30,6 +66,14 @@ workflow = Workflow.Config(
    jobs=[
        style_check_job,
        fast_test_job,
+        job_build_amd_debug,
+    ],
+    artifacts=[
+        Artifact.Config(
+            name=ArtifactNames.ch_debug_binary,
+            type=Artifact.Type.S3,
+            path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
+        )
    ],
    dockers=DOCKERS,
    secrets=SECRETS,
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@ -2,11 +2,11 @@

 # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54491)
+SET(VERSION_REVISION 54492)
 SET(VERSION_MAJOR 24)
-SET(VERSION_MINOR 10)
+SET(VERSION_MINOR 11)
 SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH b12a367741812f9e5fe754d19ebae600e2a2614c)
-SET(VERSION_DESCRIBE v24.10.1.1-testing)
-SET(VERSION_STRING 24.10.1.1)
+SET(VERSION_GITHASH c82cf25b3e5864bcc153cbe45adb8c6527e1ec6e)
+SET(VERSION_DESCRIBE v24.11.1.1-testing)
+SET(VERSION_STRING 24.11.1.1)
 # end of autochange
--- a/contrib/numactl
+++ b/contrib/numactl
@ -1 +1 @@
-Subproject commit 8d13d63a05f0c3cd88bf777cbb61541202b7da08
+Subproject commit ff32c618d63ca7ac48cce366c5a04bb3563683a0
--- a/docs/en/engines/table-engines/integrations/s3.md
+++ b/docs/en/engines/table-engines/integrations/s3.md
@ -331,6 +331,10 @@ CREATE TABLE big_table (name String, value UInt32)
    ENGINE = S3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/aapl_stock.csv', NOSIGN, 'CSVWithNames');
 ```

+## Optimizing performance
+
+For details on optimizing the performance of the s3 function see [our detailed guide](/docs/en/integrations/s3/performance).
+
 ## See also

 - [s3 table function](../../../sql-reference/table-functions/s3.md)
--- a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md
@ -37,7 +37,7 @@ For a description of request parameters, see [request description](../../../sql-

 **Query clauses**

-When creating an `AggregatingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table.
+When creating an `AggregatingMergeTree` table, the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required as when creating a `MergeTree` table.

 <details markdown="1">

@ -62,19 +62,19 @@ All of the parameters have the same meaning as in `MergeTree`.
 ## SELECT and INSERT {#select-and-insert}

 To insert data, use [INSERT SELECT](../../../sql-reference/statements/insert-into.md) query with aggregate -State- functions.
-When selecting data from `AggregatingMergeTree` table, use `GROUP BY` clause and the same aggregate functions as when inserting data, but using `-Merge` suffix.
+When selecting data from `AggregatingMergeTree` table, use `GROUP BY` clause and the same aggregate functions as when inserting data, but using the `-Merge` suffix.

-In the results of `SELECT` query, the values of `AggregateFunction` type have implementation-specific binary representation for all of the ClickHouse output formats. If dump data into, for example, `TabSeparated` format with `SELECT` query then this dump can be loaded back using `INSERT` query.
+In the results of `SELECT` query, the values of `AggregateFunction` type have implementation-specific binary representation for all of the ClickHouse output formats. For example, if you dump data into `TabSeparated` format with a `SELECT` query, then this dump can be loaded back using an `INSERT` query.

 ## Example of an Aggregated Materialized View {#example-of-an-aggregated-materialized-view}

-The following examples assumes that you have a database named `test` so make sure you create that if it doesn't already exist:
+The following example assumes that you have a database named `test`, so create it if it doesn't already exist:

 ```sql
 CREATE DATABASE test;
 ```

-We will create the table `test.visits` that contain the raw data:
+Now create the table `test.visits` that contains the raw data:

 ``` sql
 CREATE TABLE test.visits
@ -86,9 +86,9 @@ CREATE TABLE test.visits
 ) ENGINE = MergeTree ORDER BY (StartDate, CounterID);
 ```

-Next, we need to create an `AggregatingMergeTree` table that will store `AggregationFunction`s that keep track of the total number of visits and the number of unique users. 
+Next, you need an `AggregatingMergeTree` table that will store `AggregationFunction`s that keep track of the total number of visits and the number of unique users. 

-`AggregatingMergeTree` materialized view that watches the `test.visits` table, and use the `AggregateFunction` type:
+Create an `AggregatingMergeTree` materialized view that watches the `test.visits` table, and uses the `AggregateFunction` type:

 ``` sql
 CREATE TABLE test.agg_visits (
@ -100,7 +100,7 @@ CREATE TABLE test.agg_visits (
 ENGINE = AggregatingMergeTree() ORDER BY (StartDate, CounterID);
 ```

-And then let's create a materialized view that populates `test.agg_visits` from `test.visits` :
+Create a materialized view that populates `test.agg_visits` from `test.visits`:

 ```sql
 CREATE MATERIALIZED VIEW test.visits_mv TO test.agg_visits
@ -113,7 +113,7 @@ FROM test.visits
 GROUP BY StartDate, CounterID;
 ```

-Inserting data into the `test.visits` table.
+Insert data into the `test.visits` table:

 ``` sql
 INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)
@ -122,7 +122,7 @@ INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)

 The data is inserted in both `test.visits` and `test.agg_visits`.

-To get the aggregated data, we need to execute a query such as `SELECT ... GROUP BY ...` from the materialized view `test.mv_visits`:
+To get the aggregated data, execute a query such as `SELECT ... GROUP BY ...` from the materialized view `test.mv_visits`:

 ```sql
 SELECT
@ -140,14 +140,14 @@ ORDER BY StartDate;
 └─────────────────────────┴────────┴───────┘
 ```

-And how about if we add another couple of records to `test.visits`, but this time we'll use a different timestamp for one of the records:
+Add another couple of records to `test.visits`, but this time try using a different timestamp for one of the records:

 ```sql
 INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)
 VALUES (1669446031000, 2, 5, 10), (1667446031000, 3, 7, 5);
 ```

-If we then run the `SELECT` query again, we'll see the following output:
+Run the `SELECT` query again, which will return the following output:

 ```text
 ┌───────────────StartDate─┬─Visits─┬─Users─┐
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@ -12,6 +12,10 @@ Data deduplication occurs only during a merge. Merging occurs in the background

 Thus, `ReplacingMergeTree` is suitable for clearing out duplicate data in the background in order to save space, but it does not guarantee the absence of duplicates.

+:::note
+A detailed guide on ReplacingMergeTree, including best practices and how to optimize performance, is available [here](/docs/en/guides/replacing-merge-tree).
+:::
+
 ## Creating a Table {#creating-a-table}

 ``` sql
@ -162,3 +166,51 @@ All of the parameters excepting `ver` have the same meaning as in `MergeTree`.
 - `ver` - column with the version. Optional parameter. For a description, see the text above.

 </details>
+
+## Query time de-duplication & FINAL
+
+At merge time, the ReplacingMergeTree identifies duplicate rows, using the values of the `ORDER BY` columns (used to create the table) as a unique identifier, and retains only the highest version. This, however, offers eventual correctness only - it does not guarantee rows will be deduplicated, and you should not rely on it. Queries can, therefore, produce incorrect answers due to update and delete rows being considered in queries.
+
+To obtain correct answers, users will need to complement background merges with query time deduplication and deletion removal. This can be achieved using the `FINAL` operator. For example, consider the following example:
+
+```sql
+CREATE TABLE rmt_example
+(
+    `number` UInt16
+)
+ENGINE = ReplacingMergeTree
+ORDER BY number
+
+INSERT INTO rmt_example SELECT floor(randUniform(0, 100)) AS number
+FROM numbers(1000000000)
+
+0 rows in set. Elapsed: 19.958 sec. Processed 1.00 billion rows, 8.00 GB (50.11 million rows/s., 400.84 MB/s.)
+```
+Querying without `FINAL` produces an incorrect count (exact result will vary depending on merges):
+
+```sql
+SELECT count()
+FROM rmt_example
+
+┌─count()─┐
+│     200 │
+└─────────┘
+
+1 row in set. Elapsed: 0.002 sec.
+```
+
+Adding final produces a correct result:
+
+```sql
+SELECT count()
+FROM rmt_example
+FINAL
+
+┌─count()─┐
+│     100 │
+└─────────┘
+
+1 row in set. Elapsed: 0.002 sec.
+```
+
+For further details on `FINAL`, including how to optimize `FINAL` performance, we recommend reading our [detailed guide on ReplacingMergeTree](/docs/en/guides/replacing-merge-tree).
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@ -2217,6 +2217,39 @@ If the table does not exist, ClickHouse will create it. If the structure of the
 </query_log>
 ```

+# query_metric_log {#query_metric_log}
+
+It is disabled by default.
+
+**Enabling**
+
+To manually turn on metrics history collection [`system.query_metric_log`](../../operations/system-tables/query_metric_log.md), create `/etc/clickhouse-server/config.d/query_metric_log.xml` with the following content:
+
+``` xml
+<clickhouse>
+    <query_metric_log>
+        <database>system</database>
+        <table>query_metric_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
+        <max_size_rows>1048576</max_size_rows>
+        <reserved_size_rows>8192</reserved_size_rows>
+        <buffer_size_rows_flush_threshold>524288</buffer_size_rows_flush_threshold>
+        <flush_on_crash>false</flush_on_crash>
+    </query_metric_log>
+</clickhouse>
+```
+
+**Disabling**
+
+To disable `query_metric_log` setting, you should create the following file `/etc/clickhouse-server/config.d/disable_query_metric_log.xml` with the following content:
+
+``` xml
+<clickhouse>
+<query_metric_log remove="1" />
+</clickhouse>
+```
+
 ## query_cache {#server_configuration_parameters_query-cache}

 [Query cache](../query-cache.md) configuration.
@ -3109,7 +3142,7 @@ By default, tunneling (i.e, `HTTP CONNECT`) is used to make `HTTPS` requests ove

 ### no_proxy
 By default, all requests will go through the proxy. In order to disable it for specific hosts, the `no_proxy` variable must be set.
-It can be set inside the `<proxy>` clause for list and remote resolvers and as an environment variable for environment resolver. 
+It can be set inside the `<proxy>` clause for list and remote resolvers and as an environment variable for environment resolver.
 It supports IP addresses, domains, subdomains and `'*'` wildcard for full bypass. Leading dots are stripped just like curl does.

 Example:
--- a/docs/en/operations/system-tables/query_metric_log.md
+++ b/docs/en/operations/system-tables/query_metric_log.md
@ -0,0 +1,49 @@
+---
+slug: /en/operations/system-tables/query_metric_log
+---
+# query_metric_log
+
+Contains history of memory and metric values from table `system.events` for individual queries, periodically flushed to disk.
+
+Once a query starts, data is collected at periodic intervals of `query_metric_log_interval` milliseconds (which is set to 1000
+by default). The data is also collected when the query finishes if the query takes longer than `query_metric_log_interval`.
+
+Columns:
+- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the query.
+- `hostname` ([LowCardinality(String)](../../sql-reference/data-types/string.md)) — Hostname of the server executing the query.
+- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
+- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
+- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds resolution.
+
+**Example**
+
+``` sql
+SELECT * FROM system.query_metric_log LIMIT 1 FORMAT Vertical;
+```
+
+``` text
+Row 1:
+──────
+query_id:                                                        97c8ba04-b6d4-4bd7-b13e-6201c5c6e49d
+hostname:                                                        clickhouse.eu-central1.internal
+event_date:                                                      2020-09-05
+event_time:                                                      2020-09-05 16:22:33
+event_time_microseconds:                                         2020-09-05 16:22:33.196807
+memory_usage:                                                    313434219
+peak_memory_usage:                                               598951986
+ProfileEvent_Query:                                              0
+ProfileEvent_SelectQuery:                                        0
+ProfileEvent_InsertQuery:                                        0
+ProfileEvent_FailedQuery:                                        0
+ProfileEvent_FailedSelectQuery:                                  0
+...
+```
+
+**See also**
+
+- [query_metric_log setting](../../operations/server-configuration-parameters/settings.md#query_metric_log) — Enabling and disabling the setting.
+- [query_metric_log_interval](../../operations/settings/settings.md#query_metric_log_interval)
+- [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md) — Contains periodically calculated metrics.
+- [system.events](../../operations/system-tables/events.md#system_tables-events) — Contains a number of events that occurred.
+- [system.metrics](../../operations/system-tables/metrics.md) — Contains instantly calculated metrics.
+- [Monitoring](../../operations/monitoring.md) — Base concepts of ClickHouse monitoring.
--- a/docs/en/sql-reference/aggregate-functions/reference/quantileexactweighted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantileexactweighted.md
@ -23,7 +23,7 @@ Alias: `medianExactWeighted`.

 - `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` value in the range of `[0.01, 0.99]`. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median).
 - `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md).
- `weight` — Column with weights of sequence members. Weight is a number of value occurrences.
+- `weight` — Column with weights of sequence members. Weight is a number of value occurrences with [Unsigned integer types](../../../sql-reference/data-types/int-uint.md).

 **Returned value**

--- a/docs/en/sql-reference/aggregate-functions/reference/quantileexactweightedinterpolated.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantileexactweightedinterpolated.md
@ -0,0 +1,77 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/quantileExactWeightedInterpolated
+sidebar_position: 176
+---
+
+# quantileExactWeightedInterpolated
+
+Computes [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using linear interpolation, taking into account the weight of each element.
+
+To get the interpolated value, all the passed values are combined into an array, which are then sorted by their corresponding weights. Quantile interpolation is then performed using the [weighted percentile method](https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method) by building a cumulative distribution based on weights and then a linear interpolation is performed using the weights and the values to compute the quantiles.
+
+When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function.
+
+We strongly recommend using `quantileExactWeightedInterpolated` instead of `quantileInterpolatedWeighted` because `quantileExactWeightedInterpolated` is more accurate than `quantileInterpolatedWeighted`. Here is an example:
+
+``` sql
+SELECT
+    quantileExactWeightedInterpolated(0.99)(number, 1),
+    quantile(0.99)(number),
+    quantileInterpolatedWeighted(0.99)(number, 1)
+FROM numbers(9)
+
+
+┌─quantileExactWeightedInterpolated(0.99)(number, 1)─┬─quantile(0.99)(number)─┬─quantileInterpolatedWeighted(0.99)(number, 1)─┐
+│                                               7.92 │                   7.92 │                                             8 │
+└────────────────────────────────────────────────────┴────────────────────────┴───────────────────────────────────────────────┘
+```
+
+**Syntax**
+
+``` sql
+quantileExactWeightedInterpolated(level)(expr, weight)
+```
+
+Alias: `medianExactWeightedInterpolated`.
+
+**Arguments**
+
+- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` value in the range of `[0.01, 0.99]`. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median).
+- `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md).
+- `weight` — Column with weights of sequence members. Weight is a number of value occurrences with [Unsigned integer types](../../../sql-reference/data-types/int-uint.md).
+
+**Returned value**
+
+- Quantile of the specified level.
+
+Type:
+
+- [Float64](../../../sql-reference/data-types/float.md) for numeric data type input.
+- [Date](../../../sql-reference/data-types/date.md) if input values have the `Date` type.
+- [DateTime](../../../sql-reference/data-types/datetime.md) if input values have the `DateTime` type.
+
+**Example**
+
+Input table:
+
+``` text
+┌─n─┬─val─┐
+│ 0 │   3 │
+│ 1 │   2 │
+│ 2 │   1 │
+│ 5 │   4 │
+└───┴─────┘
+```
+
+Result:
+
+``` text
+┌─quantileExactWeightedInterpolated(n, val)─┐
+│                                       1.5 │
+└───────────────────────────────────────────┘
+```
+
+**See Also**
+
+- [median](../../../sql-reference/aggregate-functions/reference/median.md#median)
+- [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles)
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md
@ -9,7 +9,7 @@ sidebar_position: 177

 Syntax: `quantiles(level1, level2, ...)(x)`

-All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantileInterpolatedWeighted`, `quantilesTDigest`, `quantilesBFloat16`, `quantilesDD`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
+All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantileExactWeightedInterpolated`, `quantileInterpolatedWeighted`, `quantilesTDigest`, `quantilesBFloat16`, `quantilesDD`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.

 ## quantilesExactExclusive

--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@ -6867,6 +6867,18 @@ Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that

 Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that it returns `NULL` when it encounters a date format that cannot be processed.

+## parseDateTime64InJodaSyntax
+
+Similar to [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax). Differently, it returns a value of type [DateTime64](../data-types/datetime64.md).
+
+## parseDateTime64InJodaSyntaxOrZero
+
+Same as for [parseDateTime64InJodaSyntax](#parsedatetime64injodasyntax) except that it returns zero date when it encounters a date format that cannot be processed.
+
+## parseDateTime64InJodaSyntaxOrNull
+
+Same as for [parseDateTime64InJodaSyntax](#parsedatetime64injodasyntax) except that it returns `NULL` when it encounters a date format that cannot be processed.
+
 ## parseDateTimeBestEffort
 ## parseDateTime32BestEffort

--- a/docs/en/sql-reference/table-functions/s3Cluster.md
+++ b/docs/en/sql-reference/table-functions/s3Cluster.md
@ -70,6 +70,10 @@ SELECT count(*) FROM s3Cluster(
 )
 ```

+## Optimizing performance
+
+For details on optimizing the performance of the s3 function see [our detailed guide](/docs/en/integrations/s3/performance).
+
 **See Also**

 - [S3 engine](../../engines/table-engines/integrations/s3.md)
--- a/programs/keeper-client/KeeperClient.cpp
+++ b/programs/keeper-client/KeeperClient.cpp
@ -163,6 +163,10 @@ void KeeperClient::defineOptions(Poco::Util::OptionSet & options)
            .argument("<seconds>")
            .binding("operation-timeout"));

+    options.addOption(
+        Poco::Util::Option("use-xid-64", "", "use 64-bit XID. default false.")
+            .binding("use-xid-64"));
+
    options.addOption(
        Poco::Util::Option("config-file", "c", "if set, will try to get a connection string from clickhouse config. default `config.xml`")
            .argument("<file>")
@ -411,6 +415,7 @@ int KeeperClient::main(const std::vector<String> & /* args */)
    zk_args.connection_timeout_ms = config().getInt("connection-timeout", 10) * 1000;
    zk_args.session_timeout_ms = config().getInt("session-timeout", 10) * 1000;
    zk_args.operation_timeout_ms = config().getInt("operation-timeout", 10) * 1000;
+    zk_args.use_xid_64 = config().hasOption("use-xid-64");
    zookeeper = zkutil::ZooKeeper::createWithoutKillingPreviousSessions(zk_args);

    if (config().has("no-confirmation") || config().has("query"))
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@ -2267,6 +2267,30 @@ try
        throw;
    }

+    bool found_stop_flag = false;
+
+    if (has_zookeeper && global_context->getMacros()->getMacroMap().contains("replica"))
+    {
+        try
+        {
+            auto zookeeper = global_context->getZooKeeper();
+            String stop_flag_path = "/clickhouse/stop_replicated_ddl_queries/{replica}";
+            stop_flag_path = global_context->getMacros()->expand(stop_flag_path);
+            found_stop_flag = zookeeper->exists(stop_flag_path);
+        }
+        catch (const Coordination::Exception & e)
+        {
+            if (e.code != Coordination::Error::ZCONNECTIONLOSS)
+                throw;
+            tryLogCurrentException(log);
+        }
+    }
+
+    if (found_stop_flag)
+        LOG_INFO(log, "Found a stop flag for replicated DDL queries. They will be disabled");
+    else
+        DatabaseCatalog::instance().startReplicatedDDLQueries();
+
    LOG_DEBUG(log, "Loaded metadata.");

    if (has_trace_collector)
@ -2999,7 +3023,7 @@ void Server::updateServers(

    for (auto * server : all_servers)
    {
-        if (!server->isStopping())
+        if (server->supportsRuntimeReconfiguration() && !server->isStopping())
        {
            std::string port_name = server->getPortName();
            bool has_host = false;
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@ -1195,6 +1195,19 @@
        <flush_on_crash>false</flush_on_crash>
    </error_log>

+    <!-- Query metric log contains rows Contains history of memory and metric values from table system.events for individual queries, periodically flushed to disk
+    every "collect_interval_milliseconds" interval-->
+    <query_metric_log>
+        <database>system</database>
+        <table>query_metric_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <max_size_rows>1048576</max_size_rows>
+        <reserved_size_rows>8192</reserved_size_rows>
+        <buffer_size_rows_flush_threshold>524288</buffer_size_rows_flush_threshold>
+        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
+        <flush_on_crash>false</flush_on_crash>
+    </query_metric_log>
+
    <!--
        Asynchronous metric log contains values of metrics from
        system.asynchronous_metrics.
--- a/programs/server/config.yaml.example
+++ b/programs/server/config.yaml.example
@ -743,6 +743,13 @@ error_log:
    flush_interval_milliseconds: 7500
    collect_interval_milliseconds: 1000

+# Query metric log contains history of memory and metric values from table system.events for individual queries, periodically flushed to disk.
+query_metric_log:
+    database: system
+    table: query_metric_log
+    flush_interval_milliseconds: 7500
+    collect_interval_milliseconds: 1000
+
 # Asynchronous metric log contains values of metrics from
 # system.asynchronous_metrics.
 asynchronous_metric_log:
--- a/programs/static-files-disk-uploader/static-files-disk-uploader.cpp
+++ b/programs/static-files-disk-uploader/static-files-disk-uploader.cpp
@ -4,6 +4,7 @@

 #include <IO/ReadHelpers.h>
 #include <IO/ReadBufferFromFile.h>
+#include <IO/ReadSettings.h>
 #include <IO/WriteHelpers.h>
 #include <IO/WriteBufferFromHTTP.h>
 #include <IO/WriteBufferFromFile.h>
--- a/src/Access/AccessControl.h
+++ b/src/Access/AccessControl.h
@ -9,6 +9,8 @@

 #include <memory>

+#include "config.h"
+

 namespace Poco
 {
--- a/src/Access/Authentication.cpp
+++ b/src/Access/Authentication.cpp
@ -12,6 +12,7 @@

 #include "config.h"

+
 namespace DB
 {

--- a/src/Access/Common/AccessType.h
+++ b/src/Access/Common/AccessType.h
@ -193,6 +193,7 @@ enum class AccessType : uint8_t
    M(SYSTEM_SENDS, "SYSTEM STOP SENDS, SYSTEM START SENDS, STOP SENDS, START SENDS", GROUP, SYSTEM) \
    M(SYSTEM_REPLICATION_QUEUES, "SYSTEM STOP REPLICATION QUEUES, SYSTEM START REPLICATION QUEUES, STOP REPLICATION QUEUES, START REPLICATION QUEUES", TABLE, SYSTEM) \
    M(SYSTEM_VIRTUAL_PARTS_UPDATE, "SYSTEM STOP VIRTUAL PARTS UPDATE, SYSTEM START VIRTUAL PARTS UPDATE, STOP VIRTUAL PARTS UPDATE, START VIRTUAL PARTS UPDATE", TABLE, SYSTEM) \
+    M(SYSTEM_REDUCE_BLOCKING_PARTS, "SYSTEM STOP REDUCE BLOCKING PARTS, SYSTEM START REDUCE BLOCKING PARTS, STOP REDUCE BLOCKING PARTS, START REDUCE BLOCKING PARTS", TABLE, SYSTEM) \
    M(SYSTEM_DROP_REPLICA, "DROP REPLICA", TABLE, SYSTEM) \
    M(SYSTEM_SYNC_REPLICA, "SYNC REPLICA", TABLE, SYSTEM) \
    M(SYSTEM_REPLICA_READINESS, "SYSTEM REPLICA READY, SYSTEM REPLICA UNREADY", GLOBAL, SYSTEM) \
--- a/src/Access/RoleCache.h
+++ b/src/Access/RoleCache.h
@ -22,6 +22,10 @@ public:
        const std::vector<UUID> & current_roles,
        const std::vector<UUID> & current_roles_with_admin_option);

+    std::shared_ptr<const EnabledRoles> getEnabledRoles(
+        boost::container::flat_set<UUID> current_roles,
+        boost::container::flat_set<UUID> current_roles_with_admin_option);
+
 private:
    using SubscriptionsOnRoles = std::vector<std::shared_ptr<scope_guard>>;

--- a/src/Access/tests/gtest_access_rights_ops.cpp
+++ b/src/Access/tests/gtest_access_rights_ops.cpp
@ -284,7 +284,8 @@ TEST(AccessRights, Union)
              "CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, UNDROP TABLE, "
              "TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, "
              "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
-              "SYSTEM MOVES, SYSTEM PULLING REPLICATION LOG, SYSTEM CLEANUP, SYSTEM VIEWS, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM VIRTUAL PARTS UPDATE, "
+              "SYSTEM MOVES, SYSTEM PULLING REPLICATION LOG, SYSTEM CLEANUP, SYSTEM VIEWS, SYSTEM SENDS, "
+              "SYSTEM REPLICATION QUEUES, SYSTEM VIRTUAL PARTS UPDATE, SYSTEM REDUCE BLOCKING PARTS, "
              "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
              "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, "
              "SYSTEM UNLOAD PRIMARY KEY, dictGet ON db1.*, GRANT TABLE ENGINE ON db1, "
--- a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp
@ -59,13 +59,13 @@ constexpr size_t group_array_sorted_sort_strategy_max_elements_threshold = 10000
 template <typename T, GroupArraySortedStrategy strategy>
 struct GroupArraySortedData
 {
+    static constexpr bool is_value_generic_field = std::is_same_v<T, Field>;
+
    using Allocator = MixedAlignedArenaAllocator<alignof(T), 4096>;
-    using Array = PODArray<T, 32, Allocator>;
+    using Array = typename std::conditional_t<is_value_generic_field, std::vector<T>, PODArray<T, 32, Allocator>>;

    static constexpr size_t partial_sort_max_elements_factor = 2;

-    static constexpr bool is_value_generic_field = std::is_same_v<T, Field>;
-
    Array values;

    static bool compare(const T & lhs, const T & rhs)
@ -144,7 +144,7 @@ struct GroupArraySortedData
        }

        if (values.size() > max_elements)
-            values.resize(max_elements, arena);
+            resize(max_elements, arena);
    }

    ALWAYS_INLINE void partialSortAndLimitIfNeeded(size_t max_elements, Arena * arena)
@ -153,7 +153,23 @@ struct GroupArraySortedData
            return;

        ::nth_element(values.begin(), values.begin() + max_elements, values.end(), Comparator());
-        values.resize(max_elements, arena);
+        resize(max_elements, arena);
+    }
+
+    ALWAYS_INLINE void resize(size_t n, Arena * arena)
+    {
+        if constexpr (is_value_generic_field)
+            values.resize(n);
+        else
+            values.resize(n, arena);
+    }
+
+    ALWAYS_INLINE void push_back(T && element, Arena * arena)
+    {
+        if constexpr (is_value_generic_field)
+            values.push_back(element);
+        else
+            values.push_back(element, arena);
    }

    ALWAYS_INLINE void addElement(T && element, size_t max_elements, Arena * arena)
@ -171,12 +187,12 @@ struct GroupArraySortedData
                return;
            }

-            values.push_back(std::move(element), arena);
+            push_back(std::move(element), arena);
            std::push_heap(values.begin(), values.end(), Comparator());
        }
        else
        {
-            values.push_back(std::move(element), arena);
+            push_back(std::move(element), arena);
            partialSortAndLimitIfNeeded(max_elements, arena);
        }
    }
@ -210,14 +226,6 @@ struct GroupArraySortedData
                result_array_data[result_array_data_insert_begin + i] = values[i];
        }
    }
-
-    ~GroupArraySortedData()
-    {
-        for (auto & value : values)
-        {
-            value.~T();
-        }
-    }
 };

 template <typename T>
@ -313,14 +321,12 @@ public:
            throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size, it should not exceed {}", max_elements);

        auto & values = this->data(place).values;
-        values.resize_exact(size, arena);

-        if constexpr (std::is_same_v<T, Field>)
+        if constexpr (Data::is_value_generic_field)
        {
+            values.resize(size);
            for (Field & element : values)
            {
-                /// We must initialize the Field type since some internal functions (like operator=) use them
-                new (&element) Field;
                bool has_value = false;
                readBinary(has_value, buf);
                if (has_value)
@ -329,6 +335,7 @@ public:
        }
        else
        {
+            values.resize_exact(size, arena);
            if constexpr (std::endian::native == std::endian::little)
            {
                buf.readStrict(reinterpret_cast<char *>(values.data()), size * sizeof(values[0]));
--- a/src/AggregateFunctions/AggregateFunctionQuantile.h
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.h
@ -312,6 +312,9 @@ struct NameQuantilesExactInclusive { static constexpr auto name = "quantilesExac
 struct NameQuantileExactWeighted { static constexpr auto name = "quantileExactWeighted"; };
 struct NameQuantilesExactWeighted { static constexpr auto name = "quantilesExactWeighted"; };

+struct NameQuantileExactWeightedInterpolated { static constexpr auto name = "quantileExactWeightedInterpolated"; };
+struct NameQuantilesExactWeightedInterpolated { static constexpr auto name = "quantilesExactWeightedInterpolated"; };
+
 struct NameQuantileInterpolatedWeighted { static constexpr auto name = "quantileInterpolatedWeighted"; };
 struct NameQuantilesInterpolatedWeighted { static constexpr auto name = "quantilesInterpolatedWeighted"; };

--- a/src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.cpp
@ -1,13 +1,14 @@
-#include <AggregateFunctions/AggregateFunctionQuantile.h>
 #include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionQuantile.h>
 #include <AggregateFunctions/Helpers.h>
+#include <Core/Field.h>
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDateTime.h>
-#include <Core/Field.h>
-
 #include <Common/HashTable/HashMap.h>
 #include <Common/NaNUtils.h>

+#include <numeric>
+

 namespace DB
 {
@ -29,7 +30,7 @@ namespace
  * It uses O(distinct(N)) memory. Can be naturally applied for values with weight.
  * In case of many identical values, it can be more efficient than QuantileExact even when weight is not used.
  */
-template <typename Value>
+template <typename Value, bool interpolated>
 struct QuantileExactWeighted
 {
    struct Int128Hash
@ -46,6 +47,7 @@ struct QuantileExactWeighted

    /// When creating, the hash table must be small.
    using Map = HashMapWithStackMemory<UnderlyingType, Weight, Hasher, 4>;
+    using Pair = typename Map::value_type;

    Map map;

@ -58,8 +60,18 @@ struct QuantileExactWeighted

    void add(const Value & x, Weight weight)
    {
-        if (!isNaN(x))
-            map[x] += weight;
+        if constexpr (!interpolated)
+        {
+            /// Keep compatibility for function quantilesExactWeighted.
+            if (!isNaN(x))
+                map[x] += weight;
+        }
+        else
+        {
+            /// Ignore values with zero weight in function quantilesExactWeightedInterpolated.
+            if (!isNaN(x) && weight)
+                map[x] += weight;
+        }
    }

    void merge(const QuantileExactWeighted & rhs)
@ -85,6 +97,43 @@ struct QuantileExactWeighted

    /// Get the value of the `level` quantile. The level must be between 0 and 1.
    Value get(Float64 level) const
+    {
+        if constexpr (interpolated)
+            return getInterpolatedImpl(level);
+        else
+            return getImpl(level);
+    }
+
+    /// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address.
+    /// indices - an array of index levels such that the corresponding elements will go in ascending order.
+    void getMany(const Float64 * levels, const size_t * indices, size_t num_levels, Value * result) const
+    {
+        if constexpr (interpolated)
+            getManyInterpolatedImpl(levels, indices, num_levels, result);
+        else
+            getManyImpl(levels, indices, num_levels, result);
+    }
+
+    Float64 getFloat(Float64 level) const
+    {
+        if constexpr (interpolated)
+            return getFloatInterpolatedImpl(level);
+        else
+            return getFloatImpl(level);
+    }
+
+    void getManyFloat(const Float64 * levels, const size_t * indices, size_t num_levels, Float64 * result) const
+    {
+        if constexpr (interpolated)
+            getManyFloatInterpolatedImpl(levels, indices, num_levels, result);
+        else
+            getManyFloatImpl(levels, indices, num_levels, result);
+    }
+
+private:
+    /// get implementation without interpolation
+    Value getImpl(Float64 level) const
+    requires(!interpolated)
    {
        size_t size = map.size();

@ -92,7 +141,6 @@ struct QuantileExactWeighted
            return std::numeric_limits<Value>::quiet_NaN();

        /// Copy the data to a temporary array to get the element you need in order.
-        using Pair = typename Map::value_type;
        std::unique_ptr<Pair[]> array_holder(new Pair[size]);
        Pair * array = array_holder.get();

@ -135,9 +183,9 @@ struct QuantileExactWeighted
        return it->first;
    }

-    /// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address.
-    /// indices - an array of index levels such that the corresponding elements will go in ascending order.
-    void getMany(const Float64 * levels, const size_t * indices, size_t num_levels, Value * result) const
+    /// getMany implementation without interpolation
+    void getManyImpl(const Float64 * levels, const size_t * indices, size_t num_levels, Value * result) const
+    requires(!interpolated)
    {
        size_t size = map.size();

@ -149,7 +197,6 @@ struct QuantileExactWeighted
        }

        /// Copy the data to a temporary array to get the element you need in order.
-        using Pair = typename Map::value_type;
        std::unique_ptr<Pair[]> array_holder(new Pair[size]);
        Pair * array = array_holder.get();

@ -197,23 +244,165 @@ struct QuantileExactWeighted
        }
    }

-    /// The same, but in the case of an empty state, NaN is returned.
-    Float64 getFloat(Float64) const
+    /// getFloat implementation without interpolation
+    Float64 getFloatImpl(Float64) const
+    requires(!interpolated)
    {
        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getFloat is not implemented for QuantileExact");
    }

-    void getManyFloat(const Float64 *, const size_t *, size_t, Float64 *) const
+    /// getManyFloat implementation without interpolation
+    void getManyFloatImpl(const Float64 *, const size_t *, size_t, Float64 *) const
+    requires(!interpolated)
    {
        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getManyFloat is not implemented for QuantileExact");
    }
+
+    /// get implementation with interpolation
+    Value getInterpolatedImpl(Float64 level) const
+    requires(interpolated)
+    {
+        size_t size = map.size();
+        if (0 == size)
+            return Value();
+
+        Float64 res = getFloatInterpolatedImpl(level);
+        if constexpr (is_decimal<Value>)
+            return Value(static_cast<typename Value::NativeType>(res));
+        else
+            return static_cast<Value>(res);
+    }
+
+    /// getMany implementation with interpolation
+    void getManyInterpolatedImpl(const Float64 * levels, const size_t * indices, size_t num_levels, Value * result) const
+    requires(interpolated)
+    {
+        size_t size = map.size();
+        if (0 == size)
+        {
+            for (size_t i = 0; i < num_levels; ++i)
+                result[i] = Value();
+            return;
+        }
+
+        std::unique_ptr<Float64 []> res_holder(new Float64[num_levels]);
+        Float64 * res = res_holder.get();
+        getManyFloatInterpolatedImpl(levels, indices, num_levels, res);
+        for (size_t i = 0; i < num_levels; ++i)
+        {
+            if constexpr (is_decimal<Value>)
+                result[i] = Value(static_cast<typename Value::NativeType>(res[i]));
+            else
+                result[i] = Value(res[i]);
+        }
+    }
+
+    /// getFloat implementation with interpolation
+    Float64 getFloatInterpolatedImpl(Float64 level) const
+    requires(interpolated)
+    {
+        size_t size = map.size();
+
+        if (0 == size)
+            return std::numeric_limits<Float64>::quiet_NaN();
+
+        /// Copy the data to a temporary array to get the element you need in order.
+        std::unique_ptr<Pair[]> array_holder(new Pair[size]);
+        Pair * array = array_holder.get();
+
+        size_t i = 0;
+        for (const auto & pair : map)
+        {
+            array[i] = pair.getValue();
+            ++i;
+        }
+
+        ::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; });
+        std::partial_sum(array, array + size, array, [](const Pair & acc, const Pair & p) { return Pair(p.first, acc.second + p.second); });
+        Weight max_position = array[size - 1].second - 1;
+        Float64 position = max_position * level;
+        return quantileInterpolated(array, size, position);
+    }
+
+    /// getManyFloat implementation with interpolation
+    void getManyFloatInterpolatedImpl(const Float64 * levels, const size_t * indices, size_t num_levels, Float64 * result) const
+    requires(interpolated)
+    {
+        size_t size = map.size();
+        if (0 == size)
+        {
+            for (size_t i = 0; i < num_levels; ++i)
+                result[i] = std::numeric_limits<Float64>::quiet_NaN();
+            return;
+        }
+
+        /// Copy the data to a temporary array to get the element you need in order.
+        std::unique_ptr<Pair[]> array_holder(new Pair[size]);
+        Pair * array = array_holder.get();
+
+        size_t i = 0;
+        for (const auto & pair : map)
+        {
+            array[i] = pair.getValue();
+            ++i;
+        }
+
+        ::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; });
+        std::partial_sum(array, array + size, array, [](Pair acc, Pair & p) { return Pair(p.first, acc.second + p.second); });
+        Weight max_position = array[size - 1].second - 1;
+
+        for (size_t j = 0; j < num_levels; ++j)
+        {
+            Float64 position = max_position * levels[indices[j]];
+            result[indices[j]] = quantileInterpolated(array, size, position);
+        }
+    }
+
+    /// Calculate quantile, using linear interpolation between two closest values
+    Float64 NO_SANITIZE_UNDEFINED quantileInterpolated(const Pair * array, size_t size, Float64 position) const
+    requires(interpolated)
+    {
+        size_t lower = static_cast<size_t>(std::floor(position));
+        size_t higher = static_cast<size_t>(std::ceil(position));
+
+        const auto * lower_it = std::lower_bound(array, array + size, lower + 1, [](const Pair & a, size_t b) { return a.second < b; });
+        const auto * higher_it = std::lower_bound(array, array + size, higher + 1, [](const Pair & a, size_t b) { return a.second < b; });
+        if (lower_it == array + size)
+            lower_it = array + size - 1;
+        if (higher_it == array + size)
+            higher_it = array + size - 1;
+
+        UnderlyingType lower_key = lower_it->first;
+        UnderlyingType higher_key = higher_it->first;
+
+        if (lower == higher || lower_key == higher_key)
+            return static_cast<Float64>(lower_key);
+
+        return (static_cast<Float64>(higher) - position) * lower_key + (position - static_cast<Float64>(lower)) * higher_key;
+    }
 };


-template <typename Value, bool _> using FuncQuantileExactWeighted = AggregateFunctionQuantile<Value, QuantileExactWeighted<Value>, NameQuantileExactWeighted, true, void, false, false>;
-template <typename Value, bool _> using FuncQuantilesExactWeighted = AggregateFunctionQuantile<Value, QuantileExactWeighted<Value>, NameQuantilesExactWeighted, true, void, true, false>;
+template <typename Value, bool return_float, bool interpolated>
+using FuncQuantileExactWeighted = AggregateFunctionQuantile<
+    Value,
+    QuantileExactWeighted<Value, interpolated>,
+    NameQuantileExactWeighted,
+    true,
+    std::conditional_t<return_float, Float64, void>,
+    false,
+    false>;
+template <typename Value, bool return_float, bool interpolated>
+using FuncQuantilesExactWeighted = AggregateFunctionQuantile<
+    Value,
+    QuantileExactWeighted<Value, interpolated>,
+    NameQuantilesExactWeighted,
+    true,
+    std::conditional_t<return_float, Float64, void>,
+    true,
+    false>;

-template <template <typename, bool> class Function>
+template <template <typename, bool, bool> class Function, bool interpolated>
 AggregateFunctionPtr createAggregateFunctionQuantile(
    const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
 {
@ -224,22 +413,23 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
    WhichDataType which(argument_type);

 #define DISPATCH(TYPE) \
-    if (which.idx == TypeIndex::TYPE) return std::make_shared<Function<TYPE, true>>(argument_types, params);
+    if (which.idx == TypeIndex::TYPE) \
+        return std::make_shared<Function<TYPE, interpolated, interpolated>>(argument_types, params);
    FOR_BASIC_NUMERIC_TYPES(DISPATCH)
 #undef DISPATCH
-    if (which.idx == TypeIndex::Date) return std::make_shared<Function<DataTypeDate::FieldType, false>>(argument_types, params);
-    if (which.idx == TypeIndex::DateTime) return std::make_shared<Function<DataTypeDateTime::FieldType, false>>(argument_types, params);
+    if (which.idx == TypeIndex::Date) return std::make_shared<Function<DataTypeDate::FieldType, false, interpolated>>(argument_types, params);
+    if (which.idx == TypeIndex::DateTime) return std::make_shared<Function<DataTypeDateTime::FieldType, false, interpolated>>(argument_types, params);

-    if (which.idx == TypeIndex::Decimal32) return std::make_shared<Function<Decimal32, false>>(argument_types, params);
-    if (which.idx == TypeIndex::Decimal64) return std::make_shared<Function<Decimal64, false>>(argument_types, params);
-    if (which.idx == TypeIndex::Decimal128) return std::make_shared<Function<Decimal128, false>>(argument_types, params);
-    if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
-    if (which.idx == TypeIndex::DateTime64) return std::make_shared<Function<DateTime64, false>>(argument_types, params);
+    if (which.idx == TypeIndex::Decimal32) return std::make_shared<Function<Decimal32, false, interpolated>>(argument_types, params);
+    if (which.idx == TypeIndex::Decimal64) return std::make_shared<Function<Decimal64, false, interpolated>>(argument_types, params);
+    if (which.idx == TypeIndex::Decimal128) return std::make_shared<Function<Decimal128, false, interpolated>>(argument_types, params);
+    if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false, interpolated>>(argument_types, params);
+    if (which.idx == TypeIndex::DateTime64) return std::make_shared<Function<DateTime64, false, interpolated>>(argument_types, params);

-    if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, true>>(argument_types, params);
-    if (which.idx == TypeIndex::UInt128) return std::make_shared<Function<UInt128, true>>(argument_types, params);
-    if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, true>>(argument_types, params);
-    if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, true>>(argument_types, params);
+    if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, interpolated, interpolated>>(argument_types, params);
+    if (which.idx == TypeIndex::UInt128) return std::make_shared<Function<UInt128, interpolated, interpolated>>(argument_types, params);
+    if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, interpolated, interpolated>>(argument_types, params);
+    if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, interpolated, interpolated>>(argument_types, params);

    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument for aggregate function {}",
                    argument_type->getName(), name);
@ -252,11 +442,17 @@ void registerAggregateFunctionsQuantileExactWeighted(AggregateFunctionFactory &
    /// For aggregate functions returning array we cannot return NULL on empty set.
    AggregateFunctionProperties properties = { .returns_default_when_only_null = true };

-    factory.registerFunction(NameQuantileExactWeighted::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted>);
-    factory.registerFunction(NameQuantilesExactWeighted::name, { createAggregateFunctionQuantile<FuncQuantilesExactWeighted>, properties });
+    factory.registerFunction(NameQuantileExactWeighted::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted, false>);
+    factory.registerFunction(
+        NameQuantilesExactWeighted::name, {createAggregateFunctionQuantile<FuncQuantilesExactWeighted, false>, properties});
+
+    factory.registerFunction(NameQuantileExactWeightedInterpolated::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted, true>);
+    factory.registerFunction(
+        NameQuantilesExactWeightedInterpolated::name, {createAggregateFunctionQuantile<FuncQuantilesExactWeighted, true>, properties});

    /// 'median' is an alias for 'quantile'
    factory.registerAlias("medianExactWeighted", NameQuantileExactWeighted::name);
+    factory.registerAlias("medianExactWeightedInterpolated", NameQuantileExactWeightedInterpolated::name);
 }

 }
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@ -36,6 +36,24 @@ namespace Setting
    extern const SettingsUInt64 s3_max_redirects;
 }

+namespace S3AuthSetting
+{
+    extern const S3AuthSettingsString access_key_id;
+    extern const S3AuthSettingsUInt64 expiration_window_seconds;
+    extern const S3AuthSettingsBool no_sign_request;
+    extern const S3AuthSettingsString region;
+    extern const S3AuthSettingsString secret_access_key;
+    extern const S3AuthSettingsString server_side_encryption_customer_key_base64;
+    extern const S3AuthSettingsBool use_environment_credentials;
+    extern const S3AuthSettingsBool use_insecure_imds_request;
+}
+
+namespace S3RequestSetting
+{
+    extern const S3RequestSettingsBool allow_native_copy;
+    extern const S3RequestSettingsString storage_class_name;
+}
+
 namespace ErrorCodes
 {
    extern const int S3_ERROR;
@ -55,7 +73,7 @@ namespace
        HTTPHeaderEntries headers;
        if (access_key_id.empty())
        {
-            credentials = Aws::Auth::AWSCredentials(settings.auth_settings.access_key_id, settings.auth_settings.secret_access_key);
+            credentials = Aws::Auth::AWSCredentials(settings.auth_settings[S3AuthSetting::access_key_id], settings.auth_settings[S3AuthSetting::secret_access_key]);
            headers = settings.auth_settings.headers;
        }

@ -64,7 +82,7 @@ namespace
        const Settings & local_settings = context->getSettingsRef();

        S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
-            settings.auth_settings.region,
+            settings.auth_settings[S3AuthSetting::region],
            context->getRemoteHostFilter(),
            static_cast<unsigned>(local_settings[Setting::s3_max_redirects]),
            static_cast<unsigned>(local_settings[Setting::backup_restore_s3_retry_attempts]),
@ -95,15 +113,15 @@ namespace
            client_settings,
            credentials.GetAWSAccessKeyId(),
            credentials.GetAWSSecretKey(),
-            settings.auth_settings.server_side_encryption_customer_key_base64,
+            settings.auth_settings[S3AuthSetting::server_side_encryption_customer_key_base64],
            settings.auth_settings.server_side_encryption_kms_config,
            std::move(headers),
            S3::CredentialsConfiguration
            {
-                settings.auth_settings.use_environment_credentials,
-                settings.auth_settings.use_insecure_imds_request,
-                settings.auth_settings.expiration_window_seconds,
-                settings.auth_settings.no_sign_request
+                settings.auth_settings[S3AuthSetting::use_environment_credentials],
+                settings.auth_settings[S3AuthSetting::use_insecure_imds_request],
+                settings.auth_settings[S3AuthSetting::expiration_window_seconds],
+                settings.auth_settings[S3AuthSetting::no_sign_request]
            });
    }

@ -143,7 +161,7 @@ BackupReaderS3::BackupReaderS3(
    }

    s3_settings.request_settings.updateFromSettings(context_->getSettingsRef(), /* if_changed */true);
-    s3_settings.request_settings.allow_native_copy = allow_s3_native_copy;
+    s3_settings.request_settings[S3RequestSetting::allow_native_copy] = allow_s3_native_copy;

    client = makeS3Client(s3_uri_, access_key_id_, secret_access_key_, s3_settings, context_);

@ -242,8 +260,8 @@ BackupWriterS3::BackupWriterS3(
    }

    s3_settings.request_settings.updateFromSettings(context_->getSettingsRef(), /* if_changed */true);
-    s3_settings.request_settings.allow_native_copy = allow_s3_native_copy;
-    s3_settings.request_settings.storage_class_name = storage_class_name;
+    s3_settings.request_settings[S3RequestSetting::allow_native_copy] = allow_s3_native_copy;
+    s3_settings.request_settings[S3RequestSetting::storage_class_name] = storage_class_name;

    client = makeS3Client(s3_uri_, access_key_id_, secret_access_key_, s3_settings, context_);
    if (auto blob_storage_system_log = context_->getBlobStorageLog())
--- a/src/Common/CurrentMetrics.cpp
+++ b/src/Common/CurrentMetrics.cpp
@ -27,8 +27,8 @@
    M(BackgroundBufferFlushSchedulePoolSize, "Limit on number of tasks in BackgroundBufferFlushSchedulePool") \
    M(BackgroundDistributedSchedulePoolTask, "Number of active tasks in BackgroundDistributedSchedulePool. This pool is used for distributed sends that is done in background.") \
    M(BackgroundDistributedSchedulePoolSize, "Limit on number of tasks in BackgroundDistributedSchedulePool") \
-    M(BackgroundMessageBrokerSchedulePoolTask, "Number of active tasks in BackgroundProcessingPool for message streaming") \
-    M(BackgroundMessageBrokerSchedulePoolSize, "Limit on number of tasks in BackgroundProcessingPool for message streaming") \
+    M(BackgroundMessageBrokerSchedulePoolTask, "Number of active tasks in BackgroundMessageBrokerSchedulePool for message streaming") \
+    M(BackgroundMessageBrokerSchedulePoolSize, "Limit on number of tasks in BackgroundMessageBrokerSchedulePool for message streaming") \
    M(CacheDictionaryUpdateQueueBatches, "Number of 'batches' (a set of keys) in update queue in CacheDictionaries.") \
    M(CacheDictionaryUpdateQueueKeys, "Exact number of keys in update queue in CacheDictionaries.") \
    M(DiskSpaceReservedForMerge, "Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.") \
--- a/src/Common/CurrentMetrics.h
+++ b/src/Common/CurrentMetrics.h
@ -1,7 +1,6 @@
 #pragma once

 #include <cstddef>
-#include <cstdint>
 #include <utility>
 #include <atomic>
 #include <cassert>
--- a/src/Common/LockGuard.h
+++ b/src/Common/LockGuard.h
@ -0,0 +1,37 @@
+#pragma once
+
+#include <Common/OvercommitTracker.h>
+#include <base/defines.h>
+
+namespace DB
+{
+
+/** LockGuard provides RAII-style locking mechanism for a mutex.
+ ** It's intended to be used like std::unique_ptr but with TSA annotations
+  */
+template <typename Mutex>
+class TSA_SCOPED_LOCKABLE LockGuard
+{
+public:
+    explicit LockGuard(Mutex & mutex_) TSA_ACQUIRE(mutex_) : mutex(mutex_) { mutex.lock(); }
+    ~LockGuard() TSA_RELEASE() { mutex.unlock(); }
+
+private:
+    Mutex & mutex;
+};
+
+template <template<typename> typename TLockGuard, typename Mutex>
+class TSA_SCOPED_LOCKABLE LockAndOverCommitTrackerBlocker
+{
+public:
+    explicit LockAndOverCommitTrackerBlocker(Mutex & mutex_) TSA_ACQUIRE(mutex_) : lock(TLockGuard(mutex_)) {}
+    ~LockAndOverCommitTrackerBlocker() TSA_RELEASE() = default;
+
+    TLockGuard<Mutex> & getUnderlyingLock() { return lock; }
+
+private:
+    TLockGuard<Mutex> lock;
+    OvercommitTrackerBlockerInThread blocker = {};
+};
+
+}
--- a/src/Common/OvercommitTracker.cpp
+++ b/src/Common/OvercommitTracker.cpp
@ -45,7 +45,7 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int
    // method OvercommitTracker::onQueryStop(MemoryTracker *) is
    // always called with already acquired global mutex in
    // ProcessListEntry::~ProcessListEntry().
-    auto global_lock = process_list->unsafeLock();
+    DB::ProcessList::Lock global_lock(process_list->getMutex());
    std::unique_lock<std::mutex> lk(overcommit_m);

    size_t id = next_id++;
--- a/src/Common/SharedLockGuard.h
+++ b/src/Common/SharedLockGuard.h
@ -5,7 +5,7 @@
 namespace DB
 {

-/** SharedLockGuard provide RAII-style locking mechanism for acquiring shared ownership of the implementation
+/** SharedLockGuard provides RAII-style locking mechanism for acquiring shared ownership of the implementation
  * of the SharedLockable concept (for example std::shared_mutex or ContextSharedMutex) supplied as the
  * constructor argument. Think of it as std::lock_guard which locks shared.
  *
--- a/src/Common/SystemLogBase.cpp
+++ b/src/Common/SystemLogBase.cpp
@ -4,6 +4,7 @@
 #include <Interpreters/MetricLog.h>
 #include <Interpreters/OpenTelemetrySpanLog.h>
 #include <Interpreters/PartLog.h>
+#include <Interpreters/QueryMetricLog.h>
 #include <Interpreters/QueryLog.h>
 #include <Interpreters/QueryThreadLog.h>
 #include <Interpreters/QueryViewsLog.h>
@ -18,6 +19,7 @@
 #include <Interpreters/TransactionsInfoLog.h>
 #include <Interpreters/AsynchronousInsertLog.h>
 #include <Interpreters/BackupLog.h>
+#include <Interpreters/PeriodicLog.h>
 #include <IO/S3/BlobStorageLogWriter.h>

 #include <Common/MemoryTrackerBlockerInThread.h>
@ -299,8 +301,10 @@ void SystemLogBase<LogElement>::add(LogElement element)

 #define INSTANTIATE_SYSTEM_LOG_BASE(ELEMENT) template class SystemLogBase<ELEMENT>;
 SYSTEM_LOG_ELEMENTS(INSTANTIATE_SYSTEM_LOG_BASE)
+SYSTEM_PERIODIC_LOG_ELEMENTS(INSTANTIATE_SYSTEM_LOG_BASE)

 #define INSTANTIATE_SYSTEM_LOG_QUEUE(ELEMENT) template class SystemLogQueue<ELEMENT>;
 SYSTEM_LOG_ELEMENTS(INSTANTIATE_SYSTEM_LOG_QUEUE)
+SYSTEM_PERIODIC_LOG_ELEMENTS(INSTANTIATE_SYSTEM_LOG_QUEUE)

 }
--- a/src/Common/SystemLogBase.h
+++ b/src/Common/SystemLogBase.h
@ -14,7 +14,6 @@
 #define SYSTEM_LOG_ELEMENTS(M) \
    M(AsynchronousMetricLogElement) \
    M(CrashLogElement) \
-    M(MetricLogElement) \
    M(OpenTelemetrySpanLogElement) \
    M(PartLogElement) \
    M(QueryLogElement) \
@ -32,7 +31,7 @@
    M(AsynchronousInsertLogElement) \
    M(BackupLogElement) \
    M(BlobStorageLogElement) \
-    M(ErrorLogElement)
+    M(QueryMetricLogElement)

 namespace Poco
 {
--- a/src/Common/ZooKeeper/ZooKeeperArgs.cpp
+++ b/src/Common/ZooKeeper/ZooKeeperArgs.cpp
@ -99,9 +99,12 @@ void ZooKeeperArgs::initFromKeeperServerSection(const Poco::Util::AbstractConfig
        if (auto session_timeout_key = coordination_key + ".session_timeout_ms";
            config.has(session_timeout_key))
            session_timeout_ms = config.getInt(session_timeout_key);
-    }

-    use_xid_64 = config.getBool(std::string{config_name} + ".use_xid_64", false);
+        if (auto use_xid_64_key = coordination_key + ".use_xid_64";
+            config.has(use_xid_64_key))
+            use_xid_64 = config.getBool(use_xid_64_key);
+
+    }

    Poco::Util::AbstractConfiguration::Keys keys;
    std::string raft_configuration_key = std::string{config_name} + ".raft_configuration";
--- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp
+++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
@ -1226,6 +1226,9 @@ void ZooKeeper::pushRequest(RequestInfo && info)
        if (!info.request->xid)
        {
            info.request->xid = next_xid.fetch_add(1);
+            if (!use_xid_64)
+                info.request->xid = static_cast<int32_t>(info.request->xid);
+
            if (info.request->xid == close_xid)
                throw Exception::fromMessage(Error::ZSESSIONEXPIRED, "xid equal to close_xid");
            if (info.request->xid < 0)
--- a/src/Compression/CompressedReadBufferFromFile.h
+++ b/src/Compression/CompressedReadBufferFromFile.h
@ -1,10 +1,9 @@
 #pragma once

+#include <memory>
+#include <time.h>
 #include <Compression/CompressedReadBufferBase.h>
 #include <IO/ReadBufferFromFileBase.h>
-#include <IO/ReadSettings.h>
-#include <time.h>
-#include <memory>


 namespace DB
--- a/src/Coordination/CoordinationSettings.cpp
+++ b/src/Coordination/CoordinationSettings.cpp
@ -62,7 +62,8 @@ namespace ErrorCodes
    DECLARE(UInt64, disk_move_retries_during_init, 100, "The amount of retries after a failure which happened while a file was being moved between disks during initialization.", 0) \
    DECLARE(UInt64, log_slow_total_threshold_ms, 5000, "Requests for which the total latency is larger than this settings will be logged", 0) \
    DECLARE(UInt64, log_slow_cpu_threshold_ms, 100, "Requests for which the CPU (preprocessing and processing) latency is larger than this settings will be logged", 0) \
-    DECLARE(UInt64, log_slow_connection_operation_threshold_ms, 1000, "Log message if a certain operation took too long inside a single connection", 0)
+    DECLARE(UInt64, log_slow_connection_operation_threshold_ms, 1000, "Log message if a certain operation took too long inside a single connection", 0) \
+    DECLARE(Bool, use_xid_64, false, "Enable 64-bit XID. It is disabled by default because of backward compatibility", 0)

 DECLARE_SETTINGS_TRAITS(CoordinationSettingsTraits, LIST_OF_COORDINATION_SETTINGS)
 IMPLEMENT_SETTINGS_TRAITS(CoordinationSettingsTraits, LIST_OF_COORDINATION_SETTINGS)
--- a/src/Coordination/KeeperDispatcher.cpp
+++ b/src/Coordination/KeeperDispatcher.cpp
@ -417,7 +417,7 @@ void KeeperDispatcher::setResponse(int64_t session_id, const Coordination::ZooKe
    }
 }

-bool KeeperDispatcher::putRequest(const Coordination::ZooKeeperRequestPtr & request, int64_t session_id)
+bool KeeperDispatcher::putRequest(const Coordination::ZooKeeperRequestPtr & request, int64_t session_id, bool use_xid_64)
 {
    {
        /// If session was already disconnected than we will ignore requests
@ -427,6 +427,7 @@ bool KeeperDispatcher::putRequest(const Coordination::ZooKeeperRequestPtr & requ
    }

    KeeperStorageBase::RequestForSession request_info;
+    request_info.use_xid_64 = use_xid_64;
    request_info.request = request;
    using namespace std::chrono;
    request_info.time = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
--- a/src/Coordination/KeeperDispatcher.h
+++ b/src/Coordination/KeeperDispatcher.h
@ -140,7 +140,7 @@ public:
    void forceRecovery();

    /// Put request to ClickHouse Keeper
-    bool putRequest(const Coordination::ZooKeeperRequestPtr & request, int64_t session_id);
+    bool putRequest(const Coordination::ZooKeeperRequestPtr & request, int64_t session_id, bool use_xid_64);

    /// Get new session ID
    int64_t getSessionID(int64_t session_timeout_ms);
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@ -877,7 +877,8 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ
                auto entry_buf = entry->get_buf_ptr();

                IKeeperStateMachine::ZooKeeperLogSerializationVersion serialization_version;
-                auto request_for_session = state_machine->parseRequest(*entry_buf, /*final=*/false, &serialization_version);
+                size_t request_end_position = 0;
+                auto request_for_session = state_machine->parseRequest(*entry_buf, /*final=*/false, &serialization_version, &request_end_position);
                request_for_session->zxid = next_zxid;
                if (!state_machine->preprocess(*request_for_session))
                    return nuraft::cb_func::ReturnCode::ReturnNull;
@ -892,9 +893,6 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ
                if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_ZXID_DIGEST)
                    bytes_missing += sizeof(request_for_session->zxid) + sizeof(request_for_session->digest->version) + sizeof(request_for_session->digest->value);

-                if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_XID_64)
-                    bytes_missing += sizeof(uint32_t);
-
                if (bytes_missing != 0)
                {
                    auto new_buffer = nuraft::buffer::alloc(entry_buf->size() + bytes_missing);
@ -904,12 +902,14 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ
                }

                size_t write_buffer_header_size = sizeof(request_for_session->zxid) + sizeof(request_for_session->digest->version)
-                    + sizeof(request_for_session->digest->value) + sizeof(uint32_t);
+                    + sizeof(request_for_session->digest->value);

                if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_TIME)
                    write_buffer_header_size += sizeof(request_for_session->time);
+                else
+                    request_end_position += sizeof(request_for_session->time);

-                auto * buffer_start = reinterpret_cast<BufferBase::Position>(entry_buf->data_begin() + entry_buf->size() - write_buffer_header_size);
+                auto * buffer_start = reinterpret_cast<BufferBase::Position>(entry_buf->data_begin() + request_end_position);

                WriteBufferFromPointer write_buf(buffer_start, write_buffer_header_size);

--- a/src/Coordination/KeeperServer.h
+++ b/src/Coordination/KeeperServer.h
@ -70,7 +70,6 @@ private:

    const bool create_snapshot_on_exit;
    const bool enable_reconfiguration;
-
 public:
    KeeperServer(
        const KeeperConfigurationAndSettingsPtr & settings_,
--- a/src/Coordination/KeeperSnapshotManagerS3.cpp
+++ b/src/Coordination/KeeperSnapshotManagerS3.cpp
@ -31,16 +31,34 @@ namespace fs = std::filesystem;
 namespace DB
 {

+namespace S3AuthSetting
+{
+    extern const S3AuthSettingsString access_key_id;
+    extern const S3AuthSettingsUInt64 expiration_window_seconds;
+    extern const S3AuthSettingsBool no_sign_request;
+    extern const S3AuthSettingsString region;
+    extern const S3AuthSettingsString secret_access_key;
+    extern const S3AuthSettingsString server_side_encryption_customer_key_base64;
+    extern const S3AuthSettingsString session_token;
+    extern const S3AuthSettingsBool use_environment_credentials;
+    extern const S3AuthSettingsBool use_insecure_imds_request;
+}
+
+namespace S3RequestSetting
+{
+    extern const S3RequestSettingsUInt64 max_single_read_retries;
+}
+
 struct KeeperSnapshotManagerS3::S3Configuration
 {
-    S3Configuration(S3::URI uri_, S3::AuthSettings auth_settings_, std::shared_ptr<const S3::Client> client_)
+    S3Configuration(S3::URI uri_, S3::S3AuthSettings auth_settings_, std::shared_ptr<const S3::Client> client_)
        : uri(std::move(uri_))
        , auth_settings(std::move(auth_settings_))
        , client(std::move(client_))
    {}

    S3::URI uri;
-    S3::AuthSettings auth_settings;
+    S3::S3AuthSettings auth_settings;
    std::shared_ptr<const S3::Client> client;
 };

@ -66,7 +84,7 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo
        }

        const auto & settings = Context::getGlobalContextInstance()->getSettingsRef();
-        auto auth_settings = S3::AuthSettings(config, settings, config_prefix);
+        auto auth_settings = S3::S3AuthSettings(config, settings, config_prefix);

        String endpoint = macros->expand(config.getString(config_prefix + ".endpoint"));
        auto new_uri = S3::URI{endpoint};
@ -81,7 +99,7 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo

        LOG_INFO(log, "S3 configuration was updated");

-        auto credentials = Aws::Auth::AWSCredentials(auth_settings.access_key_id, auth_settings.secret_access_key, auth_settings.session_token);
+        auto credentials = Aws::Auth::AWSCredentials(auth_settings[S3AuthSetting::access_key_id], auth_settings[S3AuthSetting::secret_access_key], auth_settings[S3AuthSetting::session_token]);
        auto headers = auth_settings.headers;

        static constexpr size_t s3_max_redirects = 10;
@ -95,7 +113,7 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo
        }

        S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
-            auth_settings.region,
+            auth_settings[S3AuthSetting::region],
            RemoteHostFilter(), s3_max_redirects, s3_retry_attempts,
            enable_s3_requests_logging,
            /* for_disk_s3 = */ false, /* get_request_throttler = */ {}, /* put_request_throttler = */ {},
@ -115,15 +133,15 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo
            client_settings,
            credentials.GetAWSAccessKeyId(),
            credentials.GetAWSSecretKey(),
-            auth_settings.server_side_encryption_customer_key_base64,
+            auth_settings[S3AuthSetting::server_side_encryption_customer_key_base64],
            auth_settings.server_side_encryption_kms_config,
            std::move(headers),
            S3::CredentialsConfiguration
            {
-                auth_settings.use_environment_credentials,
-                auth_settings.use_insecure_imds_request,
-                auth_settings.expiration_window_seconds,
-                auth_settings.no_sign_request,
+                auth_settings[S3AuthSetting::use_environment_credentials],
+                auth_settings[S3AuthSetting::use_insecure_imds_request],
+                auth_settings[S3AuthSetting::expiration_window_seconds],
+                auth_settings[S3AuthSetting::no_sign_request],
            },
            credentials.GetSessionToken());

@ -156,7 +174,7 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh
        if (s3_client == nullptr)
            return;

-        S3::RequestSettings request_settings_1;
+        S3::S3RequestSettings request_settings_1;

        const auto create_writer = [&](const auto & key)
        {
@ -199,8 +217,8 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh
        lock_writer.finalize();

        // We read back the written UUID, if it's the same we can upload the file
-        S3::RequestSettings request_settings_2;
-        request_settings_2.max_single_read_retries = 1;
+        S3::S3RequestSettings request_settings_2;
+        request_settings_2[S3RequestSetting::max_single_read_retries] = 1;
        ReadBufferFromS3 lock_reader
        {
            s3_client->client,
--- a/src/Coordination/KeeperStateMachine.cpp
+++ b/src/Coordination/KeeperStateMachine.cpp
@ -267,7 +267,11 @@ nuraft::ptr<nuraft::buffer> IKeeperStateMachine::getZooKeeperLogEntry(const Keep
    size_t request_size = sizeof(uint32_t) + Coordination::size(request->getOpNum()) + request->sizeImpl();
    Coordination::write(static_cast<int32_t>(request_size), write_buf);
    XidHelper xid_helper{.xid = request->xid};
-    Coordination::write(xid_helper.parts.lower, write_buf);
+    if (request_for_session.use_xid_64)
+        Coordination::write(xid_helper.parts.lower, write_buf);
+    else
+        Coordination::write(static_cast<int32_t>(xid_helper.xid), write_buf);
+
    Coordination::write(request->getOpNum(), write_buf);
    request->writeImpl(write_buf);

@ -276,13 +280,15 @@ nuraft::ptr<nuraft::buffer> IKeeperStateMachine::getZooKeeperLogEntry(const Keep
    DB::writeIntBinary(static_cast<int64_t>(0), write_buf); /// zxid
    DB::writeIntBinary(KeeperStorageBase::DigestVersion::NO_DIGEST, write_buf); /// digest version or NO_DIGEST flag
    DB::writeIntBinary(static_cast<uint64_t>(0), write_buf); /// digest value
-    Coordination::write(xid_helper.parts.upper, write_buf); /// for 64bit XID MSB
+
+    if (request_for_session.use_xid_64)
+        Coordination::write(xid_helper.parts.upper, write_buf); /// for 64bit XID MSB
    /// if new fields are added, update KeeperStateMachine::ZooKeeperLogSerializationVersion along with parseRequest function and PreAppendLog callback handler
    return write_buf.getBuffer();
 }

-std::shared_ptr<KeeperStorageBase::RequestForSession>
-IKeeperStateMachine::parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version)
+std::shared_ptr<KeeperStorageBase::RequestForSession> IKeeperStateMachine::parseRequest(
+    nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version, size_t * request_end_position)
 {
    ReadBufferFromNuraftBuffer buffer(data);
    auto request_for_session = std::make_shared<KeeperStorageBase::RequestForSession>();
@ -302,6 +308,9 @@ IKeeperStateMachine::parseRequest(nuraft::buffer & data, bool final, ZooKeeperLo
    auto buffer_position = buffer.getPosition();
    buffer.seek(length - sizeof(uint32_t), SEEK_CUR);

+    if (request_end_position)
+        *request_end_position = buffer.getPosition();
+
    using enum ZooKeeperLogSerializationVersion;
    ZooKeeperLogSerializationVersion version = INITIAL;

@ -333,6 +342,10 @@ IKeeperStateMachine::parseRequest(nuraft::buffer & data, bool final, ZooKeeperLo
        version = WITH_XID_64;
        Coordination::read(xid_helper.parts.upper, buffer);
    }
+    else
+    {
+        xid_helper.xid = static_cast<int32_t>(xid_helper.parts.lower);
+    }

    if (serialization_version)
        *serialization_version = version;
--- a/src/Coordination/KeeperStateMachine.h
+++ b/src/Coordination/KeeperStateMachine.h
@ -48,8 +48,11 @@ public:
    ///
    /// final - whether it's the final time we will fetch the request so we can safely remove it from cache
    /// serialization_version - information about which fields were parsed from the buffer so we can modify the buffer accordingly
-    std::shared_ptr<KeeperStorageBase::RequestForSession>
-    parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version = nullptr);
+    std::shared_ptr<KeeperStorageBase::RequestForSession> parseRequest(
+        nuraft::buffer & data,
+        bool final,
+        ZooKeeperLogSerializationVersion * serialization_version = nullptr,
+        size_t * request_end_position = nullptr);

    static nuraft::ptr<nuraft::buffer> getZooKeeperLogEntry(const KeeperStorageBase::RequestForSession & request_for_session);

--- a/src/Coordination/KeeperStorage.h
+++ b/src/Coordination/KeeperStorage.h
@ -303,6 +303,7 @@ public:
        int64_t zxid{0};
        std::optional<Digest> digest;
        int64_t log_idx{0};
+        bool use_xid_64{false};
    };
    using RequestsForSessions = std::vector<RequestForSession>;

--- a/src/Core/BaseSettings.h
+++ b/src/Core/BaseSettings.h
@ -91,17 +91,12 @@ public:
    virtual void set(std::string_view name, const Field & value);
    Field get(std::string_view name) const;

-    void setString(std::string_view name, const String & value);
-    String getString(std::string_view name) const;
-
    bool tryGet(std::string_view name, Field & value) const;
-    bool tryGetString(std::string_view name, String & value) const;

    bool isChanged(std::string_view name) const;
    SettingsChanges changes() const;
    void applyChange(const SettingChange & change);
    void applyChanges(const SettingsChanges & changes);
-    void applyChanges(const BaseSettings & changes); /// NOLINT

    /// Resets all the settings to their default values.
    void resetToDefault();
@ -118,15 +113,12 @@ public:
    /// Checks if it's possible to assign a field to a specified value and throws an exception if not.
    /// This function doesn't change the fields, it performs check only.
    static void checkCanSet(std::string_view name, const Field & value);
-    static void checkCanSetString(std::string_view name, const String & str);

    /// Conversions without changing the fields.
    static Field castValueUtil(std::string_view name, const Field & value);
    static String valueToStringUtil(std::string_view name, const Field & value);
    static Field stringToValueUtil(std::string_view name, const String & str);

-    static std::string_view resolveName(std::string_view name);
-
    void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const;
    void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT);

@ -140,7 +132,6 @@ public:
        const String & getName() const;
        Field getValue() const;
        void setValue(const Field & value);
-        Field getDefaultValue() const;
        String getValueString() const;
        String getDefaultValueString() const;
        bool isValueChanged() const;
@ -273,27 +264,6 @@ Field BaseSettings<TTraits>::get(std::string_view name) const
    return static_cast<Field>(getCustomSetting(name));
 }

-template <typename TTraits>
-void BaseSettings<TTraits>::setString(std::string_view name, const String & value)
-{
-    name = TTraits::resolveName(name);
-    const auto & accessor = Traits::Accessor::instance();
-    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
-        accessor.setValueString(*this, index, value);
-    else
-        getCustomSetting(name).parseFromString(value);
-}
-
-template <typename TTraits>
-String BaseSettings<TTraits>::getString(std::string_view name) const
-{
-    name = TTraits::resolveName(name);
-    const auto & accessor = Traits::Accessor::instance();
-    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
-        return accessor.getValueString(*this, index);
-    return getCustomSetting(name).toString();
-}
-
 template <typename TTraits>
 bool BaseSettings<TTraits>::tryGet(std::string_view name, Field & value) const
 {
@ -312,24 +282,6 @@ bool BaseSettings<TTraits>::tryGet(std::string_view name, Field & value) const
    return false;
 }

-template <typename TTraits>
-bool BaseSettings<TTraits>::tryGetString(std::string_view name, String & value) const
-{
-    name = TTraits::resolveName(name);
-    const auto & accessor = Traits::Accessor::instance();
-    if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
-    {
-        value = accessor.getValueString(*this, index);
-        return true;
-    }
-    if (const auto * custom_setting = tryGetCustomSetting(name))
-    {
-        value = custom_setting->toString();
-        return true;
-    }
-    return false;
-}
-
 template <typename TTraits>
 bool BaseSettings<TTraits>::isChanged(std::string_view name) const
 {
@ -362,13 +314,6 @@ void BaseSettings<TTraits>::applyChanges(const SettingsChanges & changes)
        applyChange(change);
 }

-template <typename TTraits>
-void BaseSettings<TTraits>::applyChanges(const BaseSettings & other_settings)
-{
-    for (const auto & field : other_settings)
-        set(field.getName(), field.getValue());
-}
-
 template <typename TTraits>
 void BaseSettings<TTraits>::resetToDefault()
 {
@ -438,13 +383,6 @@ void BaseSettings<TTraits>::checkCanSet(std::string_view name, const Field & val
    castValueUtil(name, value);
 }

-template <typename TTraits>
-void BaseSettings<TTraits>::checkCanSetString(std::string_view name, const String & str)
-{
-    name = TTraits::resolveName(name);
-    stringToValueUtil(name, str);
-}
-
 template <typename TTraits>
 Field BaseSettings<TTraits>::castValueUtil(std::string_view name, const Field & value)
 {
@ -794,17 +732,6 @@ void BaseSettings<TTraits>::SettingFieldRef::setValue(const Field & value)
        accessor->setValue(*settings, index, value);
 }

-template <typename TTraits>
-Field BaseSettings<TTraits>::SettingFieldRef::getDefaultValue() const
-{
-    if constexpr (Traits::allow_custom_settings)
-    {
-        if (custom_setting)
-            return static_cast<Field>(custom_setting->second);
-    }
-    return accessor->getDefaultValue(index);
-}
-
 template <typename TTraits>
 String BaseSettings<TTraits>::SettingFieldRef::getValueString() const
 {
@ -921,7 +848,6 @@ using AliasMap = std::unordered_map<std::string_view, std::string_view>;
            void resetValueToDefault(Data & data, size_t index) const { return field_infos[index].reset_value_to_default_function(data); } \
            void writeBinary(const Data & data, size_t index, WriteBuffer & out) const { return field_infos[index].write_binary_function(data, out); } \
            void readBinary(Data & data, size_t index, ReadBuffer & in) const { return field_infos[index].read_binary_function(data, in); } \
-            Field getDefaultValue(size_t index) const { return field_infos[index].get_default_value_function(); } \
            String getDefaultValueString(size_t index) const { return field_infos[index].get_default_value_string_function(); } \
        private: \
            Accessor(); \
@ -943,7 +869,6 @@ using AliasMap = std::unordered_map<std::string_view, std::string_view>;
                void (*reset_value_to_default_function)(Data &) ; \
                void (*write_binary_function)(const Data &, WriteBuffer &) ; \
                void (*read_binary_function)(Data &, ReadBuffer &) ; \
-                Field (*get_default_value_function)() ; \
                String (*get_default_value_string_function)() ; \
            }; \
            std::vector<FieldInfo> field_infos; \
@ -1056,7 +981,6 @@ struct DefineAliases
            [](Data & data) { data.NAME = SettingField##TYPE{DEFAULT}; }, \
            [](const Data & data, WriteBuffer & out) { data.NAME.writeBinary(out); }, \
            [](Data & data, ReadBuffer & in) { data.NAME.readBinary(in); }, \
-            []() -> Field { return static_cast<Field>(SettingField##TYPE{DEFAULT}); }, \
            []() -> String { return SettingField##TYPE{DEFAULT}.toString(); } \
        });
 }
--- a/src/Core/ServerUUID.cpp
+++ b/src/Core/ServerUUID.cpp
@ -68,6 +68,11 @@ UUID loadServerUUID(const fs::path & server_uuid_file, Poco::Logger * log)
    }
 }

+void ServerUUID::set(UUID & uuid)
+{
+    server_uuid = uuid;
+}
+
 void ServerUUID::setRandomForUnitTests()
 {
    server_uuid = UUIDHelpers::generateV4();
--- a/src/Core/ServerUUID.h
+++ b/src/Core/ServerUUID.h
@ -20,6 +20,9 @@ public:
    /// Loads server UUID from file or creates new one. Should be called on daemon startup.
    static void load(const fs::path & server_uuid_file, Poco::Logger * log);

+    /// Sets specific server UUID.
+    static void set(UUID & uuid);
+
    static void setRandomForUnitTests();
 };

--- a/src/Core/Settings.cpp
+++ b/src/Core/Settings.cpp
@ -4,6 +4,7 @@
 #include <Core/BaseSettingsFwdMacros.h>
 #include <Core/BaseSettingsFwdMacrosImpl.h>
 #include <Core/BaseSettingsProgramOptions.h>
+#include <Core/DistributedCacheProtocol.h>
 #include <Core/FormatFactorySettings.h>
 #include <Core/Settings.h>
 #include <Core/SettingsChangesHistory.h>
@ -2749,6 +2750,15 @@ Result:
 └─────────────┴───────────┘
 ```
 )", 0) \
+    DECLARE(Int64, query_metric_log_interval, -1, R"(
+The interval in milliseconds at which the [query_metric_log](../../operations/system-tables/query_metric_log.md) for individual queries is collected.
+
+If set to any negative value, it will take the value `collect_interval_milliseconds` from the [query_metric_log setting](../../operations/server-configuration-parameters/settings.md#query_metric_log) or default to 1000 if not present.
+
+To disable the collection of a single query, set `query_metric_log_interval` to 0.
+
+Default value: -1
+    )", 0) \
    DECLARE(LogsLevel, send_logs_level, LogsLevel::fatal, R"(
 Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'
 )", 0) \
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@ -5,9 +5,7 @@
 #include <Core/SettingsEnums.h>
 #include <Core/SettingsFields.h>
 #include <Core/SettingsWriteFormat.h>
-#include <Core/ParallelReplicasMode.h>
 #include <base/types.h>
-#include <Common/SettingConstraintWritability.h>
 #include <Common/SettingsChanges.h>

 #include <string_view>
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@ -69,6 +69,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
    {"24.10",
        {
            {"enable_job_stack_trace", false, true, "Enable by default collecting stack traces from job's scheduling."},
+            {"query_metric_log_interval", 0, -1, "New setting."},
            {"enforce_strict_identifier_format", false, false, "New setting."},
            {"enable_parsing_to_custom_serialization", false, true, "New setting"},
            {"mongodb_throw_on_unsupported_query", false, true, "New setting."},
--- a/src/Core/SettingsEnums.h
+++ b/src/Core/SettingsEnums.h
@ -12,7 +12,9 @@
 #include <Core/ShortCircuitFunctionEvaluation.h>
 #include <Core/StreamingHandleErrorMode.h>
 #include <Formats/FormatSettings.h>
-#include <IO/ReadSettings.h>
+#include <IO/DistributedCacheLogMode.h>
+#include <IO/DistributedCachePoolBehaviourOnLimit.h>
+#include <IO/ReadMethod.h>
 #include <Parsers/IdentifierQuotingStyle.h>
 #include <QueryPipeline/SizeLimits.h>
 #include <Common/ShellCommandSettings.h>
--- a/src/Core/UUID.h
+++ b/src/Core/UUID.h
@ -64,6 +64,9 @@ namespace UUIDHelpers
    /// Generate random UUID.
    UUID generateV4();

+    /// Generate UUID from hash of a string.
+    UUID makeUUIDv4FromHash(const String & string);
+
    constexpr size_t HighBytes = (std::endian::native == std::endian::little) ? 0 : 1;
    constexpr size_t LowBytes = (std::endian::native == std::endian::little) ? 1 : 0;

--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@ -85,6 +85,7 @@ namespace ErrorCodes
    extern const int NO_ACTIVE_REPLICAS;
    extern const int CANNOT_GET_REPLICATED_DATABASE_SNAPSHOT;
    extern const int CANNOT_RESTORE_TABLE;
+    extern const int QUERY_IS_PROHIBITED;
    extern const int SUPPORT_IS_DISABLED;
 }

@ -1057,6 +1058,9 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex
 {
    waitDatabaseStarted();

+    if (!DatabaseCatalog::instance().canPerformReplicatedDDLQueries())
+        throw Exception(ErrorCodes::QUERY_IS_PROHIBITED, "Replicated DDL queries are disabled");
+
    if (query_context->getCurrentTransaction() && query_context->getSettingsRef()[Setting::throw_on_unsupported_query_inside_transaction])
        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Distributed DDL queries inside transactions are not supported");

@ -1237,14 +1241,16 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep
        String query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Ordinary", backQuoteIfNeed(to_db_name));
        auto query_context = Context::createCopy(getContext());
        query_context->setSetting("allow_deprecated_database_ordinary", 1);
-        executeQuery(query, query_context, QueryFlags{.internal = true});
+        query_context->setSetting("cloud_mode", false);
+        executeQuery(query, query_context, QueryFlags{ .internal = true });

        /// But we want to avoid discarding UUID of ReplicatedMergeTree tables, because it will not work
        /// if zookeeper_path contains {uuid} macro. Replicated database do not recreate replicated tables on recovery,
        /// so it's ok to save UUID of replicated table.
        query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Atomic", backQuoteIfNeed(to_db_name_replicated));
        query_context = Context::createCopy(getContext());
-        executeQuery(query, query_context, QueryFlags{.internal = true});
+        query_context->setSetting("cloud_mode", false);
+        executeQuery(query, query_context, QueryFlags{ .internal = true });
    }

    size_t moved_tables = 0;
@ -1634,7 +1640,7 @@ void DatabaseReplicated::dropTable(ContextPtr local_context, const String & tabl
    auto table = tryGetTable(table_name, getContext());
    if (!table)
        throw Exception(ErrorCodes::LOGICAL_ERROR, "Table {} doesn't exist", table_name);
-    if (table->getName() == "MaterializedView" || table->getName() == "WindowView")
+    if (table->getName() == "MaterializedView" || table->getName() == "WindowView" || table->getName() == "SharedSet" || table->getName() == "SharedJoin")
    {
        /// Avoid recursive locking of metadata_mutex
        table->dropInnerTableIfAny(sync, local_context);
--- a/src/Databases/enableAllExperimentalSettings.cpp
+++ b/src/Databases/enableAllExperimentalSettings.cpp
@ -43,6 +43,8 @@ void enableAllExperimentalSettings(ContextMutablePtr context)
    context->setSetting("enable_zstd_qat_codec", 1);
    context->setSetting("allow_create_index_without_type", 1);
    context->setSetting("allow_experimental_s3queue", 1);
+
+    /// clickhouse-private settings
    context->setSetting("allow_experimental_shared_set_join", 1);
 }

--- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp
+++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp
@ -26,6 +26,9 @@ namespace DB
 namespace Setting
 {
    extern const SettingsSeconds max_execution_time;
+
+    /// Cloud only
+    extern const SettingsBool cloud_mode;
 }

 namespace ErrorCodes
@ -33,6 +36,7 @@ namespace ErrorCodes
    extern const int LOGICAL_ERROR;
    extern const int DICTIONARY_ACCESS_DENIED;
    extern const int UNSUPPORTED_METHOD;
+    extern const int SUPPORT_IS_DISABLED;
 }

 ExecutablePoolDictionarySource::ExecutablePoolDictionarySource(
@ -192,6 +196,9 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory)
                                 const std::string & /* default_database */,
                                 bool created_from_ddl) -> DictionarySourcePtr
    {
+        if (global_context->getSettingsRef()[Setting::cloud_mode])
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Dictionary source of type `executable pool` is disabled");
+
        if (dict_struct.has_expressions)
            throw Exception(ErrorCodes::LOGICAL_ERROR, "Dictionary source of type `executable_pool` does not support attribute expressions");

--- a/src/Dictionaries/RedisDictionarySource.cpp
+++ b/src/Dictionaries/RedisDictionarySource.cpp
@ -29,7 +29,6 @@ namespace DB
                                    ContextPtr global_context,
                                    const std::string & /* default_database */,
                                    bool /* created_from_ddl */) -> DictionarySourcePtr {
-
            auto redis_config_prefix = config_prefix + ".redis";

            auto host = config.getString(redis_config_prefix + ".host");
--- a/src/Dictionaries/XDBCDictionarySource.cpp
+++ b/src/Dictionaries/XDBCDictionarySource.cpp
@ -28,6 +28,9 @@ namespace Setting
 {
    extern const SettingsSeconds http_receive_timeout;
    extern const SettingsBool odbc_bridge_use_connection_pooling;
+
+    /// Cloud only
+    extern const SettingsBool cloud_mode;
 }

 namespace ErrorCodes
@ -242,6 +245,9 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory)
                                   ContextPtr global_context,
                                   const std::string & /* default_database */,
                                   bool /* check_config */) -> DictionarySourcePtr {
+
+        if (global_context->getSettingsRef()[Setting::cloud_mode])
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Dictionary source of type `odbc` is disabled");
 #if USE_ODBC
        BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<ODBCBridgeMixin>>(
            global_context,
--- a/src/Disks/DiskEncrypted.h
+++ b/src/Disks/DiskEncrypted.h
@ -313,6 +313,8 @@ public:
            return std::make_shared<FakeDiskTransaction>(*this);
        }

+        /// Need to overwrite explicetly because this disk change
+        /// a lot of "delegate" methods.
        return createEncryptedTransaction();
    }

--- a/src/Disks/DiskEncryptedTransaction.cpp
+++ b/src/Disks/DiskEncryptedTransaction.cpp
@ -1,6 +1,5 @@
 #include <Disks/DiskEncryptedTransaction.h>

-
 #if USE_SSL
 #include <IO/FileEncryptionCommon.h>
 #include <Common/Exception.h>
--- a/src/Disks/DiskType.h
+++ b/src/Disks/DiskType.h
@ -27,9 +27,11 @@ enum class MetadataStorageType : uint8_t
 {
    None,
    Local,
+    Keeper,
    Plain,
    PlainRewritable,
    StaticWeb,
+    Memory,
 };

 MetadataStorageType metadataTypeFromString(const String & type);
--- a/src/Disks/IDisk.h
+++ b/src/Disks/IDisk.h
@ -497,7 +497,7 @@ public:


 protected:
-    friend class DiskDecorator;
+    friend class DiskReadOnlyWrapper;

    const String name;

@ -580,6 +580,7 @@ inline String directoryPath(const String & path)
    return fs::path(path).parent_path() / "";
 }

+
 }

 template <>
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
@ -21,7 +21,7 @@ namespace ErrorCodes
 size_t chooseBufferSizeForRemoteReading(const DB::ReadSettings & settings, size_t file_size)
 {
    /// Only when cache is used we could download bigger portions of FileSegments than what we actually gonna read within particular task.
-    if (!settings.enable_filesystem_cache)
+    if (!settings.enable_filesystem_cache && !settings.read_through_distributed_cache)
        return settings.remote_fs_buffer_size;

    /// Buffers used for prefetch and pre-download better to have enough size, but not bigger than the whole file.
--- a/src/Disks/IO/createReadBufferFromFileBase.h
+++ b/src/Disks/IO/createReadBufferFromFileBase.h
@ -1,13 +1,13 @@
 #pragma once

-#include <IO/ReadBufferFromFileBase.h>
-#include <IO/ReadSettings.h>
-#include <string>
 #include <memory>
+#include <string>
+#include <IO/ReadBufferFromFileBase.h>


 namespace DB
 {
+struct ReadSettings;

 /** Create an object to read data from a file.
  *
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@ -1,5 +1,5 @@
 #include <Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h>
-#include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include <Disks/ObjectStorages/InMemoryDirectoryPathMap.h>

 #include <Common/SharedLockGuard.h>
 #include <Common/getRandomASCIIString.h>
@ -11,7 +11,7 @@
 namespace DB
 {

-CommonPathPrefixKeyGenerator::CommonPathPrefixKeyGenerator(String key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_)
+CommonPathPrefixKeyGenerator::CommonPathPrefixKeyGenerator(String key_prefix_, std::weak_ptr<InMemoryDirectoryPathMap> path_map_)
    : storage_key_prefix(key_prefix_), path_map(std::move(path_map_))
 {
 }
@ -59,7 +59,7 @@ std::tuple<std::string, std::vector<std::string>> CommonPathPrefixKeyGenerator::
        if (it != ptr->map.end())
        {
            std::vector<std::string> vec(std::make_move_iterator(dq.begin()), std::make_move_iterator(dq.end()));
-            return std::make_tuple(it->second, std::move(vec));
+            return std::make_tuple(it->second.path, std::move(vec));
        }

        if (!p.filename().empty())
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
@ -20,13 +20,13 @@ namespace DB
 /// The key generator ensures that the original directory hierarchy is
 /// preserved, which is required for the MergeTree family.

-struct InMemoryPathMap;
+struct InMemoryDirectoryPathMap;
 class CommonPathPrefixKeyGenerator : public IObjectStorageKeysGenerator
 {
 public:
    /// Local to remote path map. Leverages filesystem::path comparator for paths.

-    explicit CommonPathPrefixKeyGenerator(String key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_);
+    explicit CommonPathPrefixKeyGenerator(String key_prefix_, std::weak_ptr<InMemoryDirectoryPathMap> path_map_);

    ObjectStorageKey generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const override;

@ -36,7 +36,7 @@ private:

    const String storage_key_prefix;

-    std::weak_ptr<InMemoryPathMap> path_map;
+    std::weak_ptr<InMemoryDirectoryPathMap> path_map;
 };

 }
--- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h
+++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h
@ -56,6 +56,8 @@ public:

    void deserialize(ReadBuffer & buf);
    void deserializeFromString(const std::string & data);
+    /// This method was deleted from public fork recently by Azat
+    void createFromSingleObject(ObjectStorageKey object_key, size_t bytes_size, size_t ref_count_, bool is_read_only_);

    void serialize(WriteBuffer & buf, bool sync) const;
    std::string serializeToString() const;
--- a/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp
@ -1,5 +1,5 @@
 #include "FlatDirectoryStructureKeyGenerator.h"
-#include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include <Disks/ObjectStorages/InMemoryDirectoryPathMap.h>
 #include "Common/ObjectStorageKey.h"
 #include <Common/SharedLockGuard.h>
 #include <Common/SharedMutex.h>
@ -12,7 +12,8 @@
 namespace DB
 {

-FlatDirectoryStructureKeyGenerator::FlatDirectoryStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_)
+FlatDirectoryStructureKeyGenerator::FlatDirectoryStructureKeyGenerator(
+    String storage_key_prefix_, std::weak_ptr<InMemoryDirectoryPathMap> path_map_)
    : storage_key_prefix(storage_key_prefix_), path_map(std::move(path_map_))
 {
 }
@ -31,11 +32,11 @@ ObjectStorageKey FlatDirectoryStructureKeyGenerator::generate(const String & pat
        SharedLockGuard lock(ptr->mutex);
        auto it = ptr->map.find(p);
        if (it != ptr->map.end())
-            return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, it->second);
+            return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, it->second.path);

        it = ptr->map.find(directory);
        if (it != ptr->map.end())
-            remote_path = it->second;
+            remote_path = it->second.path;
    }
    constexpr size_t part_size = 32;
    std::filesystem::path key = remote_path.has_value() ? *remote_path
--- a/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.h
+++ b/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.h
@ -6,18 +6,18 @@
 namespace DB
 {

-struct InMemoryPathMap;
+struct InMemoryDirectoryPathMap;
 class FlatDirectoryStructureKeyGenerator : public IObjectStorageKeysGenerator
 {
 public:
-    explicit FlatDirectoryStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_);
+    explicit FlatDirectoryStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryDirectoryPathMap> path_map_);

    ObjectStorageKey generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const override;

 private:
    const String storage_key_prefix;

-    std::weak_ptr<InMemoryPathMap> path_map;
+    std::weak_ptr<InMemoryDirectoryPathMap> path_map;
 };

 }
--- a/src/Disks/ObjectStorages/IMetadataStorage.h
+++ b/src/Disks/ObjectStorages/IMetadataStorage.h
@ -1,6 +1,7 @@
 #pragma once

 #include <memory>
+#include <optional>
 #include <vector>
 #include <unordered_map>
 #include <Poco/Timestamp.h>
@ -197,6 +198,13 @@ public:

    virtual Poco::Timestamp getLastModified(const std::string & path) const = 0;

+    virtual std::optional<Poco::Timestamp> getLastModifiedIfExists(const std::string & path) const
+    {
+        if (existsFileOrDirectory(path))
+            return getLastModified(path);
+        return std::nullopt;
+    }
+
    virtual time_t getLastChanged(const std::string & /* path */) const
    {
        throwNotImplemented();
--- a/src/Disks/ObjectStorages/InMemoryDirectoryPathMap.h
+++ b/src/Disks/ObjectStorages/InMemoryDirectoryPathMap.h
@ -2,14 +2,17 @@

 #include <filesystem>
 #include <map>
+#include <optional>
+#include <shared_mutex>
 #include <base/defines.h>
+#include <Common/SharedLockGuard.h>
 #include <Common/SharedMutex.h>

 namespace DB
 {


-struct InMemoryPathMap
+struct InMemoryDirectoryPathMap
 {
    struct PathComparator
    {
@ -22,8 +25,27 @@ struct InMemoryPathMap
            return path1 < path2;
        }
    };
-    /// Local -> Remote path.
-    using Map = std::map<std::filesystem::path, std::string, PathComparator>;
+    struct RemotePathInfo
+    {
+        std::string path;
+        time_t last_modified = 0;
+    };
+
+    using Map = std::map<std::filesystem::path, RemotePathInfo, PathComparator>;
+
+    std::optional<RemotePathInfo> getRemotePathInfoIfExists(const std::string & path)
+    {
+        auto base_path = path;
+        if (base_path.ends_with('/'))
+            base_path.pop_back();
+
+        SharedLockGuard lock(mutex);
+        auto it = map.find(base_path);
+        if (it == map.end())
+            return std::nullopt;
+        return it->second;
+    }
+
    mutable SharedMutex mutex;

 #ifdef OS_LINUX
--- a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp
@ -116,7 +116,8 @@ void registerPlainMetadataStorage(MetadataStorageFactory & factory)
        ObjectStoragePtr object_storage) -> MetadataStoragePtr
    {
        auto key_compatibility_prefix = getObjectKeyCompatiblePrefix(*object_storage, config, config_prefix);
-        return std::make_shared<MetadataStorageFromPlainObjectStorage>(object_storage, key_compatibility_prefix, config.getUInt64(config_prefix + ".file_sizes_cache_size", 0));
+        return std::make_shared<MetadataStorageFromPlainObjectStorage>(
+            object_storage, key_compatibility_prefix, config.getUInt64(config_prefix + ".object_metadata_cache_size", 0));
    });
 }

@ -130,7 +131,8 @@ void registerPlainRewritableMetadataStorage(MetadataStorageFactory & factory)
           ObjectStoragePtr object_storage) -> MetadataStoragePtr
        {
            auto key_compatibility_prefix = getObjectKeyCompatiblePrefix(*object_storage, config, config_prefix);
-            return std::make_shared<MetadataStorageFromPlainRewritableObjectStorage>(object_storage, key_compatibility_prefix, config.getUInt64(config_prefix + ".file_sizes_cache_size", 0));
+            return std::make_shared<MetadataStorageFromPlainRewritableObjectStorage>(
+                object_storage, key_compatibility_prefix, config.getUInt64(config_prefix + ".object_metadata_cache_size", 0));
        });
 }

--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
@ -1,15 +1,22 @@
 #include "MetadataStorageFromPlainObjectStorage.h"
+
 #include <Disks/IDisk.h>
-#include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include <Disks/ObjectStorages/IObjectStorage.h>
+#include <Disks/ObjectStorages/InMemoryDirectoryPathMap.h>
 #include <Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h>
 #include <Disks/ObjectStorages/StaticDirectoryIterator.h>
 #include <Disks/ObjectStorages/StoredObject.h>
+#include <Common/ObjectStorageKey.h>
+#include <Common/SipHash.h>

 #include <Common/filesystemHelpers.h>

 #include <filesystem>
+#include <memory>
+#include <optional>
 #include <tuple>
 #include <unordered_set>
+#include <Poco/Timestamp.h>


 namespace DB
@ -30,12 +37,12 @@ std::filesystem::path normalizeDirectoryPath(const std::filesystem::path & path)

 }

-MetadataStorageFromPlainObjectStorage::MetadataStorageFromPlainObjectStorage(ObjectStoragePtr object_storage_, String storage_path_prefix_, size_t file_sizes_cache_size)
-    : object_storage(object_storage_)
-    , storage_path_prefix(std::move(storage_path_prefix_))
+MetadataStorageFromPlainObjectStorage::MetadataStorageFromPlainObjectStorage(
+    ObjectStoragePtr object_storage_, String storage_path_prefix_, size_t object_metadata_cache_size)
+    : object_storage(object_storage_), storage_path_prefix(std::move(storage_path_prefix_))
 {
-    if (file_sizes_cache_size)
-        file_sizes_cache.emplace(file_sizes_cache_size);
+    if (object_metadata_cache_size)
+        object_metadata_cache.emplace(object_metadata_cache_size);
 }

 MetadataTransactionPtr MetadataStorageFromPlainObjectStorage::createTransaction()
@ -82,28 +89,29 @@ uint64_t MetadataStorageFromPlainObjectStorage::getFileSize(const String & path)
 {
    if (auto res = getFileSizeIfExists(path))
        return *res;
-    throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File {} does not exist on plain object storage", path);
+    throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File {} does not exist on {}", path, object_storage->getName());
 }

 std::optional<uint64_t> MetadataStorageFromPlainObjectStorage::getFileSizeIfExists(const String & path) const
 {
-    auto get = [&] -> std::shared_ptr<uint64_t>
-    {
-        auto object_key = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
-        auto metadata = object_storage->tryGetObjectMetadata(object_key.serialize());
-        if (metadata)
-            return std::make_shared<uint64_t>(metadata->size_bytes);
-        return nullptr;
-    };
+    if (auto res = getObjectMetadataEntryWithCache(path))
+        return res->file_size;
+    return std::nullopt;
+}

-    std::shared_ptr<uint64_t> res;
-    if (file_sizes_cache)
-        res = file_sizes_cache->getOrSet(path, get).first;
-    else
-        res = get();
-
-    if (res)
+Poco::Timestamp MetadataStorageFromPlainObjectStorage::getLastModified(const std::string & path) const
+{
+    if (auto res = getLastModifiedIfExists(path))
        return *res;
+    else
+        throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File or directory {} does not exist on {}", path, object_storage->getName());
+}
+
+std::optional<Poco::Timestamp> MetadataStorageFromPlainObjectStorage::getLastModifiedIfExists(const std::string & path) const
+{
+    /// Since the plain object storage is used for backups only, return the current time.
+    if (existsFileOrDirectory(path))
+        return Poco::Timestamp{};
    return std::nullopt;
 }

@ -161,6 +169,31 @@ std::optional<StoredObjects> MetadataStorageFromPlainObjectStorage::getStorageOb
    return std::nullopt;
 }

+MetadataStorageFromPlainObjectStorage::ObjectMetadataEntryPtr
+MetadataStorageFromPlainObjectStorage::getObjectMetadataEntryWithCache(const std::string & path) const
+{
+    auto object_key = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
+    auto get = [&] -> ObjectMetadataEntryPtr
+    {
+        if (auto metadata = object_storage->tryGetObjectMetadata(object_key.serialize()))
+            return std::make_shared<ObjectMetadataEntry>(metadata->size_bytes, metadata->last_modified.epochTime());
+        return nullptr;
+    };
+
+    if (object_metadata_cache)
+    {
+        SipHash hash;
+        hash.update(object_key.serialize());
+        auto hash128 = hash.get128();
+        if (auto res = object_metadata_cache->get(hash128))
+            return res;
+        if (auto mapped = get())
+            return object_metadata_cache->getOrSet(hash128, [&] { return mapped; }).first;
+        return object_metadata_cache->get(hash128);
+    }
+    return get();
+}
+
 const IMetadataStorage & MetadataStorageFromPlainObjectStorageTransaction::getStorageForNonTransactionalReads() const
 {
    return metadata_storage;
@ -225,8 +258,17 @@ void MetadataStorageFromPlainObjectStorageTransaction::addBlobToMetadata(
    /// Noop, local metadata files is only one file, it is the metadata file itself.
 }

-UnlinkMetadataFileOperationOutcomePtr MetadataStorageFromPlainObjectStorageTransaction::unlinkMetadata(const std::string &)
+UnlinkMetadataFileOperationOutcomePtr MetadataStorageFromPlainObjectStorageTransaction::unlinkMetadata(const std::string & path)
 {
+    /// The record has become stale, remove it from cache.
+    if (metadata_storage.object_metadata_cache)
+    {
+        auto object_key = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
+        SipHash hash;
+        hash.update(object_key.serialize());
+        metadata_storage.object_metadata_cache->remove(hash.get128());
+    }
+
    /// No hardlinks, so will always remove file.
    return std::make_shared<UnlinkMetadataFileOperationOutcome>(UnlinkMetadataFileOperationOutcome{0});
 }
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
@ -1,21 +1,24 @@
 #pragma once

+#include <Core/Types.h>
 #include <Disks/IDisk.h>
 #include <Disks/ObjectStorages/IMetadataStorage.h>
-#include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include <Disks/ObjectStorages/InMemoryDirectoryPathMap.h>
 #include <Disks/ObjectStorages/MetadataOperationsHolder.h>
 #include <Disks/ObjectStorages/MetadataStorageTransactionState.h>
 #include <Common/CacheBase.h>

 #include <map>
+#include <memory>
 #include <string>
 #include <unordered_set>
+#include <Poco/Timestamp.h>


 namespace DB
 {

-struct InMemoryPathMap;
+struct InMemoryDirectoryPathMap;
 struct UnlinkMetadataFileOperationOutcome;
 using UnlinkMetadataFileOperationOutcomePtr = std::shared_ptr<UnlinkMetadataFileOperationOutcome>;

@ -33,16 +36,24 @@ class MetadataStorageFromPlainObjectStorage : public IMetadataStorage
 {
 private:
    friend class MetadataStorageFromPlainObjectStorageTransaction;
-    mutable std::optional<CacheBase<String, uint64_t>> file_sizes_cache;

 protected:
+    struct ObjectMetadataEntry
+    {
+        uint64_t file_size;
+        time_t last_modified;
+    };
+    using ObjectMetadataEntryPtr = std::shared_ptr<ObjectMetadataEntry>;
+
    ObjectStoragePtr object_storage;
-    String storage_path_prefix;
+    const String storage_path_prefix;
+
+    mutable std::optional<CacheBase<UInt128, ObjectMetadataEntry>> object_metadata_cache;

    mutable SharedMutex metadata_mutex;

 public:
-    MetadataStorageFromPlainObjectStorage(ObjectStoragePtr object_storage_, String storage_path_prefix_, size_t file_sizes_cache_size);
+    MetadataStorageFromPlainObjectStorage(ObjectStoragePtr object_storage_, String storage_path_prefix_, size_t object_metadata_cache_size);

    MetadataTransactionPtr createTransaction() override;

@ -66,11 +77,8 @@ public:
    StoredObjects getStorageObjects(const std::string & path) const override;
    std::optional<StoredObjects> getStorageObjectsIfExist(const std::string & path) const override;

-    Poco::Timestamp getLastModified(const std::string & /* path */) const override
-    {
-        /// Required by MergeTree
-        return {};
-    }
+    Poco::Timestamp getLastModified(const std::string & path) const override;
+    std::optional<Poco::Timestamp> getLastModifiedIfExists(const String & path) const override;

    uint32_t getHardlinkCount(const std::string & /* path */) const override
    {
@ -85,7 +93,9 @@ protected:
    virtual std::string getMetadataKeyPrefix() const { return object_storage->getCommonKeyPrefix(); }

    /// Returns a map of virtual filesystem paths to paths in the object storage.
-    virtual std::shared_ptr<InMemoryPathMap> getPathMap() const { throwNotImplemented(); }
+    virtual std::shared_ptr<InMemoryDirectoryPathMap> getPathMap() const { throwNotImplemented(); }
+
+    ObjectMetadataEntryPtr getObjectMetadataEntryWithCache(const std::string & path) const;
 };

 class MetadataStorageFromPlainObjectStorageTransaction final : public IMetadataTransaction, private MetadataOperationsHolder
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@ -1,8 +1,9 @@
 #include "MetadataStorageFromPlainObjectStorageOperations.h"
-#include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include <Disks/ObjectStorages/InMemoryDirectoryPathMap.h>

 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <Poco/Timestamp.h>
 #include <Common/Exception.h>
 #include <Common/SharedLockGuard.h>
 #include <Common/logger_useful.h>
@ -30,7 +31,10 @@ ObjectStorageKey createMetadataObjectKey(const std::string & object_key_prefix,
 }

 MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
-    std::filesystem::path && path_, InMemoryPathMap & path_map_, ObjectStoragePtr object_storage_, const std::string & metadata_key_prefix_)
+    std::filesystem::path && path_,
+    InMemoryDirectoryPathMap & path_map_,
+    ObjectStoragePtr object_storage_,
+    const std::string & metadata_key_prefix_)
    : path(std::move(path_))
    , path_map(path_map_)
    , object_storage(object_storage_)
@ -71,7 +75,8 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
    {
        std::lock_guard lock(path_map.mutex);
        auto & map = path_map.map;
-        [[maybe_unused]] auto result = map.emplace(base_path, object_key_prefix);
+        [[maybe_unused]] auto result
+            = map.emplace(base_path, InMemoryDirectoryPathMap::RemotePathInfo{object_key_prefix, Poco::Timestamp{}.epochTime()});
        chassert(result.second);
    }
    auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
@ -109,7 +114,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::un
 MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFromPlainObjectStorageMoveDirectoryOperation(
    std::filesystem::path && path_from_,
    std::filesystem::path && path_to_,
-    InMemoryPathMap & path_map_,
+    InMemoryDirectoryPathMap & path_map_,
    ObjectStoragePtr object_storage_,
    const std::string & metadata_key_prefix_)
    : path_from(std::move(path_from_))
@ -139,7 +144,7 @@ std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMo
            throw Exception(
                ErrorCodes::FILE_ALREADY_EXISTS, "Metadata object for the new (destination) path '{}' already exists", new_path);

-        remote_path = expected_it->second;
+        remote_path = expected_it->second.path;
    }

    auto metadata_object_key = createMetadataObjectKey(remote_path, metadata_key_prefix);
@ -190,6 +195,7 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
        auto & map = path_map.map;
        [[maybe_unused]] auto result = map.emplace(base_path_to, map.extract(base_path_from).mapped());
        chassert(result.second);
+        result.first->second.last_modified = Poco::Timestamp{}.epochTime();
    }

    write_finalized = true;
@ -213,7 +219,10 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq
 }

 MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
-    std::filesystem::path && path_, InMemoryPathMap & path_map_, ObjectStoragePtr object_storage_, const std::string & metadata_key_prefix_)
+    std::filesystem::path && path_,
+    InMemoryDirectoryPathMap & path_map_,
+    ObjectStoragePtr object_storage_,
+    const std::string & metadata_key_prefix_)
    : path(std::move(path_)), path_map(path_map_), object_storage(object_storage_), metadata_key_prefix(metadata_key_prefix_)
 {
    chassert(path.string().ends_with('/'));
@ -229,7 +238,7 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std:
        auto path_it = map.find(base_path);
        if (path_it == map.end())
            return;
-        key_prefix = path_it->second;
+        key_prefix = path_it->second.path;
    }

    LOG_TRACE(getLogger("MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation"), "Removing directory '{}'", path);
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
@ -1,7 +1,7 @@
 #pragma once

 #include <Disks/ObjectStorages/IMetadataOperation.h>
-#include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include <Disks/ObjectStorages/InMemoryDirectoryPathMap.h>
 #include <Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h>

 #include <filesystem>
@ -14,7 +14,7 @@ class MetadataStorageFromPlainObjectStorageCreateDirectoryOperation final : publ
 {
 private:
    std::filesystem::path path;
-    InMemoryPathMap & path_map;
+    InMemoryDirectoryPathMap & path_map;
    ObjectStoragePtr object_storage;
    const std::string metadata_key_prefix;
    const std::string object_key_prefix;
@ -26,7 +26,7 @@ public:
    MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
        /// path_ must end with a trailing '/'.
        std::filesystem::path && path_,
-        InMemoryPathMap & path_map_,
+        InMemoryDirectoryPathMap & path_map_,
        ObjectStoragePtr object_storage_,
        const std::string & metadata_key_prefix_);

@ -39,7 +39,7 @@ class MetadataStorageFromPlainObjectStorageMoveDirectoryOperation final : public
 private:
    std::filesystem::path path_from;
    std::filesystem::path path_to;
-    InMemoryPathMap & path_map;
+    InMemoryDirectoryPathMap & path_map;
    ObjectStoragePtr object_storage;
    const std::string metadata_key_prefix;

@ -54,7 +54,7 @@ public:
        /// Both path_from_ and path_to_ must end with a trailing '/'.
        std::filesystem::path && path_from_,
        std::filesystem::path && path_to_,
-        InMemoryPathMap & path_map_,
+        InMemoryDirectoryPathMap & path_map_,
        ObjectStoragePtr object_storage_,
        const std::string & metadata_key_prefix_);

@ -68,7 +68,7 @@ class MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation final : publ
 private:
    std::filesystem::path path;

-    InMemoryPathMap & path_map;
+    InMemoryDirectoryPathMap & path_map;
    ObjectStoragePtr object_storage;
    const std::string metadata_key_prefix;

@ -79,7 +79,7 @@ public:
    MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
        /// path_ must end with a trailing '/'.
        std::filesystem::path && path_,
-        InMemoryPathMap & path_map_,
+        InMemoryDirectoryPathMap & path_map_,
        ObjectStoragePtr object_storage_,
        const std::string & metadata_key_prefix_);

--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@ -1,12 +1,17 @@
 #include <Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.h>
-#include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include <Disks/ObjectStorages/InMemoryDirectoryPathMap.h>
 #include <Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h>
 #include <Disks/ObjectStorages/ObjectStorageIterator.h>

+#include <cstddef>
+#include <exception>
+#include <optional>
 #include <unordered_set>
 #include <IO/ReadHelpers.h>
 #include <IO/S3Common.h>
 #include <IO/SharedThreadPools.h>
+#include <Poco/Timestamp.h>
+#include "Common/Exception.h"
 #include <Common/SharedLockGuard.h>
 #include <Common/SharedMutex.h>
 #include <Common/logger_useful.h>
@ -40,10 +45,10 @@ std::string getMetadataKeyPrefix(ObjectStoragePtr object_storage)
        : metadata_key_prefix;
 }

-std::shared_ptr<InMemoryPathMap> loadPathPrefixMap(const std::string & metadata_key_prefix, ObjectStoragePtr object_storage)
+std::shared_ptr<InMemoryDirectoryPathMap> loadPathPrefixMap(const std::string & metadata_key_prefix, ObjectStoragePtr object_storage)
 {
-    auto result = std::make_shared<InMemoryPathMap>();
-    using Map = InMemoryPathMap::Map;
+    auto result = std::make_shared<InMemoryDirectoryPathMap>();
+    using Map = InMemoryDirectoryPathMap::Map;

    ThreadPool & pool = getIOThreadPool().get();
    ThreadPoolCallbackRunnerLocal<void> runner(pool, "PlainRWMetaLoad");
@ -73,17 +78,24 @@ std::shared_ptr<InMemoryPathMap> loadPathPrefixMap(const std::string & metadata_

                StoredObject object{path};
                String local_path;
+                Poco::Timestamp last_modified{};

                try
                {
                    auto read_buf = object_storage->readObject(object, settings);
                    readStringUntilEOF(local_path, *read_buf);
+                    auto object_metadata = object_storage->tryGetObjectMetadata(path);
+                    /// It ok if a directory was removed just now.
+                    /// We support attaching a filesystem that is concurrently modified by someone else.
+                    if (!object_metadata)
+                        return;
+                    /// Assuming that local and the object storage clocks are synchronized.
+                    last_modified = object_metadata->last_modified;
                }
 #if USE_AWS_S3
                catch (const S3Exception & e)
                {
                    /// It is ok if a directory was removed just now.
-                    /// We support attaching a filesystem that is concurrently modified by someone else.
                    if (e.getS3ErrorCode() == Aws::S3::S3Errors::NO_SUCH_KEY)
                        return;
                    throw;
@ -101,18 +113,19 @@ std::shared_ptr<InMemoryPathMap> loadPathPrefixMap(const std::string & metadata_
                std::pair<Map::iterator, bool> res;
                {
                    std::lock_guard lock(result->mutex);
-                    res = result->map.emplace(std::filesystem::path(local_path).parent_path(), remote_path.parent_path());
+                    res = result->map.emplace(
+                        std::filesystem::path(local_path).parent_path(),
+                        InMemoryDirectoryPathMap::RemotePathInfo{remote_path.parent_path(), last_modified.epochTime()});
                }

                /// This can happen if table replication is enabled, then the same local path is written
                /// in `prefix.path` of each replica.
-                /// TODO: should replicated tables (e.g., RMT) be explicitly disallowed?
                if (!res.second)
                    LOG_WARNING(
                        log,
                        "The local path '{}' is already mapped to a remote path '{}', ignoring: '{}'",
                        local_path,
-                        res.first->second,
+                        res.first->second.path,
                        remote_path.parent_path().string());
            });
    }
@ -132,7 +145,7 @@ void getDirectChildrenOnDiskImpl(
    const std::string & storage_key,
    const RelativePathsWithMetadata & remote_paths,
    const std::string & local_path,
-    const InMemoryPathMap & path_map,
+    const InMemoryDirectoryPathMap & path_map,
    std::unordered_set<std::string> & result)
 {
    /// Directories are retrieved from the in-memory path map.
@ -180,8 +193,8 @@ void getDirectChildrenOnDiskImpl(
 }

 MetadataStorageFromPlainRewritableObjectStorage::MetadataStorageFromPlainRewritableObjectStorage(
-    ObjectStoragePtr object_storage_, String storage_path_prefix_, size_t file_sizes_cache_size)
-    : MetadataStorageFromPlainObjectStorage(object_storage_, storage_path_prefix_, file_sizes_cache_size)
+    ObjectStoragePtr object_storage_, String storage_path_prefix_, size_t object_metadata_cache_size)
+    : MetadataStorageFromPlainObjectStorage(object_storage_, storage_path_prefix_, object_metadata_cache_size)
    , metadata_key_prefix(DB::getMetadataKeyPrefix(object_storage))
    , path_map(loadPathPrefixMap(metadata_key_prefix, object_storage))
 {
@ -215,9 +228,7 @@ bool MetadataStorageFromPlainRewritableObjectStorage::existsFileOrDirectory(cons
    if (existsDirectory(path))
        return true;

-    ObjectStorageKey object_key = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
-    StoredObject object(object_key.serialize(), path);
-    return object_storage->exists(object);
+    return getObjectMetadataEntryWithCache(path) != nullptr;
 }

 bool MetadataStorageFromPlainRewritableObjectStorage::existsFile(const std::string & path) const
@ -225,19 +236,12 @@ bool MetadataStorageFromPlainRewritableObjectStorage::existsFile(const std::stri
    if (existsDirectory(path))
        return false;

-    ObjectStorageKey object_key = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
-    StoredObject object(object_key.serialize(), path);
-    return object_storage->exists(object);
+    return getObjectMetadataEntryWithCache(path) != nullptr;
 }

 bool MetadataStorageFromPlainRewritableObjectStorage::existsDirectory(const std::string & path) const
 {
-    auto base_path = path;
-    if (base_path.ends_with('/'))
-        base_path.pop_back();
-
-    SharedLockGuard lock(path_map->mutex);
-    return path_map->map.find(base_path) != path_map->map.end();
+    return path_map->getRemotePathInfoIfExists(path) != std::nullopt;
 }

 std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDirectory(const std::string & path) const
@ -255,6 +259,18 @@ std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDi
    return std::vector<std::string>(std::make_move_iterator(directories.begin()), std::make_move_iterator(directories.end()));
 }

+std::optional<Poco::Timestamp> MetadataStorageFromPlainRewritableObjectStorage::getLastModifiedIfExists(const String & path) const
+{
+    /// Path corresponds to a directory.
+    if (auto remote = path_map->getRemotePathInfoIfExists(path))
+        return Poco::Timestamp::fromEpochTime(remote->last_modified);
+
+    /// A file.
+    if (auto res = getObjectMetadataEntryWithCache(path))
+        return Poco::Timestamp::fromEpochTime(res->last_modified);
+    return std::nullopt;
+}
+
 void MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
    const std::string & storage_key,
    const RelativePathsWithMetadata & remote_paths,
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
@ -13,21 +13,28 @@ class MetadataStorageFromPlainRewritableObjectStorage final : public MetadataSto
 {
 private:
    const std::string metadata_key_prefix;
-    std::shared_ptr<InMemoryPathMap> path_map;
+    std::shared_ptr<InMemoryDirectoryPathMap> path_map;

 public:
-    MetadataStorageFromPlainRewritableObjectStorage(ObjectStoragePtr object_storage_, String storage_path_prefix_, size_t file_sizes_cache_size);
+    MetadataStorageFromPlainRewritableObjectStorage(
+        ObjectStoragePtr object_storage_, String storage_path_prefix_, size_t object_metadata_cache_size);
    ~MetadataStorageFromPlainRewritableObjectStorage() override;

    MetadataStorageType getType() const override { return MetadataStorageType::PlainRewritable; }
+
    bool existsFile(const std::string & path) const override;
+
    bool existsDirectory(const std::string & path) const override;
+
    bool existsFileOrDirectory(const std::string & path) const override;
+
    std::vector<std::string> listDirectory(const std::string & path) const override;

+    std::optional<Poco::Timestamp> getLastModifiedIfExists(const String & path) const override;
+
 protected:
    std::string getMetadataKeyPrefix() const override { return metadata_key_prefix; }
-    std::shared_ptr<InMemoryPathMap> getPathMap() const override { return path_map; }
+    std::shared_ptr<InMemoryDirectoryPathMap> getPathMap() const override { return path_map; }
    void getDirectChildrenOnDisk(
        const std::string & storage_key,
        const RelativePathsWithMetadata & remote_paths,
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@ -9,6 +9,7 @@
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 #include <Disks/IO/AsynchronousBoundedReadBuffer.h>
 #include <Disks/IO/ThreadPoolRemoteFSReader.h>
+#include <Disks/IO/getThreadPoolReader.h>
 #include <IO/WriteBufferFromS3.h>
 #include <IO/ReadBufferFromS3.h>
 #include <IO/S3/getObjectInfo.h>
@ -195,7 +196,7 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
    if (mode != WriteMode::Rewrite)
        throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 doesn't support append to files");

-    S3::RequestSettings request_settings = s3_settings.get()->request_settings;
+    S3::S3RequestSettings request_settings = s3_settings.get()->request_settings;
    /// NOTE: For background operations settings are not propagated from session or query. They are taken from
    /// default user's .xml config. It's obscure and unclear behavior. For them it's always better
    /// to rely on settings from disk.
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
@ -20,8 +20,8 @@ struct S3ObjectStorageSettings
    S3ObjectStorageSettings() = default;

    S3ObjectStorageSettings(
-        const S3::RequestSettings & request_settings_,
-        const S3::AuthSettings & auth_settings_,
+        const S3::S3RequestSettings & request_settings_,
+        const S3::S3AuthSettings & auth_settings_,
        uint64_t min_bytes_for_seek_,
        int32_t list_object_keys_size_,
        int32_t objects_chunk_size_to_delete_,
@ -34,8 +34,8 @@ struct S3ObjectStorageSettings
        , read_only(read_only_)
    {}

-    S3::RequestSettings request_settings;
-    S3::AuthSettings auth_settings;
+    S3::S3RequestSettings request_settings;
+    S3::S3AuthSettings auth_settings;

    uint64_t min_bytes_for_seek;
    int32_t list_object_keys_size;
--- a/src/Disks/ObjectStorages/S3/diskSettings.cpp
+++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp
@ -33,6 +33,27 @@ namespace Setting
    extern const SettingsUInt64 s3_retry_attempts;
 }

+namespace S3AuthSetting
+{
+    extern const S3AuthSettingsString access_key_id;
+    extern const S3AuthSettingsUInt64 connect_timeout_ms;
+    extern const S3AuthSettingsBool disable_checksum;
+    extern const S3AuthSettingsUInt64 expiration_window_seconds;
+    extern const S3AuthSettingsBool gcs_issue_compose_request;
+    extern const S3AuthSettingsUInt64 http_keep_alive_max_requests;
+    extern const S3AuthSettingsUInt64 http_keep_alive_timeout;
+    extern const S3AuthSettingsUInt64 max_connections;
+    extern const S3AuthSettingsBool no_sign_request;
+    extern const S3AuthSettingsString region;
+    extern const S3AuthSettingsUInt64 request_timeout_ms;
+    extern const S3AuthSettingsString secret_access_key;
+    extern const S3AuthSettingsString server_side_encryption_customer_key_base64;
+    extern const S3AuthSettingsString session_token;
+    extern const S3AuthSettingsBool use_adaptive_timeouts;
+    extern const S3AuthSettingsBool use_environment_credentials;
+    extern const S3AuthSettingsBool use_insecure_imds_request;
+}
+
 namespace ErrorCodes
 {
 extern const int NO_ELEMENTS_IN_CONFIG;
@ -47,8 +68,8 @@ std::unique_ptr<S3ObjectStorageSettings> getSettings(
 {
    const auto & settings = context->getSettingsRef();

-    auto auth_settings = S3::AuthSettings(config, settings, config_prefix);
-    auto request_settings = S3::RequestSettings(config, settings, config_prefix, "s3_", validate_settings);
+    auto auth_settings = S3::S3AuthSettings(config, settings, config_prefix);
+    auto request_settings = S3::S3RequestSettings(config, settings, config_prefix, "s3_", validate_settings);

    request_settings.proxy_resolver = DB::ProxyConfigurationResolverProvider::getFromOldSettingsFormat(
        ProxyConfiguration::protocolFromString(S3::URI(endpoint).uri.getScheme()), config_prefix, config);
@ -85,7 +106,7 @@ std::unique_ptr<S3::Client> getClient(
    const auto & request_settings = settings.request_settings;

    const bool is_s3_express_bucket = S3::isS3ExpressEndpoint(url.endpoint);
-    if (is_s3_express_bucket && auth_settings.region.value.empty())
+    if (is_s3_express_bucket && auth_settings[S3AuthSetting::region].value.empty())
    {
        throw Exception(
            ErrorCodes::NO_ELEMENTS_IN_CONFIG,
@ -107,7 +128,7 @@ std::unique_ptr<S3::Client> getClient(
        enable_s3_requests_logging = local_settings[Setting::enable_s3_requests_logging];

    S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
-        auth_settings.region,
+        auth_settings[S3AuthSetting::region],
        context->getRemoteHostFilter(),
        s3_max_redirects,
        s3_retry_attempts,
@ -117,14 +138,14 @@ std::unique_ptr<S3::Client> getClient(
        request_settings.put_request_throttler,
        url.uri.getScheme());

-    client_configuration.connectTimeoutMs = auth_settings.connect_timeout_ms;
-    client_configuration.requestTimeoutMs = auth_settings.request_timeout_ms;
-    client_configuration.maxConnections = static_cast<uint32_t>(auth_settings.max_connections);
-    client_configuration.http_keep_alive_timeout = auth_settings.http_keep_alive_timeout;
-    client_configuration.http_keep_alive_max_requests = auth_settings.http_keep_alive_max_requests;
+    client_configuration.connectTimeoutMs = auth_settings[S3AuthSetting::connect_timeout_ms];
+    client_configuration.requestTimeoutMs = auth_settings[S3AuthSetting::request_timeout_ms];
+    client_configuration.maxConnections = static_cast<uint32_t>(auth_settings[S3AuthSetting::max_connections]);
+    client_configuration.http_keep_alive_timeout = auth_settings[S3AuthSetting::http_keep_alive_timeout];
+    client_configuration.http_keep_alive_max_requests = auth_settings[S3AuthSetting::http_keep_alive_max_requests];

    client_configuration.endpointOverride = url.endpoint;
-    client_configuration.s3_use_adaptive_timeouts = auth_settings.use_adaptive_timeouts;
+    client_configuration.s3_use_adaptive_timeouts = auth_settings[S3AuthSetting::use_adaptive_timeouts];

    if (request_settings.proxy_resolver)
    {
@ -137,28 +158,28 @@ std::unique_ptr<S3::Client> getClient(

    S3::ClientSettings client_settings{
        .use_virtual_addressing = url.is_virtual_hosted_style,
-        .disable_checksum = auth_settings.disable_checksum,
-        .gcs_issue_compose_request = auth_settings.gcs_issue_compose_request,
+        .disable_checksum = auth_settings[S3AuthSetting::disable_checksum],
+        .gcs_issue_compose_request = auth_settings[S3AuthSetting::gcs_issue_compose_request],
    };

    auto credentials_configuration = S3::CredentialsConfiguration
    {
-        auth_settings.use_environment_credentials,
-        auth_settings.use_insecure_imds_request,
-        auth_settings.expiration_window_seconds,
-        auth_settings.no_sign_request,
+        auth_settings[S3AuthSetting::use_environment_credentials],
+        auth_settings[S3AuthSetting::use_insecure_imds_request],
+        auth_settings[S3AuthSetting::expiration_window_seconds],
+        auth_settings[S3AuthSetting::no_sign_request],
    };

    return S3::ClientFactory::instance().create(
        client_configuration,
        client_settings,
-        auth_settings.access_key_id,
-        auth_settings.secret_access_key,
-        auth_settings.server_side_encryption_customer_key_base64,
+        auth_settings[S3AuthSetting::access_key_id],
+        auth_settings[S3AuthSetting::secret_access_key],
+        auth_settings[S3AuthSetting::server_side_encryption_customer_key_base64],
        auth_settings.server_side_encryption_kms_config,
        auth_settings.headers,
        credentials_configuration,
-        auth_settings.session_token);
+        auth_settings[S3AuthSetting::session_token]);
 }

 }
--- a/src/Functions/CountSubstringsImpl.h
+++ b/src/Functions/CountSubstringsImpl.h
@ -62,7 +62,7 @@ struct CountSubstringsImpl
        while (pos < end && end != (pos = searcher.search(pos, end - pos)))
        {
            /// Determine which index it refers to.
-            while (begin + haystack_offsets[i] <= pos)
+            while (i < input_rows_count - 1 && begin + haystack_offsets[i] <= pos)
                ++i;

            auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0;
@ -80,9 +80,10 @@ struct CountSubstringsImpl
                continue;
            }
            pos = begin + haystack_offsets[i];
-            ++i;

-            chassert(i < input_rows_count);
+            ++i;
+            if (i >= input_rows_count)
+                break; // Handle the end of the haystacks
        }
    }

--- a/src/Functions/IFunctionDateOrDateTime.h
+++ b/src/Functions/IFunctionDateOrDateTime.h
@ -22,13 +22,8 @@ namespace ErrorCodes
    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }

-template <typename Transform>
-class IFunctionDateOrDateTime : public IFunction
+class FunctionDateOrDateTimeBase : public IFunction
 {
-public:
-    static constexpr auto name = Transform::name;
-    String getName() const override { return name; }
-
    bool isVariadic() const override { return true; }

    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
@ -44,6 +39,46 @@ public:
        return true;
    }

+protected:
+    void checkArguments(const ColumnsWithTypeAndName & arguments, bool is_result_type_date_or_date32) const
+    {
+        if (arguments.size() == 1)
+        {
+            if (!isDateOrDate32OrDateTimeOrDateTime64(arguments[0].type))
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of argument of function {}. Should be Date, Date32, DateTime or DateTime64",
+                    arguments[0].type->getName(), getName());
+        }
+        else if (arguments.size() == 2)
+        {
+            if (!isDateOrDate32OrDateTimeOrDateTime64(arguments[0].type))
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of argument of function {}. Should be Date, Date32, DateTime or DateTime64",
+                    arguments[0].type->getName(), getName());
+            if (!isString(arguments[1].type))
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Function {} supports 1 or 2 arguments. The optional 2nd argument must be "
+                    "a constant string with a timezone name",
+                    getName());
+            if (isDateOrDate32(arguments[0].type) && is_result_type_date_or_date32)
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "The timezone argument of function {} is allowed only when the 1st argument has the type DateTime or DateTime64",
+                    getName());
+        }
+        else
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                "Number of arguments for function {} doesn't match: passed {}, should be 1 or 2",
+                getName(), arguments.size());
+    }
+};
+
+template <typename Transform>
+class IFunctionDateOrDateTime : public FunctionDateOrDateTimeBase
+{
+public:
+    static constexpr auto name = Transform::name;
+    String getName() const override { return name; }
+
    Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override
    {
        if constexpr (std::is_same_v<typename Transform::FactorTransform, ZeroTransform>)
@ -105,38 +140,6 @@ public:
                : is_not_monotonic;
        }
    }
-
-protected:
-    void checkArguments(const ColumnsWithTypeAndName & arguments, bool is_result_type_date_or_date32) const
-    {
-        if (arguments.size() == 1)
-        {
-            if (!isDateOrDate32OrDateTimeOrDateTime64(arguments[0].type))
-                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Illegal type {} of argument of function {}. Should be Date, Date32, DateTime or DateTime64",
-                    arguments[0].type->getName(), getName());
-        }
-        else if (arguments.size() == 2)
-        {
-            if (!isDateOrDate32OrDateTimeOrDateTime64(arguments[0].type))
-                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Illegal type {} of argument of function {}. Should be Date, Date32, DateTime or DateTime64",
-                    arguments[0].type->getName(), getName());
-            if (!isString(arguments[1].type))
-                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Function {} supports 1 or 2 arguments. The optional 2nd argument must be "
-                    "a constant string with a timezone name",
-                    getName());
-            if (isDateOrDate32(arguments[0].type) && is_result_type_date_or_date32)
-                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "The timezone argument of function {} is allowed only when the 1st argument has the type DateTime or DateTime64",
-                    getName());
-        }
-        else
-            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-                "Number of arguments for function {} doesn't match: passed {}, should be 1 or 2",
-                getName(), arguments.size());
-    }
 };

 }
--- a/src/Functions/UTCTimestampTransform.cpp
+++ b/src/Functions/UTCTimestampTransform.cpp
@ -27,7 +27,7 @@ namespace ErrorCodes

 namespace
 {
-    template <typename Name>
+    template <typename Name, bool toUTC>
    class UTCTimestampTransform : public IFunction
    {
    public:
@ -77,7 +77,7 @@ namespace
            if (!time_zone_const_col)
                throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of 2nd argument of function {}. Excepted const(String).", arg2.column->getName(), name);
            String time_zone_val = time_zone_const_col->getDataAt(0).toString();
-            const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC");
+            const DateLUTImpl & time_zone = DateLUT::instance(time_zone_val);
            if (WhichDataType(arg1.type).isDateTime())
            {
                const auto & date_time_col = checkAndGetColumn<ColumnDateTime>(*arg1.column);
@ -87,9 +87,11 @@ namespace
                for (size_t i = 0; i < input_rows_count; ++i)
                {
                    UInt32 date_time_val = date_time_col.getElement(i);
-                    LocalDateTime date_time(date_time_val, Name::to ? utc_time_zone : DateLUT::instance(time_zone_val));
-                    time_t time_val = date_time.to_time_t(Name::from ? utc_time_zone : DateLUT::instance(time_zone_val));
-                    result_data[i] = static_cast<UInt32>(time_val);
+                    auto time_zone_offset = time_zone.timezoneOffset(date_time_val);
+                    if constexpr (toUTC)
+                        result_data[i] = date_time_val - static_cast<UInt32>(time_zone_offset);
+                    else
+                        result_data[i] = date_time_val + static_cast<UInt32>(time_zone_offset);
                }
                return result_column;
            }
@ -107,8 +109,12 @@ namespace
                    DateTime64 date_time_val = date_time_col.getElement(i);
                    Int64 seconds = date_time_val.value / scale_multiplier;
                    Int64 micros = date_time_val.value % scale_multiplier;
-                    LocalDateTime date_time(seconds, Name::to ? utc_time_zone : DateLUT::instance(time_zone_val));
-                    time_t time_val = date_time.to_time_t(Name::from ? utc_time_zone : DateLUT::instance(time_zone_val));
+                    auto time_zone_offset = time_zone.timezoneOffset(seconds);
+                    Int64 time_val = seconds;
+                    if constexpr (toUTC)
+                        time_val -= time_zone_offset;
+                    else
+                        time_val += time_zone_offset;
                    DateTime64 date_time_64(time_val * scale_multiplier + micros);
                    result_data[i] = date_time_64;
                }
@ -122,19 +128,15 @@ namespace
    struct NameToUTCTimestamp
    {
        static constexpr auto name = "toUTCTimestamp";
-        static constexpr auto from = false;
-        static constexpr auto to = true;
    };

    struct NameFromUTCTimestamp
    {
        static constexpr auto name = "fromUTCTimestamp";
-        static constexpr auto from = true;
-        static constexpr auto to = false;
    };

-    using ToUTCTimestampFunction = UTCTimestampTransform<NameToUTCTimestamp>;
-    using FromUTCTimestampFunction = UTCTimestampTransform<NameFromUTCTimestamp>;
+    using ToUTCTimestampFunction = UTCTimestampTransform<NameToUTCTimestamp, true>;
+    using FromUTCTimestampFunction = UTCTimestampTransform<NameFromUTCTimestamp, false>;
 }

 REGISTER_FUNCTION(UTCTimestampTransform)
--- a/Show More
+++ b/Show More