Merge remote-tracking branch 'upstream/master' into HEAD

2024-11-22 07:31:57 +00:00 · 2021-08-02 02:56:00 +03:00 · 2021-08-02 02:56:00 +03:00 · cc3ed12ed6
commit cc3ed12ed6
parent f61ea15d56 d9f55a3f31
1190 changed files with 19877 additions and 11408 deletions
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@ -2,28 +2,26 @@ I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla

 Changelog category (leave one):
 - New Feature
- Bug Fix
 - Improvement
+- Bug Fix
 - Performance Improvement
 - Backward Incompatible Change
 - Build/Testing/Packaging Improvement
 - Documentation (changelog entry is not required)
- Other
 - Not for changelog (changelog entry is not required)


 Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
-
 ...


 Detailed description / Documentation draft:
-
 ...

-By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.

-If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.
+> By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
+
+> If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.


-Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/
+> Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/
--- a/base/common/DateLUTImpl.h
+++ b/base/common/DateLUTImpl.h
@ -18,6 +18,8 @@

 #define DATE_LUT_MAX (0xFFFFFFFFU - 86400)
 #define DATE_LUT_MAX_DAY_NUM 0xFFFF
+/// Max int value of Date32, DATE LUT cache size minus daynum_offset_epoch
+#define DATE_LUT_MAX_EXTEND_DAY_NUM (DATE_LUT_SIZE - 16436)

 /// A constant to add to time_t so every supported time point becomes non-negative and still has the same remainder of division by 3600.
 /// If we treat "remainder of division" operation in the sense of modular arithmetic (not like in C++).
@ -270,6 +272,8 @@ public:
    auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; }
    auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; }

+    auto getDayNumOffsetEpoch() const { return daynum_offset_epoch; }
+
    /// All functions below are thread-safe; arguments are not checked.

    inline ExtendedDayNum toDayNum(ExtendedDayNum d) const
@ -926,15 +930,17 @@ public:
    {
        if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
            return LUTIndex(0);
-
-        return LUTIndex{years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1};
+        auto year_lut_index = (year - DATE_LUT_MIN_YEAR) * 12 + month - 1;
+        UInt32 index = years_months_lut[year_lut_index].toUnderType() + day_of_month - 1;
+        /// When date is out of range, default value is DATE_LUT_SIZE - 1 (2283-11-11)
+        return LUTIndex{std::min(index, static_cast<UInt32>(DATE_LUT_SIZE - 1))};
    }

    /// Create DayNum from year, month, day of month.
-    inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month) const
+    inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month, Int32 default_error_day_num = 0) const
    {
        if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
-            return ExtendedDayNum(0);
+            return ExtendedDayNum(default_error_day_num);

        return toDayNum(makeLUTIndex(year, month, day_of_month));
    }
@ -1091,9 +1097,9 @@ public:
        return lut[new_index].date + time;
    }

-    inline NO_SANITIZE_UNDEFINED Time addWeeks(Time t, Int64 delta) const
+    inline NO_SANITIZE_UNDEFINED Time addWeeks(Time t, Int32 delta) const
    {
-        return addDays(t, delta * 7);
+        return addDays(t, static_cast<Int64>(delta) * 7);
    }

    inline UInt8 saturateDayOfMonth(Int16 year, UInt8 month, UInt8 day_of_month) const
@ -1158,14 +1164,14 @@ public:
        return toDayNum(addMonthsIndex(d, delta));
    }

-    inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int64 delta) const
+    inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int32 delta) const
    {
-        return addMonths(t, delta * 3);
+        return addMonths(t, static_cast<Int64>(delta) * 3);
    }

-    inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int64 delta) const
+    inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int32 delta) const
    {
-        return addMonths(d, delta * 3);
+        return addMonths(d, static_cast<Int64>(delta) * 3);
    }

    template <typename DateOrTime>
--- a/base/common/LocalDate.h
+++ b/base/common/LocalDate.h
@ -70,6 +70,14 @@ public:
        m_day   = values.day_of_month;
    }

+    explicit LocalDate(ExtendedDayNum day_num)
+    {
+        const auto & values = DateLUT::instance().getValues(day_num);
+        m_year  = values.year;
+        m_month = values.month;
+        m_day   = values.day_of_month;
+    }
+
    LocalDate(unsigned short year_, unsigned char month_, unsigned char day_)
        : m_year(year_), m_month(month_), m_day(day_)
    {
@ -98,6 +106,12 @@ public:
        return DayNum(lut.makeDayNum(m_year, m_month, m_day).toUnderType());
    }

+    ExtendedDayNum  getExtenedDayNum() const
+    {
+        const auto & lut = DateLUT::instance();
+        return ExtendedDayNum (lut.makeDayNum(m_year, m_month, m_day).toUnderType());
+    }
+
    operator DayNum() const
    {
        return getDayNum();
--- a/base/common/ReplxxLineReader.cpp
+++ b/base/common/ReplxxLineReader.cpp
@ -69,7 +69,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
    }

    std::string line;
-    if (!getline(in, line).good())
+    if (getline(in, line).bad())
    {
        rx.print("Cannot read from %s (for conversion): %s\n",
            path.c_str(), errnoToString(errno).c_str());
@ -78,7 +78,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)

    /// This is the marker of the date, no need to convert.
    static char const REPLXX_TIMESTAMP_PATTERN[] = "### dddd-dd-dd dd:dd:dd.ddd";
-    if (line.starts_with("### ") && line.size() == strlen(REPLXX_TIMESTAMP_PATTERN))
+    if (line.empty() || (line.starts_with("### ") && line.size() == strlen(REPLXX_TIMESTAMP_PATTERN)))
    {
        return;
    }
--- a/base/common/removeDuplicates.h
+++ b/base/common/removeDuplicates.h
@ -0,0 +1,24 @@
+#pragma once
+#include <vector>
+
+/// Removes duplicates from a container without changing the order of its elements.
+/// Keeps the last occurrence of each element.
+/// Should NOT be used for containers with a lot of elements because it has O(N^2) complexity.
+template <typename T>
+void removeDuplicatesKeepLast(std::vector<T> & vec)
+{
+    auto begin = vec.begin();
+    auto end = vec.end();
+    auto new_begin = end;
+    for (auto current = end; current != begin;)
+    {
+        --current;
+        if (std::find(new_begin, end, *current) == end)
+        {
+            --new_begin;
+            if (new_begin != current)
+                *new_begin = *current;
+        }
+    }
+    vec.erase(begin, new_begin);
+}
--- a/base/daemon/BaseDaemon.cpp
+++ b/base/daemon/BaseDaemon.cpp
@ -259,10 +259,25 @@ private:
    Poco::Logger * log;
    BaseDaemon & daemon;

-    void onTerminate(const std::string & message, UInt32 thread_num) const
+    void onTerminate(std::string_view message, UInt32 thread_num) const
    {
+        size_t pos = message.find('\n');
+
        LOG_FATAL(log, "(version {}{}, {}) (from thread {}) {}",
-            VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, thread_num, message);
+            VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, thread_num, message.substr(0, pos));
+
+        /// Print trace from std::terminate exception line-by-line to make it easy for grep.
+        while (pos != std::string_view::npos)
+        {
+            ++pos;
+            size_t next_pos = message.find('\n', pos);
+            size_t size = next_pos;
+            if (next_pos != std::string_view::npos)
+                size = next_pos - pos;
+
+            LOG_FATAL(log, "{}", message.substr(pos, size));
+            pos = next_pos;
+        }
    }

    void onFault(
--- a/benchmark/clickhouse/benchmark-new.sh
+++ b/benchmark/clickhouse/benchmark-new.sh
@ -4,13 +4,24 @@ QUERIES_FILE="queries.sql"
 TABLE=$1
 TRIES=3

+if [ -x ./clickhouse ]
+then
+    CLICKHOUSE_CLIENT="./clickhouse client"
+elif command -v clickhouse-client >/dev/null 2>&1
+then
+    CLICKHOUSE_CLIENT="clickhouse-client"
+else
+    echo "clickhouse-client is not found"
+    exit 1
+fi
+
 cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do
    sync
    echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null

    echo -n "["
    for i in $(seq 1 $TRIES); do
-        RES=$(clickhouse-client --time --format=Null --query="$query" 2>&1)
+        RES=$(${CLICKHOUSE_CLIENT} --time --format=Null --max_memory_usage=100G --query="$query" 2>&1)
        [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null"
        [[ "$i" != $TRIES ]] && echo -n ", "
    done
--- a/benchmark/hardware.sh
+++ b/benchmark/hardware.sh
@ -11,8 +11,8 @@ DATASET="${TABLE}_v1.tar.xz"
 QUERIES_FILE="queries.sql"
 TRIES=3

-AMD64_BIN_URL="https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"
-AARCH64_BIN_URL="https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_special_build_check/clang-10-aarch64_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"
+AMD64_BIN_URL="https://builds.clickhouse.tech/master/amd64/clickhouse"
+AARCH64_BIN_URL="https://builds.clickhouse.tech/master/aarch64/clickhouse"

 # Note: on older Ubuntu versions, 'axel' does not support IPv6. If you are using IPv6-only servers on very old Ubuntu, just don't install 'axel'.

@ -89,7 +89,7 @@ cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do

    echo -n "["
    for i in $(seq 1 $TRIES); do
-        RES=$(./clickhouse client --max_memory_usage 100000000000 --time --format=Null --query="$query" 2>&1 ||:)
+        RES=$(./clickhouse client --max_memory_usage 100G --time --format=Null --query="$query" 2>&1 ||:)
        [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null"
        [[ "$i" != $TRIES ]] && echo -n ", "
    done
--- a/contrib/NuRaft
+++ b/contrib/NuRaft
@ -1 +1 @@
-Subproject commit 976874b7aa7f422bf4ea595bb7d1166c617b1c26
+Subproject commit 0ce9490093021c63564cca159571a8b27772ad48
--- a/contrib/nuraft-cmake/CMakeLists.txt
+++ b/contrib/nuraft-cmake/CMakeLists.txt
@ -22,6 +22,7 @@ set(SRCS
    "${LIBRARY_DIR}/src/launcher.cxx"
    "${LIBRARY_DIR}/src/srv_config.cxx"
    "${LIBRARY_DIR}/src/snapshot_sync_req.cxx"
+    "${LIBRARY_DIR}/src/snapshot_sync_ctx.cxx"
    "${LIBRARY_DIR}/src/handle_timeout.cxx"
    "${LIBRARY_DIR}/src/handle_append_entries.cxx"
    "${LIBRARY_DIR}/src/cluster_config.cxx"
--- a/contrib/protobuf
+++ b/contrib/protobuf
@ -1 +1 @@
-Subproject commit 73b12814204ad9068ba352914d0dc244648b48ee
+Subproject commit 75601841d172c73ae6bf4ce8121f42b875cdbabd
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@ -1 +1 @@
-Subproject commit 6ff0adefdc84dac44e78804f7ca4122fe992cf8d
+Subproject commit b6480c69bf3ab6e298e0d019a07fd4f69029b26a
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@ -70,11 +70,6 @@ else()
  endif()
 endif()

-set(BUILD_VERSION_CC rocksdb_build_version.cc)
-add_library(rocksdb_build_version OBJECT ${BUILD_VERSION_CC})
-
-target_include_directories(rocksdb_build_version PRIVATE "${ROCKSDB_SOURCE_DIR}/util")
-
 include(CheckCCompilerFlag)
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
  CHECK_C_COMPILER_FLAG("-mcpu=power9" HAS_POWER9)
@ -243,272 +238,293 @@ find_package(Threads REQUIRED)
 # Main library source code

 set(SOURCES
-        "${ROCKSDB_SOURCE_DIR}/cache/cache.cc"
-        "${ROCKSDB_SOURCE_DIR}/cache/clock_cache.cc"
-        "${ROCKSDB_SOURCE_DIR}/cache/lru_cache.cc"
-        "${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_addition.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_builder.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_cache.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_garbage.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_meta.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_format.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_sequential_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_writer.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/builder.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/c.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/column_family.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compacted_db_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compaction/compaction.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_iterator.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_job.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_fifo.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_level.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_universal.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/compaction/sst_partitioner.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/convenience.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_filesnapshot.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_write.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_compaction_flush.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_files.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_open.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_debug.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_experimental.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_readonly.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_secondary.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_info_dumper.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/db_iter.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/dbformat.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/error_handler.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/event_helpers.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/experimental.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/external_sst_file_ingestion_job.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/file_indexer.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/flush_job.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/flush_scheduler.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/forward_iterator.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/import_column_family_job.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/internal_stats.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/logs_with_prep_tracker.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/log_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/log_writer.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/malloc_stats.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/memtable.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/memtable_list.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/output_validator.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/periodic_work_scheduler.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/range_tombstone_fragmenter.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/repair.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/snapshot_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/table_cache.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/table_properties_collector.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/transaction_log_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/trim_history_scheduler.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/version_builder.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/version_edit.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/version_edit_handler.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/version_set.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/wal_edit.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/wal_manager.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/write_batch.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/write_batch_base.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/write_controller.cc"
-        "${ROCKSDB_SOURCE_DIR}/db/write_thread.cc"
-        "${ROCKSDB_SOURCE_DIR}/env/env.cc"
-        "${ROCKSDB_SOURCE_DIR}/env/env_chroot.cc"
-        "${ROCKSDB_SOURCE_DIR}/env/env_encryption.cc"
-        "${ROCKSDB_SOURCE_DIR}/env/env_hdfs.cc"
-        "${ROCKSDB_SOURCE_DIR}/env/file_system.cc"
-        "${ROCKSDB_SOURCE_DIR}/env/file_system_tracer.cc"
-        "${ROCKSDB_SOURCE_DIR}/env/mock_env.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/delete_scheduler.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/file_prefetch_buffer.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/file_util.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/filename.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/random_access_file_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/read_write_util.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/readahead_raf.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/sequence_file_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/sst_file_manager_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/file/writable_file_writer.cc"
-        "${ROCKSDB_SOURCE_DIR}/logging/auto_roll_logger.cc"
-        "${ROCKSDB_SOURCE_DIR}/logging/event_logger.cc"
-        "${ROCKSDB_SOURCE_DIR}/logging/log_buffer.cc"
-        "${ROCKSDB_SOURCE_DIR}/memory/arena.cc"
-        "${ROCKSDB_SOURCE_DIR}/memory/concurrent_arena.cc"
-        "${ROCKSDB_SOURCE_DIR}/memory/jemalloc_nodump_allocator.cc"
-        "${ROCKSDB_SOURCE_DIR}/memory/memkind_kmem_allocator.cc"
-        "${ROCKSDB_SOURCE_DIR}/memtable/alloc_tracker.cc"
-        "${ROCKSDB_SOURCE_DIR}/memtable/hash_linklist_rep.cc"
-        "${ROCKSDB_SOURCE_DIR}/memtable/hash_skiplist_rep.cc"
-        "${ROCKSDB_SOURCE_DIR}/memtable/skiplistrep.cc"
-        "${ROCKSDB_SOURCE_DIR}/memtable/vectorrep.cc"
-        "${ROCKSDB_SOURCE_DIR}/memtable/write_buffer_manager.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/histogram.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/histogram_windowing.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/in_memory_stats_history.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/instrumented_mutex.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/iostats_context.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/perf_context.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/perf_level.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/persistent_stats_history.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/statistics.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_updater.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_util.cc"
-        "${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_util_debug.cc"
-        "${ROCKSDB_SOURCE_DIR}/options/cf_options.cc"
-        "${ROCKSDB_SOURCE_DIR}/options/configurable.cc"
-        "${ROCKSDB_SOURCE_DIR}/options/customizable.cc"
-        "${ROCKSDB_SOURCE_DIR}/options/db_options.cc"
-        "${ROCKSDB_SOURCE_DIR}/options/options.cc"
-        "${ROCKSDB_SOURCE_DIR}/options/options_helper.cc"
-        "${ROCKSDB_SOURCE_DIR}/options/options_parser.cc"
-        "${ROCKSDB_SOURCE_DIR}/port/stack_trace.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/adaptive/adaptive_table_factory.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/binary_search_index_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_filter_block.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_builder.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_factory.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_iterator.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block_builder.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefetcher.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefix_index.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/data_block_hash_index.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/data_block_footer.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/filter_block_reader_common.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/filter_policy.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/flush_block_policy.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/full_filter_block.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/hash_index_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/index_builder.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/index_reader_common.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/parsed_full_filter_block.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_filter_block.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_index_iterator.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_index_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/reader_common.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_based/uncompression_dict_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/block_fetcher.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_builder.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_factory.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/format.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/get_context.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/iterator.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/merging_iterator.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/meta_blocks.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/persistent_cache_helper.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_bloom.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_builder.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_factory.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_index.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_key_coding.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/sst_file_dumper.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/sst_file_reader.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/sst_file_writer.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/table_factory.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/table_properties.cc"
-        "${ROCKSDB_SOURCE_DIR}/table/two_level_iterator.cc"
-        "${ROCKSDB_SOURCE_DIR}/test_util/sync_point.cc"
-        "${ROCKSDB_SOURCE_DIR}/test_util/sync_point_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/test_util/testutil.cc"
-        "${ROCKSDB_SOURCE_DIR}/test_util/transaction_test_util.cc"
-        "${ROCKSDB_SOURCE_DIR}/tools/block_cache_analyzer/block_cache_trace_analyzer.cc"
-        "${ROCKSDB_SOURCE_DIR}/tools/dump/db_dump_tool.cc"
-        "${ROCKSDB_SOURCE_DIR}/tools/io_tracer_parser_tool.cc"
-        "${ROCKSDB_SOURCE_DIR}/tools/ldb_cmd.cc"
-        "${ROCKSDB_SOURCE_DIR}/tools/ldb_tool.cc"
-        "${ROCKSDB_SOURCE_DIR}/tools/sst_dump_tool.cc"
-        "${ROCKSDB_SOURCE_DIR}/tools/trace_analyzer_tool.cc"
-        "${ROCKSDB_SOURCE_DIR}/trace_replay/trace_replay.cc"
-        "${ROCKSDB_SOURCE_DIR}/trace_replay/block_cache_tracer.cc"
-        "${ROCKSDB_SOURCE_DIR}/trace_replay/io_tracer.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/coding.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/compaction_job_stats_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/comparator.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/compression_context_cache.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/concurrent_task_limiter_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/crc32c.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/dynamic_bloom.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/hash.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/murmurhash.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/random.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/rate_limiter.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/slice.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/file_checksum_helper.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/status.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/string_util.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/thread_local.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/threadpool_imp.cc"
-        "${ROCKSDB_SOURCE_DIR}/util/xxhash.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/backupable/backupable_db.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_compaction_filter.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db_impl_filesnapshot.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_dump_tool.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_file.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/cassandra/cassandra_compaction_filter.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/cassandra/format.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/cassandra/merge_operator.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/checkpoint/checkpoint_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/debug.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/env_mirror.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/env_timed.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/fault_injection_env.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/fault_injection_fs.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/leveldb_options/leveldb_options.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/memory/memory_util.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/bytesxor.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/max.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/put.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/sortlist.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/string_append/stringappend.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/string_append/stringappend2.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/uint64add.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/object_registry.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/option_change_migration/option_change_migration.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/options/options_util.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier_file.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier_metadata.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/persistent_cache_tier.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/volatile_tier_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/cache_simulator.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/sim_cache.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/table_properties_collectors/compact_on_deletion_collector.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/trace/file_trace_reader_writer.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/lock_manager.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/point/point_lock_tracker.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/point/point_lock_manager.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/optimistic_transaction_db_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/optimistic_transaction.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/pessimistic_transaction.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/pessimistic_transaction_db.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/snapshot_checker.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_base.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_db_mutex_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_util.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_prepared_txn.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_prepared_txn_db.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn_db.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/ttl/db_ttl_impl.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/write_batch_with_index/write_batch_with_index.cc"
-        "${ROCKSDB_SOURCE_DIR}/utilities/write_batch_with_index/write_batch_with_index_internal.cc"
-        $<TARGET_OBJECTS:rocksdb_build_version>)
+    ${ROCKSDB_SOURCE_DIR}/cache/cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/cache_entry_roles.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/clock_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/lru_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_fetcher.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_addition.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_garbage.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_meta.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_garbage_meter.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_format.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_sequential_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_writer.cc
+    ${ROCKSDB_SOURCE_DIR}/db/builder.cc
+    ${ROCKSDB_SOURCE_DIR}/db/c.cc
+    ${ROCKSDB_SOURCE_DIR}/db/column_family.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_iterator.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_job.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_fifo.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_level.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_universal.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/sst_partitioner.cc
+    ${ROCKSDB_SOURCE_DIR}/db/convenience.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_filesnapshot.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/compacted_db_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_write.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_compaction_flush.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_files.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_open.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_debug.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_experimental.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_readonly.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_secondary.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_info_dumper.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_iter.cc
+    ${ROCKSDB_SOURCE_DIR}/db/dbformat.cc
+    ${ROCKSDB_SOURCE_DIR}/db/error_handler.cc
+    ${ROCKSDB_SOURCE_DIR}/db/event_helpers.cc
+    ${ROCKSDB_SOURCE_DIR}/db/experimental.cc
+    ${ROCKSDB_SOURCE_DIR}/db/external_sst_file_ingestion_job.cc
+    ${ROCKSDB_SOURCE_DIR}/db/file_indexer.cc
+    ${ROCKSDB_SOURCE_DIR}/db/flush_job.cc
+    ${ROCKSDB_SOURCE_DIR}/db/flush_scheduler.cc
+    ${ROCKSDB_SOURCE_DIR}/db/forward_iterator.cc
+    ${ROCKSDB_SOURCE_DIR}/db/import_column_family_job.cc
+    ${ROCKSDB_SOURCE_DIR}/db/internal_stats.cc
+    ${ROCKSDB_SOURCE_DIR}/db/logs_with_prep_tracker.cc
+    ${ROCKSDB_SOURCE_DIR}/db/log_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/db/log_writer.cc
+    ${ROCKSDB_SOURCE_DIR}/db/malloc_stats.cc
+    ${ROCKSDB_SOURCE_DIR}/db/memtable.cc
+    ${ROCKSDB_SOURCE_DIR}/db/memtable_list.cc
+    ${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc
+    ${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc
+    ${ROCKSDB_SOURCE_DIR}/db/output_validator.cc
+    ${ROCKSDB_SOURCE_DIR}/db/periodic_work_scheduler.cc
+    ${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc
+    ${ROCKSDB_SOURCE_DIR}/db/range_tombstone_fragmenter.cc
+    ${ROCKSDB_SOURCE_DIR}/db/repair.cc
+    ${ROCKSDB_SOURCE_DIR}/db/snapshot_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/db/table_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/db/table_properties_collector.cc
+    ${ROCKSDB_SOURCE_DIR}/db/transaction_log_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/db/trim_history_scheduler.cc
+    ${ROCKSDB_SOURCE_DIR}/db/version_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/db/version_edit.cc
+    ${ROCKSDB_SOURCE_DIR}/db/version_edit_handler.cc
+    ${ROCKSDB_SOURCE_DIR}/db/version_set.cc
+    ${ROCKSDB_SOURCE_DIR}/db/wal_edit.cc
+    ${ROCKSDB_SOURCE_DIR}/db/wal_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/db/write_batch.cc
+    ${ROCKSDB_SOURCE_DIR}/db/write_batch_base.cc
+    ${ROCKSDB_SOURCE_DIR}/db/write_controller.cc
+    ${ROCKSDB_SOURCE_DIR}/db/write_thread.cc
+    ${ROCKSDB_SOURCE_DIR}/env/composite_env.cc
+    ${ROCKSDB_SOURCE_DIR}/env/env.cc
+    ${ROCKSDB_SOURCE_DIR}/env/env_chroot.cc
+    ${ROCKSDB_SOURCE_DIR}/env/env_encryption.cc
+    ${ROCKSDB_SOURCE_DIR}/env/env_hdfs.cc
+    ${ROCKSDB_SOURCE_DIR}/env/file_system.cc
+    ${ROCKSDB_SOURCE_DIR}/env/file_system_tracer.cc
+    ${ROCKSDB_SOURCE_DIR}/env/fs_remap.cc
+    ${ROCKSDB_SOURCE_DIR}/env/mock_env.cc
+    ${ROCKSDB_SOURCE_DIR}/file/delete_scheduler.cc
+    ${ROCKSDB_SOURCE_DIR}/file/file_prefetch_buffer.cc
+    ${ROCKSDB_SOURCE_DIR}/file/file_util.cc
+    ${ROCKSDB_SOURCE_DIR}/file/filename.cc
+    ${ROCKSDB_SOURCE_DIR}/file/line_file_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/file/random_access_file_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/file/read_write_util.cc
+    ${ROCKSDB_SOURCE_DIR}/file/readahead_raf.cc
+    ${ROCKSDB_SOURCE_DIR}/file/sequence_file_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/file/sst_file_manager_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/file/writable_file_writer.cc
+    ${ROCKSDB_SOURCE_DIR}/logging/auto_roll_logger.cc
+    ${ROCKSDB_SOURCE_DIR}/logging/event_logger.cc
+    ${ROCKSDB_SOURCE_DIR}/logging/log_buffer.cc
+    ${ROCKSDB_SOURCE_DIR}/memory/arena.cc
+    ${ROCKSDB_SOURCE_DIR}/memory/concurrent_arena.cc
+    ${ROCKSDB_SOURCE_DIR}/memory/jemalloc_nodump_allocator.cc
+    ${ROCKSDB_SOURCE_DIR}/memory/memkind_kmem_allocator.cc
+    ${ROCKSDB_SOURCE_DIR}/memtable/alloc_tracker.cc
+    ${ROCKSDB_SOURCE_DIR}/memtable/hash_linklist_rep.cc
+    ${ROCKSDB_SOURCE_DIR}/memtable/hash_skiplist_rep.cc
+    ${ROCKSDB_SOURCE_DIR}/memtable/skiplistrep.cc
+    ${ROCKSDB_SOURCE_DIR}/memtable/vectorrep.cc
+    ${ROCKSDB_SOURCE_DIR}/memtable/write_buffer_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/histogram.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/histogram_windowing.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/in_memory_stats_history.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/instrumented_mutex.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/iostats_context.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/perf_context.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/perf_level.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/persistent_stats_history.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/statistics.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_updater.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_util.cc
+    ${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_util_debug.cc
+    ${ROCKSDB_SOURCE_DIR}/options/cf_options.cc
+    ${ROCKSDB_SOURCE_DIR}/options/configurable.cc
+    ${ROCKSDB_SOURCE_DIR}/options/customizable.cc
+    ${ROCKSDB_SOURCE_DIR}/options/db_options.cc
+    ${ROCKSDB_SOURCE_DIR}/options/options.cc
+    ${ROCKSDB_SOURCE_DIR}/options/options_helper.cc
+    ${ROCKSDB_SOURCE_DIR}/options/options_parser.cc
+    ${ROCKSDB_SOURCE_DIR}/port/stack_trace.cc
+    ${ROCKSDB_SOURCE_DIR}/table/adaptive/adaptive_table_factory.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/binary_search_index_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_filter_block.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_factory.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_iterator.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefetcher.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefix_index.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/data_block_hash_index.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/data_block_footer.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/filter_block_reader_common.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/filter_policy.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/flush_block_policy.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/full_filter_block.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/hash_index_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/index_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/index_reader_common.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/parsed_full_filter_block.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_filter_block.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_index_iterator.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_index_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/reader_common.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/uncompression_dict_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_fetcher.cc
+    ${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_factory.cc
+    ${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/table/format.cc
+    ${ROCKSDB_SOURCE_DIR}/table/get_context.cc
+    ${ROCKSDB_SOURCE_DIR}/table/iterator.cc
+    ${ROCKSDB_SOURCE_DIR}/table/merging_iterator.cc
+    ${ROCKSDB_SOURCE_DIR}/table/meta_blocks.cc
+    ${ROCKSDB_SOURCE_DIR}/table/persistent_cache_helper.cc
+    ${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_bloom.cc
+    ${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_factory.cc
+    ${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_index.cc
+    ${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_key_coding.cc
+    ${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/table/sst_file_dumper.cc
+    ${ROCKSDB_SOURCE_DIR}/table/sst_file_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/table/sst_file_writer.cc
+    ${ROCKSDB_SOURCE_DIR}/table/table_factory.cc
+    ${ROCKSDB_SOURCE_DIR}/table/table_properties.cc
+    ${ROCKSDB_SOURCE_DIR}/table/two_level_iterator.cc
+    ${ROCKSDB_SOURCE_DIR}/test_util/sync_point.cc
+    ${ROCKSDB_SOURCE_DIR}/test_util/sync_point_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/test_util/testutil.cc
+    ${ROCKSDB_SOURCE_DIR}/test_util/transaction_test_util.cc
+    ${ROCKSDB_SOURCE_DIR}/tools/block_cache_analyzer/block_cache_trace_analyzer.cc
+    ${ROCKSDB_SOURCE_DIR}/tools/dump/db_dump_tool.cc
+    ${ROCKSDB_SOURCE_DIR}/tools/io_tracer_parser_tool.cc
+    ${ROCKSDB_SOURCE_DIR}/tools/ldb_cmd.cc
+    ${ROCKSDB_SOURCE_DIR}/tools/ldb_tool.cc
+    ${ROCKSDB_SOURCE_DIR}/tools/sst_dump_tool.cc
+    ${ROCKSDB_SOURCE_DIR}/tools/trace_analyzer_tool.cc
+    ${ROCKSDB_SOURCE_DIR}/trace_replay/trace_replay.cc
+    ${ROCKSDB_SOURCE_DIR}/trace_replay/block_cache_tracer.cc
+    ${ROCKSDB_SOURCE_DIR}/trace_replay/io_tracer.cc
+    ${ROCKSDB_SOURCE_DIR}/util/coding.cc
+    ${ROCKSDB_SOURCE_DIR}/util/compaction_job_stats_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/util/comparator.cc
+    ${ROCKSDB_SOURCE_DIR}/util/compression_context_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/util/concurrent_task_limiter_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/util/crc32c.cc
+    ${ROCKSDB_SOURCE_DIR}/util/dynamic_bloom.cc
+    ${ROCKSDB_SOURCE_DIR}/util/hash.cc
+    ${ROCKSDB_SOURCE_DIR}/util/murmurhash.cc
+    ${ROCKSDB_SOURCE_DIR}/util/random.cc
+    ${ROCKSDB_SOURCE_DIR}/util/rate_limiter.cc
+    ${ROCKSDB_SOURCE_DIR}/util/ribbon_config.cc
+    ${ROCKSDB_SOURCE_DIR}/util/slice.cc
+    ${ROCKSDB_SOURCE_DIR}/util/file_checksum_helper.cc
+    ${ROCKSDB_SOURCE_DIR}/util/status.cc
+    ${ROCKSDB_SOURCE_DIR}/util/string_util.cc
+    ${ROCKSDB_SOURCE_DIR}/util/thread_local.cc
+    ${ROCKSDB_SOURCE_DIR}/util/threadpool_imp.cc
+    ${ROCKSDB_SOURCE_DIR}/util/xxhash.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/backupable/backupable_db.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_compaction_filter.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db_impl_filesnapshot.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_dump_tool.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_file.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/cassandra/cassandra_compaction_filter.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/cassandra/format.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/cassandra/merge_operator.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/checkpoint/checkpoint_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/debug.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/env_mirror.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/env_timed.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/fault_injection_env.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/fault_injection_fs.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/leveldb_options/leveldb_options.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/memory/memory_util.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/bytesxor.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/max.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/put.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/sortlist.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/string_append/stringappend.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/string_append/stringappend2.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/uint64add.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/object_registry.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/option_change_migration/option_change_migration.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/options/options_util.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier_file.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier_metadata.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/persistent_cache_tier.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/volatile_tier_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/cache_simulator.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/sim_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/table_properties_collectors/compact_on_deletion_collector.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/trace/file_trace_reader_writer.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/lock_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/point/point_lock_tracker.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/point/point_lock_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/optimistic_transaction_db_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/optimistic_transaction.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/pessimistic_transaction.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/pessimistic_transaction_db.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/snapshot_checker.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_base.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_db_mutex_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_util.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_prepared_txn.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_prepared_txn_db.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn_db.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/ttl/db_ttl_impl.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/write_batch_with_index/write_batch_with_index.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/write_batch_with_index/write_batch_with_index_internal.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/standalone_port.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/util/dbt.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/util/memarena.cc
+    rocksdb_build_version.cc)

 if(HAVE_SSE42 AND NOT MSVC)
  set_source_files_properties(
--- a/contrib/rocksdb-cmake/rocksdb_build_version.cc
+++ b/contrib/rocksdb-cmake/rocksdb_build_version.cc
@ -1,3 +1,62 @@
-const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:0";
-const char* rocksdb_build_git_date = "rocksdb_build_git_date:2000-01-01";
-const char* rocksdb_build_compile_date = "2000-01-01";
+// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+/// This file was edited for ClickHouse.
+
+#include <memory>
+
+#include "rocksdb/version.h"
+#include "util/string_util.h"
+
+// The build script may replace these values with real values based
+// on whether or not GIT is available and the platform settings
+static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:0";
+static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:master";
+static const std::string rocksdb_build_date = "rocksdb_build_date:2000-01-01";
+
+namespace ROCKSDB_NAMESPACE {
+static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
+  size_t colon = name.find(":");
+  if (colon != std::string::npos && colon > 0 && colon < name.length() - 1) {
+    // If we found a "@:", then this property was a build-time substitution that failed.  Skip it
+    size_t at = name.find("@", colon);
+    if (at != colon + 1) {
+      // Everything before the colon is the name, after is the value
+      (*props)[name.substr(0, colon)] = name.substr(colon + 1);
+    }
+  }
+}
+
+static std::unordered_map<std::string, std::string>* LoadPropertiesSet() {
+  auto * properties = new std::unordered_map<std::string, std::string>();
+  AddProperty(properties, rocksdb_build_git_sha);
+  AddProperty(properties, rocksdb_build_git_tag);
+  AddProperty(properties, rocksdb_build_date);
+  return properties;
+}
+
+const std::unordered_map<std::string, std::string>& GetRocksBuildProperties() {
+  static std::unique_ptr<std::unordered_map<std::string, std::string>> props(LoadPropertiesSet());
+  return *props;
+}
+
+std::string GetRocksVersionAsString(bool with_patch) {
+  std::string version = ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR);
+  if (with_patch) {
+    return version + "." + ToString(ROCKSDB_PATCH);
+  } else {
+    return version;
+  }
+}
+
+std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {
+  std::string info = program + " (RocksDB) " + GetRocksVersionAsString(true);
+  if (verbose) {
+    for (const auto& it : GetRocksBuildProperties()) {
+      info.append("\n    ");
+      info.append(it.first);
+      info.append(": ");
+      info.append(it.second);
+    }
+  }
+  return info;
+}
+} // namespace ROCKSDB_NAMESPACE
--- a/contrib/s2geometry-cmake/CMakeLists.txt
+++ b/contrib/s2geometry-cmake/CMakeLists.txt
@ -115,6 +115,8 @@ set(S2_SRCS

 add_library(s2 ${S2_SRCS})

+set_property(TARGET s2 PROPERTY CXX_STANDARD 11)
+
 if (OPENSSL_FOUND)
    target_link_libraries(s2 PRIVATE ${OPENSSL_LIBRARIES})
 endif()
--- a/docker/packager/deb/Dockerfile
+++ b/docker/packager/deb/Dockerfile
@ -27,7 +27,7 @@ RUN apt-get update \
 # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
 # to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
 # Significantly increase deb packaging speed and compatible with old systems
-RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \
+RUN curl -O https://clickhouse-datasets.s3.yandex.net/utils/1/dpkg-deb \
    && chmod +x dpkg-deb \
    && cp dpkg-deb /usr/bin

--- a/docker/packager/unbundled/Dockerfile
+++ b/docker/packager/unbundled/Dockerfile
@ -2,7 +2,7 @@
 FROM yandex/clickhouse-deb-builder

 RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && wget -nv -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \
+    && wget -nv -O /tmp/arrow-keyring.deb "https://apache.jfrog.io/artifactory/arrow/ubuntu/apache-arrow-apt-source-latest-${CODENAME}.deb" \
    && dpkg -i /tmp/arrow-keyring.deb

 # Libraries from OS are only needed to test the "unbundled" build (that is not used in production).
--- a/docker/server/entrypoint.sh
+++ b/docker/server/entrypoint.sh
@ -72,7 +72,10 @@ do

    if [ "$DO_CHOWN" = "1" ]; then
        # ensure proper directories permissions
-        chown -R "$USER:$GROUP" "$dir"
+        # but skip it for if directory already has proper premissions, cause recursive chown may be slow
+        if [ "$(stat -c %u "$dir")" != "$USER" ] || [ "$(stat -c %g "$dir")" != "$GROUP" ]; then
+            chown -R "$USER:$GROUP" "$dir"
+        fi
    elif ! $gosu test -d "$dir" -a -w "$dir" -a -r "$dir"; then
        echo "Necessary directory '$dir' isn't accessible by user with id '$USER'"
        exit 1
@ -161,6 +164,10 @@ fi

 # if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments
 if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
+    # Watchdog is launched by default, but does not send SIGINT to the main process,
+    # so the container can't be finished by ctrl+c
+    CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0}
+    export CLICKHOUSE_WATCHDOG_ENABLE
    exec $gosu /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@"
 fi

--- a/docker/test/base/Dockerfile
+++ b/docker/test/base/Dockerfile
@ -27,7 +27,7 @@ RUN apt-get update \
 # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
 # to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
 # Significantly increase deb packaging speed and compatible with old systems
-RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \
+RUN curl -O https://clickhouse-datasets.s3.yandex.net/utils/1/dpkg-deb \
    && chmod +x dpkg-deb \
    && cp dpkg-deb /usr/bin

@ -61,4 +61,7 @@ ENV TSAN_OPTIONS='halt_on_error=1 history_size=7'
 ENV UBSAN_OPTIONS='print_stacktrace=1'
 ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'

+ENV TZ=Europe/Moscow
+RUN ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone
+
 CMD sleep 1
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@ -27,7 +27,7 @@ RUN apt-get update \
 # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
 # to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
 # Significantly increase deb packaging speed and compatible with old systems
-RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \
+RUN curl -O https://clickhouse-datasets.s3.yandex.net/utils/1/dpkg-deb \
    && chmod +x dpkg-deb \
    && cp dpkg-deb /usr/bin

@ -65,7 +65,7 @@ RUN apt-get update \
        unixodbc \
       --yes --no-install-recommends

-RUN pip3 install numpy scipy pandas
+RUN pip3 install numpy scipy pandas Jinja2

 # This symlink required by gcc to find lld compiler
 RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@ -299,6 +299,7 @@ function run_tests
        01318_decrypt                           # Depends on OpenSSL
        01663_aes_msan                          # Depends on OpenSSL
        01667_aes_args_check                    # Depends on OpenSSL
+        01683_codec_encrypted                   # Depends on OpenSSL
        01776_decrypt_aead_size_check           # Depends on OpenSSL
        01811_filter_by_null                    # Depends on OpenSSL
        01281_unsucceeded_insert_select_queries_counter
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@ -194,6 +194,10 @@ continue
    jobs
    pstree -aspgT

+    server_exit_code=0
+    wait $server_pid || server_exit_code=$?
+    echo "Server exit code is $server_exit_code"
+
    # Make files with status and description we'll show for this check on Github.
    task_exit_code=$fuzzer_exit_code
    if [ "$server_died" == 1 ]
--- a/docker/test/integration/runner/Dockerfile
+++ b/docker/test/integration/runner/Dockerfile
@ -76,6 +76,7 @@ RUN python3 -m pip install \
    pytest \
    pytest-timeout \
    pytest-xdist \
+    pytest-repeat \
    redis \
    tzlocal \
    urllib3 \
--- a/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml
+++ b/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml
@ -14,10 +14,14 @@ services:
      }
      EOF
      ./docker-entrypoint.sh'
-    ports:
-      - 9020:9019
+    expose:
+      - 9019
    healthcheck:
      test: ["CMD", "curl", "-s", "localhost:9019/ping"]
      interval: 5s
      timeout: 3s
      retries: 30
+    volumes:
+      - type: ${JDBC_BRIDGE_FS:-tmpfs}
+        source: ${JDBC_BRIDGE_LOGS:-}
+        target: /app/logs
--- a/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml
+++ b/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml
--- a/docker/test/integration/runner/compose/docker_compose_postgres.yml
+++ b/docker/test/integration/runner/compose/docker_compose_postgres.yml
@ -2,7 +2,7 @@ version: '2.3'
 services:
    postgres1:
        image: postgres
-        command: ["postgres", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all"]
+        command: ["postgres", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all", "-c", "max_connections=200"]
        restart: always
        expose:
            - ${POSTGRES_PORT}
--- a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml
+++ b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml
@ -2,7 +2,7 @@ version: '2.3'

 services:
    rabbitmq1:
-        image: rabbitmq:3-management-alpine
+        image: rabbitmq:3.8-management-alpine
        hostname: rabbitmq1
        expose:
            - ${RABBITMQ_PORT}
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@ -1196,7 +1196,7 @@ create table changes engine File(TSV, 'metrics/changes.tsv') as
            if(left > right, left / right, right / left) times_diff
        from metrics
        group by metric
-        having abs(diff) > 0.05 and isFinite(diff)
+        having abs(diff) > 0.05 and isFinite(diff) and isFinite(times_diff)
    )
    order by diff desc
    ;
--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@ -2,6 +2,11 @@

 set -e -x

+# Choose random timezone for this test run
+TZ="$(grep -v '#' /usr/share/zoneinfo/zone.tab  | awk '{print $3}' | shuf | head -n1)"
+echo "Choosen random timezone $TZ"
+ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone
+
 dpkg -i package_folder/clickhouse-common-static_*.deb;
 dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
 dpkg -i package_folder/clickhouse-server_*.deb
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@ -32,7 +32,7 @@ RUN apt-get update -y \
            postgresql-client \
            sqlite3

-RUN pip3 install numpy scipy pandas
+RUN pip3 install numpy scipy pandas Jinja2

 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
   && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
--- a/docker/test/stateless/process_functional_tests_result.py
+++ b/docker/test/stateless/process_functional_tests_result.py
@ -12,7 +12,7 @@ UNKNOWN_SIGN = "[ UNKNOWN "
 SKIPPED_SIGN = "[ SKIPPED "
 HUNG_SIGN = "Found hung queries in processlist"

-NO_TASK_TIMEOUT_SIGN = "All tests have finished"
+NO_TASK_TIMEOUT_SIGNS = ["All tests have finished", "No tests were run"]

 RETRIES_SIGN = "Some tests were restarted"

@ -29,7 +29,7 @@ def process_test_log(log_path):
    with open(log_path, 'r') as test_file:
        for line in test_file:
            line = line.strip()
-            if NO_TASK_TIMEOUT_SIGN in line:
+            if any(s in line for s in NO_TASK_TIMEOUT_SIGNS):
                task_timeout = False
            if HUNG_SIGN in line:
                hung = True
@ -80,6 +80,7 @@ def process_result(result_path):
    if result_path and os.path.exists(result_path):
        total, skipped, unknown, failed, success, hung, task_timeout, retries, test_results = process_test_log(result_path)
        is_flacky_check = 1 < int(os.environ.get('NUM_TRIES', 1))
+        logging.info("Is flacky check: %s", is_flacky_check)
        # If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately)
        # But it's Ok for "flaky checks" - they can contain just one test for check which is marked as skipped.
        if failed != 0 or unknown != 0 or (success == 0 and (not is_flacky_check)):
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@ -3,6 +3,11 @@
 # fail on errors, verbose and export all env variables
 set -e -x -a

+# Choose random timezone for this test run.
+TZ="$(grep -v '#' /usr/share/zoneinfo/zone.tab  | awk '{print $3}' | shuf | head -n1)"
+echo "Choosen random timezone $TZ"
+ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone
+
 dpkg -i package_folder/clickhouse-common-static_*.deb
 dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
 dpkg -i package_folder/clickhouse-server_*.deb
@ -138,15 +143,18 @@ if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
 fi
 tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||:
 tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||:
+tar -chf /test_output/zookeeper_log_dump.tar /var/lib/clickhouse/data/system/zookeeper_log ||:
 tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:

 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
-  grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||:
-  grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
+    grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||:
+    grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
    pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
    pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
    mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
    mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
+    tar -chf /test_output/zookeeper_log_dump1.tar /var/lib/clickhouse1/data/system/zookeeper_log ||:
+    tar -chf /test_output/zookeeper_log_dump2.tar /var/lib/clickhouse2/data/system/zookeeper_log ||:
    tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||:
    tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||:
 fi
--- a/docker/test/stateless_unbundled/Dockerfile
+++ b/docker/test/stateless_unbundled/Dockerfile
@ -77,9 +77,6 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \
   && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
   && rm -rf /tmp/clickhouse-odbc-tmp

-ENV TZ=Europe/Moscow
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
 COPY run.sh /
 CMD ["/bin/bash", "/run.sh"]

--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@ -58,11 +58,11 @@ function start()
            echo "Cannot start clickhouse-server"
            cat /var/log/clickhouse-server/stdout.log
            tail -n1000 /var/log/clickhouse-server/stderr.log
-            tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
+            tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | grep -F -v '<Warning> RaftInstance:' -e '<Information> RaftInstance' | tail -n1000
            break
        fi
        # use root to match with current uid
-        clickhouse start --user root >/var/log/clickhouse-server/stdout.log 2>/var/log/clickhouse-server/stderr.log
+        clickhouse start --user root >/var/log/clickhouse-server/stdout.log 2>>/var/log/clickhouse-server/stderr.log
        sleep 0.5
        counter=$((counter + 1))
    done
@ -125,7 +125,7 @@ zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.log*
 # Sanitizer asserts
 zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
 zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
-zgrep -Fav "ASan doesn't fully support makecontext/swapcontext functions" > /dev/null \
+zgrep -Fav "ASan doesn't fully support makecontext/swapcontext functions" /test_output/tmp > /dev/null \
    && echo -e 'Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
    || echo -e 'No sanitizer asserts\tOK' >> /test_output/test_results.tsv
 rm -f /test_output/tmp
--- a/docker/test/stress/stress
+++ b/docker/test/stress/stress
@ -20,6 +20,7 @@ def get_skip_list_cmd(path):

 def get_options(i):
    options = []
+    client_options = []
    if 0 < i:
        options.append("--order=random")

@ -27,25 +28,29 @@ def get_options(i):
        options.append("--db-engine=Ordinary")

    if i % 3 == 2:
-        options.append('''--client-option='allow_experimental_database_replicated=1' --db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i))
+        options.append('''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i))
+        client_options.append('allow_experimental_database_replicated=1')

    # If database name is not specified, new database is created for each functional test.
    # Run some threads with one database for all tests.
    if i % 2 == 1:
        options.append(" --database=test_{}".format(i))

-    if i % 7 == 0:
-        options.append(" --client-option='join_use_nulls=1'")
+    if i % 5 == 1:
+        client_options.append("join_use_nulls=1")

-    if i % 14 == 0:
-        options.append(' --client-option="join_algorithm=\'partial_merge\'"')
+    if i % 15 == 6:
+        client_options.append("join_algorithm='partial_merge'")

-    if i % 21 == 0:
-        options.append(' --client-option="join_algorithm=\'auto\'"')
-        options.append(' --client-option="max_rows_in_join=1000"')
+    if i % 15 == 11:
+        client_options.append("join_algorithm='auto'")
+        client_options.append('max_rows_in_join=1000')

    if i == 13:
-        options.append(" --client-option='memory_tracker_fault_probability=0.00001'")
+        client_options.append('memory_tracker_fault_probability=0.001')
+
+    if client_options:
+        options.append(" --client-option " + ' '.join(client_options))

    return ' '.join(options)

--- a/docker/test/testflows/runner/Dockerfile
+++ b/docker/test/testflows/runner/Dockerfile
@ -35,7 +35,7 @@ RUN apt-get update \
 ENV TZ=Europe/Moscow
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone

-RUN pip3 install urllib3 testflows==1.6.90 docker-compose==1.29.1 docker==5.0.0 dicttoxml kazoo tzlocal python-dateutil numpy
+RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0 dicttoxml kazoo tzlocal python-dateutil numpy

 ENV DOCKER_CHANNEL stable
 ENV DOCKER_VERSION 20.10.6
--- a/docker/test/unit/Dockerfile
+++ b/docker/test/unit/Dockerfile
@ -1,8 +1,6 @@
 # docker build -t yandex/clickhouse-unit-test .
 FROM yandex/clickhouse-stateless-test

-ENV TZ=Europe/Moscow
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 RUN apt-get install gdb

 COPY run.sh /
--- a/docs/README.md
+++ b/docs/README.md
@ -9,7 +9,7 @@ Many developers can say that the code is the best docs by itself, and they are r
 If you want to help ClickHouse with documentation you can face, for example, the following questions:

 - "I don't know how to write."
-    
+
    We have prepared some [recommendations](#what-to-write) for you.

 - "I know what I want to write, but I don't know how to contribute to docs."
@ -71,17 +71,17 @@ Contribute all new information in English language. Other languages are translat
    ```

 - Bold text: `**asterisks**` or `__underlines__`.
- Links: `[link text](uri)`. Examples: 
+- Links: `[link text](uri)`. Examples:

    - External link: `[ClickHouse repo](https://github.com/ClickHouse/ClickHouse)`
    - Cross link: `[How to build docs](tools/README.md)`

 - Images: `![Exclamation sign](uri)`. You can refer to local images as well as remote in internet.
 - Lists: Lists can be of two types:
-    
+
    - `- unordered`: Each item starts from the `-`.
    - `1. ordered`: Each item starts from the number.
-    
+
    A list must be separated from the text by an empty line. Nested lists must be indented with 4 spaces.

 - Inline code: `` `in backticks` ``.
@ -107,7 +107,7 @@ Contribute all new information in English language. Other languages are translat
 - Text hidden behind a cut (single sting that opens on click):

    ```text
-    <details markdown="1"> <summary>Visible text</summary> 
+    <details markdown="1"> <summary>Visible text</summary>
        Hidden content.
    </details>`.
    ```
--- a/docs/_description_templates/template-data-type.md
+++ b/docs/_description_templates/template-data-type.md
@ -1,6 +1,6 @@
 ---
-toc_priority: 
-toc_title: 
+toc_priority:
+toc_title:
 ---

 # data_type_name {#data_type-name}
--- a/docs/_description_templates/template-engine.md
+++ b/docs/_description_templates/template-engine.md
@ -58,6 +58,6 @@ Result:

 Follow up with any text to clarify the example.

-**See Also** 
+**See Also**

 -   [link](#)
--- a/docs/_description_templates/template-function.md
+++ b/docs/_description_templates/template-function.md
@ -14,8 +14,8 @@ More text (Optional).

 **Arguments** (Optional)

-   `x` — Description. Optional (only for optional arguments). Possible values: <values list>. Default value: <value>. [Type name](relative/path/to/type/dscr.md#type). 
-   `y` — Description. Optional (only for optional arguments). Possible values: <values list>.Default value: <value>. [Type name](relative/path/to/type/dscr.md#type). 
+-   `x` — Description. Optional (only for optional arguments). Possible values: <values list>. Default value: <value>. [Type name](relative/path/to/type/dscr.md#type).
+-   `y` — Description. Optional (only for optional arguments). Possible values: <values list>.Default value: <value>. [Type name](relative/path/to/type/dscr.md#type).

 **Parameters** (Optional, only for parametric aggregate functions)

@ -23,7 +23,7 @@ More text (Optional).

 **Returned value(s)**

-   Returned values list. 
+-   Returned values list.

 Type: [Type name](relative/path/to/type/dscr.md#type).

--- a/docs/_includes/cmake_in_clickhouse_footer.md
+++ b/docs/_includes/cmake_in_clickhouse_footer.md
@ -16,8 +16,8 @@ Better:
 option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF)
 ```

-If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some 
-pre-conditions, leave a comment above the `option()` line and explain what it does. 
+If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some
+pre-conditions, leave a comment above the `option()` line and explain what it does.
 The best way would be linking the docs page (if it exists).
 The comment is parsed into a separate column (see below).

@ -33,7 +33,7 @@ option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests"

 Suppose you have an option that may strip debug symbols from the ClickHouse's part.
 This can speed up the linking process, but produces a binary that cannot be debugged.
-In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. 
+In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong.
 Also, such options should be disabled if applies.

 Bad:
--- a/docs/en/commercial/support.md
+++ b/docs/en/commercial/support.md
@ -7,7 +7,7 @@ toc_title: Support

 !!! info "Info"
    If you have launched a ClickHouse commercial support service, feel free to [open a pull-request](https://github.com/ClickHouse/ClickHouse/edit/master/docs/en/commercial/support.md) adding it to the following list.
-    
+
 ## Yandex.Cloud

 ClickHouse worldwide support from the authors of ClickHouse. Supports on-premise and cloud deployments. Ask details on clickhouse-support@yandex-team.com
--- a/docs/en/development/adding_test_queries.md
+++ b/docs/en/development/adding_test_queries.md
@ -4,11 +4,11 @@ ClickHouse has hundreds (or even thousands) of features. Every commit gets check

 The core functionality is very well tested, but some corner-cases and different combinations of features can be uncovered with ClickHouse CI.

-Most of the bugs/regressions we see happen in that 'grey area' where test coverage is poor. 
+Most of the bugs/regressions we see happen in that 'grey area' where test coverage is poor.

-And we are very interested in covering most of the possible scenarios and feature combinations used in real life by tests. 
+And we are very interested in covering most of the possible scenarios and feature combinations used in real life by tests.

-## Why adding tests 
+## Why adding tests

 Why/when you should add a test case into ClickHouse code:
 1) you use some complicated scenarios / feature combinations / you have some corner case which is probably not widely used
@ -17,18 +17,18 @@ Why/when you should add a test case into ClickHouse code:
 4) once the test is added/accepted, you can be sure the corner case you check will never be accidentally broken.
 5) you will be a part of great open-source community
 6) your name will be visible in the `system.contributors` table!
-7) you will make a world bit better :) 
+7) you will make a world bit better :)

 ### Steps to do

-#### Prerequisite 
+#### Prerequisite

-I assume you run some Linux machine (you can use docker / virtual machines on other OS) and any modern browser / internet connection, and you have some basic Linux & SQL skills. 
+I assume you run some Linux machine (you can use docker / virtual machines on other OS) and any modern browser / internet connection, and you have some basic Linux & SQL skills.

 Any highly specialized knowledge is not needed (so you don't need to know C++ or know something about how ClickHouse CI works).


-#### Preparation 
+#### Preparation

 1) [create GitHub account](https://github.com/join) (if you haven't one yet)
 2) [setup git](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/set-up-git)
@ -54,17 +54,17 @@ git remote add upstream https://github.com/ClickHouse/ClickHouse

 #### New branch for the test

-1) create a new branch from the latest clickhouse master 
+1) create a new branch from the latest clickhouse master
 ```
 cd ~/workspace/ClickHouse
 git fetch upstream
-git checkout -b name_for_a_branch_with_my_test upstream/master 
+git checkout -b name_for_a_branch_with_my_test upstream/master
 ```

-#### Install & run clickhouse 
+#### Install & run clickhouse

 1) install `clickhouse-server` (follow [official docs](https://clickhouse.tech/docs/en/getting-started/install/))
-2) install test configurations (it will use Zookeeper mock implementation and adjust some settings) 
+2) install test configurations (it will use Zookeeper mock implementation and adjust some settings)
 ```
 cd ~/workspace/ClickHouse/tests/config
 sudo ./install.sh
@ -74,7 +74,7 @@ sudo ./install.sh
 sudo systemctl restart clickhouse-server
 ```

-#### Creating the test file 
+#### Creating the test file


 1) find the number for your test - find the file with the biggest number in `tests/queries/0_stateless/`
@ -86,7 +86,7 @@ tests/queries/0_stateless/01520_client_print_query_id.reference
 ```
 Currently, the last number for the test is `01520`, so my test will have the number `01521`

-2) create an SQL file with the next number and name of the feature you test 
+2) create an SQL file with the next number and name of the feature you test

 ```sh
 touch tests/queries/0_stateless/01521_dummy_test.sql
@ -112,16 +112,16 @@ clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee te
 	- fast - should not take longer than a few seconds (better subseconds)
 	- correct - fails then feature is not working
        - deterministic
-	- isolated / stateless 
+	- isolated / stateless
 		- don't rely on some environment things
-		- don't rely on timing when possible 
- try to cover corner cases (zeros / Nulls / empty sets / throwing exceptions) 
+		- don't rely on timing when possible
+- try to cover corner cases (zeros / Nulls / empty sets / throwing exceptions)
 - to test that query return errors, you can put special comment after the query: `-- { serverError 60 }` or `-- { clientError 20 }`
 - don't switch databases (unless necessary)
 - you can create several table replicas on the same node if needed
 - you can use one of the test cluster definitions when needed (see system.clusters)
 - use `number` / `numbers_mt` / `zeros` / `zeros_mt` and similar for queries / to initialize data when applicable
- clean up the created objects after test and before the test (DROP IF EXISTS) - in case of some dirty state 
+- clean up the created objects after test and before the test (DROP IF EXISTS) - in case of some dirty state
 - prefer sync mode of operations (mutations, merges, etc.)
 - use other SQL files in the `0_stateless` folder as an example
 - ensure the feature / feature combination you want to test is not yet covered with existing tests
@ -138,7 +138,7 @@ It's important to name tests correctly, so one could turn some tests subset off

 #### Commit / push / create PR.

-1) commit & push your changes 
+1) commit & push your changes
 ```sh
 cd ~/workspace/ClickHouse
 git add tests/queries/0_stateless/01521_dummy_test.sql
@ -147,5 +147,5 @@ git commit # use some nice commit message when possible
 git push origin HEAD
 ```
 2) use a link which was shown during the push, to create a PR into the main repo
-3) adjust the PR title and contents, in `Changelog category (leave one)` keep 
-`Build/Testing/Packaging Improvement`, fill the rest of the fields if you want. 
+3) adjust the PR title and contents, in `Changelog category (leave one)` keep
+`Build/Testing/Packaging Improvement`, fill the rest of the fields if you want.
--- a/docs/en/development/contrib.md
+++ b/docs/en/development/contrib.md
@ -8,7 +8,7 @@ toc_title: Third-Party Libraries Used
 The list of third-party libraries can be obtained by the following query:

 ``` sql
-SELECT library_name, license_type, license_path FROM system.licenses ORDER BY library_name COLLATE 'en'
+SELECT library_name, license_type, license_path FROM system.licenses ORDER BY library_name COLLATE 'en';
 ```

 [Example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIGxpYnJhcnlfbmFtZSwgbGljZW5zZV90eXBlLCBsaWNlbnNlX3BhdGggRlJPTSBzeXN0ZW0ubGljZW5zZXMgT1JERVIgQlkgbGlicmFyeV9uYW1lIENPTExBVEUgJ2VuJw==)
--- a/docs/en/development/developer-instruction.md
+++ b/docs/en/development/developer-instruction.md
@ -123,7 +123,7 @@ For installing CMake and Ninja on Mac OS X first install Homebrew and then insta
    /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
    brew install cmake ninja

-Next, check the version of CMake: `cmake --version`. If it is below 3.3, you should install a newer version from the website: https://cmake.org/download/.
+Next, check the version of CMake: `cmake --version`. If it is below 3.12, you should install a newer version from the website: https://cmake.org/download/.

 ## Optional External Libraries {#optional-external-libraries}

--- a/docs/en/development/style.md
+++ b/docs/en/development/style.md
@ -749,7 +749,7 @@ If your code in the `master` branch is not buildable yet, exclude it from the bu

 **1.** The C++20 standard library is used (experimental extensions are allowed), as well as `boost` and `Poco` frameworks.

-**2.** It is not allowed to use libraries from OS packages. It is also not allowed to use pre-installed libraries. All libraries should be placed in form of source code in `contrib` directory and built with ClickHouse.
+**2.** It is not allowed to use libraries from OS packages. It is also not allowed to use pre-installed libraries. All libraries should be placed in form of source code in `contrib` directory and built with ClickHouse. See [Guidelines for adding new third-party libraries](contrib.md#adding-third-party-libraries) for details.

 **3.** Preference is always given to libraries that are already in use.

--- a/docs/en/development/tests.md
+++ b/docs/en/development/tests.md
@ -70,7 +70,13 @@ Note that integration of ClickHouse with third-party drivers is not tested. Also

 Unit tests are useful when you want to test not the ClickHouse as a whole, but a single isolated library or class. You can enable or disable build of tests with `ENABLE_TESTS` CMake option. Unit tests (and other test programs) are located in `tests` subdirectories across the code. To run unit tests, type `ninja test`. Some tests use `gtest`, but some are just programs that return non-zero exit code on test failure.

-It’s not necessarily to have unit tests if the code is already covered by functional tests (and functional tests are usually much more simple to use).
+It’s not necessary to have unit tests if the code is already covered by functional tests (and functional tests are usually much more simple to use).
+
+You can run individual gtest checks by calling the executable directly, for example:
+
+```bash
+$ ./src/unit_tests_dbms --gtest_filter=LocalAddress*
+```

 ## Performance Tests {#performance-tests}

--- a/docs/en/engines/database-engines/atomic.md
+++ b/docs/en/engines/database-engines/atomic.md
@ -17,7 +17,7 @@ It supports non-blocking [DROP TABLE](#drop-detach-table) and [RENAME TABLE](#re

 ### Table UUID {#table-uuid}

-All tables in database `Atomic` have persistent [UUID](../../sql-reference/data-types/uuid.md) and store data in directory `/clickhouse_path/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/`, where `xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy` is UUID of the table. 
+All tables in database `Atomic` have persistent [UUID](../../sql-reference/data-types/uuid.md) and store data in directory `/clickhouse_path/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/`, where `xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy` is UUID of the table.
 Usually, the UUID is generated automatically, but the user can also explicitly specify the UUID in the same way when creating the table (this is not recommended). To display the `SHOW CREATE` query with the UUID you can use setting [show_table_uuid_in_table_create_query_if_not_nil](../../operations/settings/settings.md#show_table_uuid_in_table_create_query_if_not_nil). For example:

 ```sql
--- a/docs/en/engines/database-engines/index.md
+++ b/docs/en/engines/database-engines/index.md
@ -14,7 +14,7 @@ You can also use the following database engines:

 -   [MySQL](../../engines/database-engines/mysql.md)

-   [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md)
+-   [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md)

 -   [Lazy](../../engines/database-engines/lazy.md)

--- a/docs/en/engines/database-engines/materialized-mysql.md
+++ b/docs/en/engines/database-engines/materialized-mysql.md
@ -1,9 +1,11 @@
 ---
 toc_priority: 29
-toc_title: MaterializeMySQL
+toc_title: MaterializedMySQL
 ---

-# MaterializeMySQL {#materialize-mysql}
+# MaterializedMySQL {#materialized-mysql}
+
+**This is experimental feature that should not be used in production.**

 Creates ClickHouse database with all the tables existing in MySQL, and all the data in those tables.

@ -15,7 +17,7 @@ This feature is experimental.

 ``` sql
 CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
-ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
+ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
 ```

 **Engine Parameters**
@ -25,13 +27,35 @@ ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'passwor
 -   `user` — MySQL user.
 -   `password` — User password.

+**Engine Settings**
+-   `max_rows_in_buffer` — Max rows that data is allowed to cache in memory(for single table and the cache data unable to query). when rows is exceeded, the data will be materialized. Default: `65505`.
+-   `max_bytes_in_buffer` —  Max bytes that data is allowed to cache in memory(for single table and the cache data unable to query). when rows is exceeded, the data will be materialized. Default: `1048576`.
+-   `max_rows_in_buffers` — Max rows that data is allowed to cache in memory(for database and the cache data unable to query). when rows is exceeded, the data will be materialized. Default: `65505`.
+-   `max_bytes_in_buffers` — Max bytes that data is allowed to cache in memory(for database and the cache data unable to query). when rows is exceeded, the data will be materialized. Default: `1048576`.
+-   `max_flush_data_time` — Max milliseconds that data is allowed to cache in memory(for database and the cache data unable to query). when this time is exceeded, the data will be materialized. Default: `1000`.
+-   `max_wait_time_when_mysql_unavailable` — Retry interval when MySQL is not available (milliseconds). Negative value disable retry. Default: `1000`.
+-   `allows_query_when_mysql_lost` — Allow query materialized table when mysql is lost. Default: `0` (`false`).
+```
+CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***')
+     SETTINGS
+        allows_query_when_mysql_lost=true,
+        max_wait_time_when_mysql_unavailable=10000;
+```
+
+**Settings on MySQL-server side**
+
+For the correct work of `MaterializeMySQL`, there are few mandatory `MySQL`-side configuration settings that should be set:
+
+- `default_authentication_plugin = mysql_native_password` since `MaterializeMySQL` can only authorize with this method.
+- `gtid_mode = on` since GTID based logging is a mandatory for providing correct `MaterializeMySQL` replication. Pay attention that while turning this mode `On` you should also specify `enforce_gtid_consistency = on`.
+
 ## Virtual columns {#virtual-columns}

-When working with the `MaterializeMySQL` database engine, [ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md) tables are used with virtual `_sign` and `_version` columns.
- 
+When working with the `MaterializedMySQL` database engine, [ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md) tables are used with virtual `_sign` and `_version` columns.
+
 - `_version` — Transaction counter. Type [UInt64](../../sql-reference/data-types/int-uint.md).
 - `_sign` — Deletion mark. Type [Int8](../../sql-reference/data-types/int-uint.md). Possible values:
-    - `1` — Row is not deleted, 
+    - `1` — Row is not deleted,
    - `-1` — Row is deleted.

 ## Data Types Support {#data_types-support}
@ -53,6 +77,7 @@ When working with the `MaterializeMySQL` database engine, [ReplacingMergeTree](.
 | STRING                  | [String](../../sql-reference/data-types/string.md)           |
 | VARCHAR, VAR_STRING     | [String](../../sql-reference/data-types/string.md)           |
 | BLOB                    | [String](../../sql-reference/data-types/string.md)           |
+| BINARY                  | [FixedString](../../sql-reference/data-types/fixedstring.md) |

 Other types are not supported. If MySQL table contains a column of such type, ClickHouse throws exception "Unhandled data type" and stops replication.

@ -60,28 +85,38 @@ Other types are not supported. If MySQL table contains a column of such type, Cl

 ## Specifics and Recommendations {#specifics-and-recommendations}

+### Compatibility restrictions
+
+Apart of the data types limitations there are few restrictions comparing to `MySQL` databases, that should be resolved before replication will be possible:
+
+- Each table in `MySQL` should contain `PRIMARY KEY`.
+
+- Replication for tables, those are containing rows with `ENUM` field values out of range (specified in `ENUM` signature) will not work.
+
 ### DDL Queries {#ddl-queries}

 MySQL DDL queries are converted into the corresponding ClickHouse DDL queries ([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create/index.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md)). If ClickHouse cannot parse some DDL query, the query is ignored.

 ### Data Replication {#data-replication}

-`MaterializeMySQL` does not support direct `INSERT`, `DELETE` and `UPDATE` queries. However, they are supported in terms of data replication:
+`MaterializedMySQL` does not support direct `INSERT`, `DELETE` and `UPDATE` queries. However, they are supported in terms of data replication:

 - MySQL `INSERT` query is converted into `INSERT` with `_sign=1`.

- MySQL `DELETE` query is converted into `INSERT` with `_sign=-1`. 
+- MySQL `DELETE` query is converted into `INSERT` with `_sign=-1`.

 - MySQL `UPDATE` query is converted into `INSERT` with `_sign=-1` and `INSERT` with `_sign=1`.

-### Selecting from MaterializeMySQL Tables {#select}
+### Selecting from MaterializedMySQL Tables {#select}

-`SELECT` query from `MaterializeMySQL` tables has some specifics:
+`SELECT` query from `MaterializedMySQL` tables has some specifics:

 - If `_version` is not specified in the `SELECT` query, [FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier is used. So only rows with `MAX(_version)` are selected.

 - If `_sign` is not specified in the `SELECT` query, `WHERE _sign=1` is used by default. So the deleted rows are not included into the result set.

+- The result includes columns comments in case they exist in MySQL database tables.
+
 ### Index Conversion {#index-conversion}

 MySQL `PRIMARY KEY` and `INDEX` clauses are converted into `ORDER BY` tuples in ClickHouse tables.
@ -91,10 +126,10 @@ ClickHouse has only one physical order, which is determined by `ORDER BY` clause
 **Notes**

 - Rows with `_sign=-1` are not deleted physically from the tables.
- Cascade `UPDATE/DELETE` queries are not supported by the `MaterializeMySQL` engine.
+- Cascade `UPDATE/DELETE` queries are not supported by the `MaterializedMySQL` engine.
 - Replication can be easily broken.
 - Manual operations on database and tables are forbidden.
- `MaterializeMySQL` is influenced by [optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert) setting. The data is merged in the corresponding table in the `MaterializeMySQL` database when a table in the MySQL server changes.
+- `MaterializedMySQL` is influenced by [optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert) setting. The data is merged in the corresponding table in the `MaterializedMySQL` database when a table in the MySQL server changes.

 ## Examples of Use {#examples-of-use}

@ -111,9 +146,9 @@ mysql> SELECT * FROM test;
 ```

 ```text
-+---+------+------+ 
+---+------+------+
 | a |    b |    c |
-+---+------+------+ 
+---+------+------+
 | 2 |  222 | Wow! |
 +---+------+------+
 ```
@ -123,7 +158,7 @@ Database in ClickHouse, exchanging data with the MySQL server:
 The database and the table created:

 ``` sql
-CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***');
+CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***');
 SHOW TABLES FROM mysql;
 ```

@ -140,9 +175,9 @@ SELECT * FROM mysql.test;
 ```

 ``` text
-┌─a─┬──b─┐ 
-│ 1 │ 11 │ 
-│ 2 │ 22 │ 
+┌─a─┬──b─┐
+│ 1 │ 11 │
+│ 2 │ 22 │
 └───┴────┘
 ```

@ -153,9 +188,9 @@ SELECT * FROM mysql.test;
 ```

 ``` text
-┌─a─┬───b─┬─c────┐ 
-│ 2 │ 222 │ Wow! │ 
+┌─a─┬───b─┬─c────┐
+│ 2 │ 222 │ Wow! │
 └───┴─────┴──────┘
 ```

-[Original article](https://clickhouse.tech/docs/en/engines/database-engines/materialize-mysql/) <!--hide-->
+[Original article](https://clickhouse.tech/docs/en/engines/database-engines/materialized-mysql/) <!--hide-->
--- a/docs/en/engines/database-engines/mysql.md
+++ b/docs/en/engines/database-engines/mysql.md
@ -53,7 +53,7 @@ All other MySQL data types are converted into [String](../../sql-reference/data-

 ## Global Variables Support {#global-variables-support}

-For better compatibility you may address global variables in MySQL style, as `@@identifier`. 
+For better compatibility you may address global variables in MySQL style, as `@@identifier`.

 These variables are supported:
 - `version`
--- a/docs/en/engines/database-engines/postgresql.md
+++ b/docs/en/engines/database-engines/postgresql.md
@ -14,7 +14,7 @@ Supports table structure modifications (`ALTER TABLE ... ADD|DROP COLUMN`). If `
 ## Creating a Database {#creating-a-database}

 ``` sql
-CREATE DATABASE test_database 
+CREATE DATABASE test_database
 ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cache`]);
 ```

@ -43,14 +43,14 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cac
 | TEXT, CHAR       | [String](../../sql-reference/data-types/string.md)           |
 | INTEGER          | Nullable([Int32](../../sql-reference/data-types/int-uint.md))|
 | ARRAY            | [Array](../../sql-reference/data-types/array.md)             |
- 
+

 ## Examples of Use {#examples-of-use}

 Database in ClickHouse, exchanging data with the PostgreSQL server:

 ``` sql
-CREATE DATABASE test_database 
+CREATE DATABASE test_database
 ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', 1);
 ```

@ -102,7 +102,7 @@ SELECT * FROM test_database.test_table;
 └────────┴───────┘
 ```

-Consider the table structure was modified in PostgreSQL: 
+Consider the table structure was modified in PostgreSQL:

 ``` sql
 postgre> ALTER TABLE test_table ADD COLUMN data Text
--- a/docs/en/engines/database-engines/replicated.md
+++ b/docs/en/engines/database-engines/replicated.md
@ -1,6 +1,6 @@
 # [experimental] Replicated {#replicated}

-The engine is based on the [Atomic](../../engines/database-engines/atomic.md) engine. It supports replication of metadata via DDL log being written to ZooKeeper and executed on all of the replicas for a given database. 
+The engine is based on the [Atomic](../../engines/database-engines/atomic.md) engine. It supports replication of metadata via DDL log being written to ZooKeeper and executed on all of the replicas for a given database.

 One ClickHouse server can have multiple replicated databases running and updating at the same time. But there can't be multiple replicas of the same replicated database.

@ -20,9 +20,9 @@ One ClickHouse server can have multiple replicated databases running and updatin

 ## Specifics and Recommendations {#specifics-and-recommendations}

-DDL queries with `Replicated` database work in a similar way to [ON CLUSTER](../../sql-reference/distributed-ddl.md) queries, but with minor differences. 
+DDL queries with `Replicated` database work in a similar way to [ON CLUSTER](../../sql-reference/distributed-ddl.md) queries, but with minor differences.

-First, the DDL request tries to execute on the initiator (the host that originally received the request from the user). If the request is not fulfilled, then the user immediately receives an error, other hosts do not try to fulfill it. If the request has been successfully completed on the initiator, then all other hosts will automatically retry until they complete it. The initiator will try to wait for the query to be completed on other hosts (no longer than [distributed_ddl_task_timeout](../../operations/settings/settings.md#distributed_ddl_task_timeout)) and will return a table with the query execution statuses on each host. 
+First, the DDL request tries to execute on the initiator (the host that originally received the request from the user). If the request is not fulfilled, then the user immediately receives an error, other hosts do not try to fulfill it. If the request has been successfully completed on the initiator, then all other hosts will automatically retry until they complete it. The initiator will try to wait for the query to be completed on other hosts (no longer than [distributed_ddl_task_timeout](../../operations/settings/settings.md#distributed_ddl_task_timeout)) and will return a table with the query execution statuses on each host.

 The behavior in case of errors is regulated by the [distributed_ddl_output_mode](../../operations/settings/settings.md#distributed_ddl_output_mode) setting, for a `Replicated` database it is better to set it to `null_status_on_timeout` — i.e. if some hosts did not have time to execute the request for [distributed_ddl_task_timeout](../../operations/settings/settings.md#distributed_ddl_task_timeout), then do not throw an exception, but show the `NULL` status for them in the table.

@ -47,8 +47,8 @@ CREATE TABLE r.rmt (n UInt64) ENGINE=ReplicatedMergeTree ORDER BY n;
 ```

 ``` text
-┌─────hosts────────────┬──status─┬─error─┬─num_hosts_remaining─┬─num_hosts_active─┐ 
-│ shard1|replica1      │    0    │       │          2          │        0         │ 
+┌─────hosts────────────┬──status─┬─error─┬─num_hosts_remaining─┬─num_hosts_active─┐
+│ shard1|replica1      │    0    │       │          2          │        0         │
 │ shard1|other_replica │    0    │       │          1          │        0         │
 │ other_shard|r1       │    0    │       │          0          │        0         │
 └──────────────────────┴─────────┴───────┴─────────────────────┴──────────────────┘
@ -57,13 +57,13 @@ CREATE TABLE r.rmt (n UInt64) ENGINE=ReplicatedMergeTree ORDER BY n;
 Showing the system table:

 ``` sql
-SELECT cluster, shard_num, replica_num, host_name, host_address, port, is_local 
+SELECT cluster, shard_num, replica_num, host_name, host_address, port, is_local
 FROM system.clusters WHERE cluster='r';
 ```

 ``` text
-┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐ 
-│ r       │     1     │      1      │   node3   │  127.0.0.1   │ 9002 │     0    │ 
+┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐
+│ r       │     1     │      1      │   node3   │  127.0.0.1   │ 9002 │     0    │
 │ r       │     2     │      1      │   node2   │  127.0.0.1   │ 9001 │     0    │
 │ r       │     2     │      2      │   node1   │  127.0.0.1   │ 9000 │     1    │
 └─────────┴───────────┴─────────────┴───────────┴──────────────┴──────┴──────────┘
@ -78,9 +78,9 @@ node1 :) SELECT materialize(hostName()) AS host, groupArray(n) FROM r.d GROUP BY
 ```

 ``` text
-┌─hosts─┬─groupArray(n)─┐ 
-│ node1 │  [1,3,5,7,9]  │   
-│ node2 │  [0,2,4,6,8]  │    
+┌─hosts─┬─groupArray(n)─┐
+│ node1 │  [1,3,5,7,9]  │
+│ node2 │  [0,2,4,6,8]  │
 └───────┴───────────────┘
 ```

@ -93,8 +93,8 @@ node4 :) CREATE DATABASE r ENGINE=Replicated('some/path/r','other_shard','r2');
 The cluster configuration will look like this:

 ``` text
-┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐ 
-│ r       │     1     │      1      │   node3   │  127.0.0.1   │ 9002 │     0    │ 
+┌─cluster─┬─shard_num─┬─replica_num─┬─host_name─┬─host_address─┬─port─┬─is_local─┐
+│ r       │     1     │      1      │   node3   │  127.0.0.1   │ 9002 │     0    │
 │ r       │     1     │      2      │   node4   │  127.0.0.1   │ 9003 │     0    │
 │ r       │     2     │      1      │   node2   │  127.0.0.1   │ 9001 │     0    │
 │ r       │     2     │      2      │   node1   │  127.0.0.1   │ 9000 │     1    │
@ -108,8 +108,8 @@ node2 :) SELECT materialize(hostName()) AS host, groupArray(n) FROM r.d GROUP BY
 ```

 ```text
-┌─hosts─┬─groupArray(n)─┐ 
-│ node2 │  [1,3,5,7,9]  │   
-│ node4 │  [0,2,4,6,8]  │    
+┌─hosts─┬─groupArray(n)─┐
+│ node2 │  [1,3,5,7,9]  │
+│ node4 │  [0,2,4,6,8]  │
 └───────┴───────────────┘
 ```
--- a/docs/en/engines/table-engines/integrations/ExternalDistributed.md
+++ b/docs/en/engines/table-engines/integrations/ExternalDistributed.md
@ -35,7 +35,7 @@ The table structure can differ from the original table structure:
 -   `password` — User password.

 ## Implementation Details {#implementation-details}
-	
+
 Supports multiple replicas that must be listed by `|` and shards must be listed by `,`. For example:

 ```sql
--- a/docs/en/engines/table-engines/integrations/embedded-rocksdb.md
+++ b/docs/en/engines/table-engines/integrations/embedded-rocksdb.md
@ -20,7 +20,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]

 Required parameters:

-  `primary_key_name` – any column name in the column list.
+- `primary_key_name` – any column name in the column list.
 - `primary key` must be specified, it supports only one column in the primary key. The primary key will be serialized in binary as a `rocksdb key`.
 - columns other than the primary key will be serialized in binary as `rocksdb` value in corresponding order.
 - queries with key `equals` or `in` filtering will be optimized to multi keys lookup from `rocksdb`.
@ -39,4 +39,46 @@ ENGINE = EmbeddedRocksDB
 PRIMARY KEY key
 ```

+## Metrics
+
+There is also `system.rocksdb` table, that expose rocksdb statistics:
+
+```sql
+SELECT
+    name,
+    value
+FROM system.rocksdb
+
+┌─name──────────────────────┬─value─┐
+│ no.file.opens             │     1 │
+│ number.block.decompressed │     1 │
+└───────────────────────────┴───────┘
+```
+
+## Configuration
+
+You can also change any [rocksdb options](https://github.com/facebook/rocksdb/wiki/Option-String-and-Option-Map) using config:
+
+```xml
+<rocksdb>
+    <options>
+        <max_background_jobs>8</max_background_jobs>
+    </options>
+    <column_family_options>
+        <num_levels>2</num_levels>
+    </column_family_options>
+    <tables>
+        <table>
+            <name>TABLE</name>
+            <options>
+                <max_background_jobs>8</max_background_jobs>
+            </options>
+            <column_family_options>
+                <num_levels>2</num_levels>
+            </column_family_options>
+        </table>
+    </tables>
+</rocksdb>
+```
+
 [Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/embedded-rocksdb/) <!--hide-->
--- a/docs/en/engines/table-engines/integrations/mongodb.md
+++ b/docs/en/engines/table-engines/integrations/mongodb.md
@ -37,7 +37,7 @@ Table in ClickHouse which allows to read data from MongoDB collection:
 ``` text
 CREATE TABLE mongo_table
 (
-    key UInt64, 
+    key UInt64,
    data String
 ) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'testuser', 'clickhouse');
 ```
--- a/docs/en/engines/table-engines/integrations/postgresql.md
+++ b/docs/en/engines/table-engines/integrations/postgresql.md
@ -49,14 +49,14 @@ PostgreSQL `Array` types are converted into ClickHouse arrays.

 !!! info "Note"
    Be careful - in PostgreSQL an array data, created like a `type_name[]`, may contain multi-dimensional arrays of different dimensions in different table rows in same column. But in ClickHouse it is only allowed to have multidimensional arrays of the same count of dimensions in all table rows in same column.
-	
+
 Supports multiple replicas that must be listed by `|`. For example:

 ```sql
 CREATE TABLE test_replicas (id UInt32, name String) ENGINE = PostgreSQL(`postgres{2|3|4}:5432`, 'clickhouse', 'test_replicas', 'postgres', 'mysecretpassword');
 ```

-Replicas priority for PostgreSQL dictionary source is supported. The bigger the number in map, the less the priority. The highest priority is `0`. 
+Replicas priority for PostgreSQL dictionary source is supported. The bigger the number in map, the less the priority. The highest priority is `0`.

 In the example below replica `example01-1` has the highest priority:

--- a/docs/en/engines/table-engines/log-family/index.md
+++ b/docs/en/engines/table-engines/log-family/index.md
@ -14,6 +14,8 @@ Engines of the family:
 -   [Log](../../../engines/table-engines/log-family/log.md)
 -   [TinyLog](../../../engines/table-engines/log-family/tinylog.md)

+`Log` family table engines can store data to [HDFS](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-hdfs) or [S3](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-s3) distributed file systems.
+
 ## Common Properties {#common-properties}

 Engines:
--- a/docs/en/engines/table-engines/log-family/log.md
+++ b/docs/en/engines/table-engines/log-family/log.md
@ -5,10 +5,8 @@ toc_title: Log

 # Log {#log}

-Engine belongs to the family of log engines. See the common properties of log engines and their differences in the [Log Engine Family](../../../engines/table-engines/log-family/index.md) article.
+The engine belongs to the family of `Log` engines. See the common properties of `Log` engines and their differences in the [Log Engine Family](../../../engines/table-engines/log-family/index.md) article.

-Log differs from [TinyLog](../../../engines/table-engines/log-family/tinylog.md) in that a small file of “marks” resides with the column files. These marks are written on every data block and contain offsets that indicate where to start reading the file in order to skip the specified number of rows. This makes it possible to read table data in multiple threads.
+`Log` differs from [TinyLog](../../../engines/table-engines/log-family/tinylog.md) in that a small file of "marks" resides with the column files. These marks are written on every data block and contain offsets that indicate where to start reading the file in order to skip the specified number of rows. This makes it possible to read table data in multiple threads.
 For concurrent data access, the read operations can be performed simultaneously, while write operations block reads and each other.
-The Log engine does not support indexes. Similarly, if writing to a table failed, the table is broken, and reading from it returns an error. The Log engine is appropriate for temporary data, write-once tables, and for testing or demonstration purposes.
-
-[Original article](https://clickhouse.tech/docs/en/operations/table_engines/log/) <!--hide-->
+The `Log` engine does not support indexes. Similarly, if writing to a table failed, the table is broken, and reading from it returns an error. The `Log` engine is appropriate for temporary data, write-once tables, and for testing or demonstration purposes.
--- a/docs/en/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md
@ -728,7 +728,9 @@ During this time, they are not moved to other volumes or disks. Therefore, until

 ## Using S3 for Data Storage {#table_engine-mergetree-s3}

-`MergeTree` family table engines is able to store data to [S3](https://aws.amazon.com/s3/) using a disk with type `s3`.
+`MergeTree` family table engines can store data to [S3](https://aws.amazon.com/s3/) using a disk with type `s3`.
+
+This feature is under development and not ready for production. There are known drawbacks such as very low performance.

 Configuration markup:
 ``` xml
@ -762,11 +764,13 @@ Configuration markup:
 ```

 Required parameters:
-   `endpoint` — S3 endpoint url in `path` or `virtual hosted` [styles](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). Endpoint url should contain bucket and root path to store data.
+
+-   `endpoint` — S3 endpoint URL in `path` or `virtual hosted` [styles](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). Endpoint URL should contain a bucket and root path to store data.
 -   `access_key_id` — S3 access key id.
 -   `secret_access_key` — S3 secret access key.

 Optional parameters:
+
 -   `region` — S3 region name.
 -   `use_environment_credentials` — Reads AWS credentials from the Environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN if they exist. Default value is `false`.
 -   `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Default value is `false`.
@ -782,7 +786,6 @@ Optional parameters:
 -   `skip_access_check` — If true, disk access checks will not be performed on disk start-up. Default value is `false`.
 -   `server_side_encryption_customer_key_base64` — If specified, required headers for accessing S3 objects with SSE-C encryption will be set.

-
 S3 disk can be configured as `main` or `cold` storage:
 ``` xml
 <storage_configuration>
@ -821,4 +824,43 @@ S3 disk can be configured as `main` or `cold` storage:

 In case of `cold` option a data can be moved to S3 if local disk free size will be smaller than `move_factor * disk_size` or by TTL move rule.

-[Original article](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) <!--hide-->
+## Using HDFS for Data Storage {#table_engine-mergetree-hdfs}
+
+[HDFS](https://hadoop.apache.org/docs/r1.2.1/hdfs_design.html) is a distributed file system for remote data storage.
+
+`MergeTree` family table engines can store data to HDFS using a disk with type `HDFS`.
+
+Configuration markup:
+``` xml
+<yandex>
+    <storage_configuration>
+        <disks>
+            <hdfs>
+                <type>hdfs</type>
+                <endpoint>hdfs://hdfs1:9000/clickhouse/</endpoint>
+            </hdfs>
+        </disks>
+        <policies>
+            <hdfs>
+                <volumes>
+                    <main>
+                        <disk>hdfs</disk>
+                    </main>
+                </volumes>
+            </hdfs>
+        </policies>
+    </storage_configuration>
+
+    <merge_tree>
+        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
+    </merge_tree>
+</yandex>
+```
+
+Required parameters:
+
+-   `endpoint` — HDFS endpoint URL in `path` format. Endpoint URL should contain a root path to store data.
+
+Optional parameters:
+
+-   `min_bytes_for_seek` — The minimal number of bytes to use seek operation instead of sequential read. Default value: `1 Mb`.
--- a/docs/en/engines/table-engines/mergetree-family/replication.md
+++ b/docs/en/engines/table-engines/mergetree-family/replication.md
@ -101,7 +101,7 @@ For very large clusters, you can use different ZooKeeper clusters for different

 Replication is asynchronous and multi-master. `INSERT` queries (as well as `ALTER`) can be sent to any available server. Data is inserted on the server where the query is run, and then it is copied to the other servers. Because it is asynchronous, recently inserted data appears on the other replicas with some latency. If part of the replicas are not available, the data is written when they become available. If a replica is available, the latency is the amount of time it takes to transfer the block of compressed data over the network. The number of threads performing background tasks for replicated tables can be set by [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) setting.

-`ReplicatedMergeTree` engine uses a separate thread pool for replicated fetches. Size of the pool is limited by the [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) setting which can be tuned with a server restart. 
+`ReplicatedMergeTree` engine uses a separate thread pool for replicated fetches. Size of the pool is limited by the [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) setting which can be tuned with a server restart.

 By default, an INSERT query waits for confirmation of writing the data from only one replica. If the data was successfully written to only one replica and the server with this replica ceases to exist, the stored data will be lost. To enable getting confirmation of data writes from multiple replicas, use the `insert_quorum` option.

@ -155,7 +155,7 @@ CREATE TABLE table_name

 </details>

-As the example shows, these parameters can contain substitutions in curly brackets. The substituted values are taken from the «[macros](../../../operations/server-configuration-parameters/settings/#macros) section of the configuration file. 
+As the example shows, these parameters can contain substitutions in curly brackets. The substituted values are taken from the «[macros](../../../operations/server-configuration-parameters/settings/#macros) section of the configuration file.

 Example:

@ -198,7 +198,7 @@ In this case, you can omit arguments when creating tables:
 ``` sql
 CREATE TABLE table_name (
 	x UInt32
-) ENGINE = ReplicatedMergeTree 
+) ENGINE = ReplicatedMergeTree
 ORDER BY x;
 ```

@ -207,7 +207,7 @@ It is equivalent to:
 ``` sql
 CREATE TABLE table_name (
 	x UInt32
-) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/table_name', '{replica}') 
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/table_name', '{replica}')
 ORDER BY x;
 ```

--- a/docs/en/engines/table-engines/special/distributed.md
+++ b/docs/en/engines/table-engines/special/distributed.md
@ -37,6 +37,14 @@ Also, it accepts the following settings:

 - `max_delay_to_insert` - max delay of inserting data into Distributed table in seconds, if there are a lot of pending bytes for async send. Default 60.

+- `monitor_batch_inserts` - same as [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts)
+
+- `monitor_split_batch_on_failure` - same as [distributed_directory_monitor_split_batch_on_failure](../../../operations/settings/settings.md#distributed_directory_monitor_split_batch_on_failure)
+
+- `monitor_sleep_time_ms` - same as [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms)
+
+- `monitor_max_sleep_time_ms` - same as [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms)
+
 !!! note "Note"

    **Durability settings** (`fsync_...`):
--- a/docs/en/interfaces/http.md
+++ b/docs/en/interfaces/http.md
@ -16,7 +16,7 @@ $ curl 'http://localhost:8123/'
 Ok.
 ```

-Web UI can be accessed here: `http://localhost:8123/play`. 
+Web UI can be accessed here: `http://localhost:8123/play`.

 ![Web UI](../images/play.png)

--- a/docs/en/interfaces/third-party/integrations.md
+++ b/docs/en/interfaces/third-party/integrations.md
@ -43,7 +43,7 @@ toc_title: Integrations
 -   Monitoring
    -   [Graphite](https://graphiteapp.org)
        -   [graphouse](https://github.com/yandex/graphouse)
-        -   [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) +
+        -   [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse)
        -   [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse)
        -   [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - optimizes staled partitions in [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) if rules from [rollup configuration](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) could be applied
    -   [Grafana](https://grafana.com/)
--- a/docs/en/operations/clickhouse-keeper.md
+++ b/docs/en/operations/clickhouse-keeper.md
@ -30,21 +30,25 @@ Other common parameters are inherited from clickhouse-server config (`listen_hos

 Internal coordination settings are located in `<keeper_server>.<coordination_settings>` section:

-    `operation_timeout_ms` — timeout for a single client operation
-    `session_timeout_ms` — timeout for client session
-    `dead_session_check_period_ms` — how often clickhouse-keeper check dead sessions and remove them
-    `heart_beat_interval_ms` — how often a clickhouse-keeper leader will send heartbeats to followers
-    `election_timeout_lower_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election
-    `election_timeout_upper_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election
-    `rotate_log_storage_interval` — how many logs to store in a single file
-    `reserved_log_items` — how many coordination logs to store before compaction
-    `snapshot_distance` — how often clickhouse-keeper will create new snapshots (in the number of logs)
-    `snapshots_to_keep` — how many snapshots to keep
-    `stale_log_gap` — the threshold when leader consider follower as stale and send snapshot to it instead of logs
-    `force_sync` — call `fsync` on each write to coordination log
-    `raft_logs_level` — text logging level about coordination (trace, debug, and so on)
-    `shutdown_timeout` — wait to finish internal connections and shutdown
-    `startup_timeout` — if the server doesn't connect to other quorum participants in the specified timeout it will terminate
+-    `operation_timeout_ms` — timeout for a single client operation (default: 10000)
+-    `session_timeout_ms` — timeout for client session (default: 30000)
+-    `dead_session_check_period_ms` — how often clickhouse-keeper check dead sessions and remove them (default: 500)
+-    `heart_beat_interval_ms` — how often a clickhouse-keeper leader will send heartbeats to followers (default: 500)
+-    `election_timeout_lower_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election (default: 1000)
+-    `election_timeout_upper_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election (default: 2000)
+-    `rotate_log_storage_interval` — how many log records to store in a single file (default: 100000)
+-    `reserved_log_items` — how many coordination log records to store before compaction (default: 100000)
+-    `snapshot_distance` — how often clickhouse-keeper will create new snapshots (in the number of records in logs) (default: 100000)
+-    `snapshots_to_keep` — how many snapshots to keep (default: 3)
+-    `stale_log_gap` — the threshold when leader consider follower as stale and send snapshot to it instead of logs (default: 10000)
+-    `fresh_log_gap` - when node became fresh (default: 200)
+-    `max_requests_batch_size` - max size of batch in requests count before it will be sent to RAFT (default: 100)
+-    `force_sync` — call `fsync` on each write to coordination log (default: true)
+-    `quorum_reads` - execute read requests as writes through whole RAFT consesus with similar speed (default: false)
+-    `raft_logs_level` — text logging level about coordination (trace, debug, and so on) (default: system default)
+-    `auto_forwarding` - allow to forward write requests from followers to leader (default: true)
+-    `shutdown_timeout` — wait to finish internal connections and shutdown (ms) (default: 5000)
+-    `startup_timeout` — if the server doesn't connect to other quorum participants in the specified timeout it will terminate (ms) (default: 30000)

 Quorum configuration is located in `<keeper_server>.<raft_configuration>` section and contain servers description. The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The main parameters for each `<server>` are:

--- a/docs/en/operations/performance-test.md
+++ b/docs/en/operations/performance-test.md
@ -5,50 +5,67 @@ toc_title: Testing Hardware

 # How to Test Your Hardware with ClickHouse {#how-to-test-your-hardware-with-clickhouse}

-With this instruction you can run basic ClickHouse performance test on any server without installation of ClickHouse packages.
+You can run basic ClickHouse performance test on any server without installation of ClickHouse packages.

-1.  Go to “commits” page: https://github.com/ClickHouse/ClickHouse/commits/master
-2.  Click on the first green check mark or red cross with green “ClickHouse Build Check” and click on the “Details” link near “ClickHouse Build Check”. There is no such link in some commits, for example commits with documentation. In this case, choose the nearest commit having this link.
-3.  Copy the link to `clickhouse` binary for amd64 or aarch64.
-4.  ssh to the server and download it with wget:
+
+## Automated Run
+
+You can run benchmark with a single script.
+
+1. Download the script.
+```
+wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/hardware.sh
+```
+
+2. Run the script.
+```
+chmod a+x ./hardware.sh
+./hardware.sh
+```
+
+3. Copy the output and send it to clickhouse-feedback@yandex-team.com
+
+All the results are published here: https://clickhouse.tech/benchmark/hardware/
+
+
+## Manual Run
+
+Alternatively you can perform benchmark in the following steps.
+
+1.  ssh to the server and download the binary with wget:
 ```bash
-# These links are outdated, please obtain the fresh link from the "commits" page.
 # For amd64:
-wget https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse
+wget https://builds.clickhouse.tech/master/amd64/clickhouse
 # For aarch64:
-wget https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_special_build_check/clang-10-aarch64_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse
+wget https://builds.clickhouse.tech/master/aarch64/clickhouse
 # Then do:
 chmod a+x clickhouse
 ```
-5.  Download benchmark files:
+2.  Download benchmark files:
 ```bash
 wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/benchmark-new.sh
 chmod a+x benchmark-new.sh
 wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/queries.sql
 ```
-6.  Download test data according to the [Yandex.Metrica dataset](../getting-started/example-datasets/metrica.md) instruction (“hits” table containing 100 million rows).
+3.  Download test data according to the [Yandex.Metrica dataset](../getting-started/example-datasets/metrica.md) instruction (“hits” table containing 100 million rows).
 ```bash
 wget https://datasets.clickhouse.tech/hits/partitions/hits_100m_obfuscated_v1.tar.xz
 tar xvf hits_100m_obfuscated_v1.tar.xz -C .
 mv hits_100m_obfuscated_v1/* .
 ```
-7.  Run the server:
+4.  Run the server:
 ```bash
 ./clickhouse server
 ```
-8.  Check the data: ssh to the server in another terminal
+5.  Check the data: ssh to the server in another terminal
 ```bash
 ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated"
 100000000
 ```
-9.  Edit the benchmark-new.sh, change `clickhouse-client` to `./clickhouse client` and add `--max_memory_usage 100000000000` parameter.
-```bash
-mcedit benchmark-new.sh
-```
-10.  Run the benchmark:
+6.  Run the benchmark:
 ```bash
 ./benchmark-new.sh hits_100m_obfuscated
 ```
-11.  Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com
+7.  Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com

 All the results are published here: https://clickhouse.tech/benchmark/hardware/
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@ -34,6 +34,7 @@ Configuration template:
      <min_part_size>...</min_part_size>
      <min_part_size_ratio>...</min_part_size_ratio>
      <method>...</method>
+      <level>...</level>
    </case>
    ...
 </compression>
@ -43,7 +44,8 @@ Configuration template:

 -   `min_part_size` – The minimum size of a data part.
 -   `min_part_size_ratio` – The ratio of the data part size to the table size.
-   `method` – Compression method. Acceptable values: `lz4` or `zstd`.
+-   `method` – Compression method. Acceptable values: `lz4`, `lz4hc`, `zstd`.
+-   `level` – Compression level. See [Codecs](../../sql-reference/statements/create/table/#create-query-general-purpose-codecs).

 You can configure multiple `<case>` sections.

@ -62,10 +64,33 @@ If no conditions met for a data part, ClickHouse uses the `lz4` compression.
        <min_part_size>10000000000</min_part_size>
        <min_part_size_ratio>0.01</min_part_size_ratio>
        <method>zstd</method>
+        <level>1</level>
    </case>
 </compression>
 ```

+## encryption {#server-settings-encryption}
+
+Configures a command to obtain a key to be used by [encryption codecs](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). The command, or a shell script, is expected to write a Base64-encoded key of any length to the stdout.
+
+**Example**
+
+For Linux with systemd:
+
+```xml
+<encryption>
+    <key_command>/usr/bin/systemd-ask-password --id="clickhouse-server" --timeout=0 "Enter the ClickHouse encryption passphrase:" | base64</key_command>
+</encryption>
+```
+
+For other systems:
+
+```xml
+<encryption>
+    <key_command><![CDATA[IFS=; echo -n >/dev/tty "Enter the ClickHouse encryption passphrase: "; stty=`stty -F /dev/tty -g`; stty -F /dev/tty -echo; read k </dev/tty; stty -F /dev/tty "$stty"; echo -n $k | base64]]></key_command>
+</encryption>
+```
+
 ## custom_settings_prefixes {#custom_settings_prefixes}

 List of prefixes for [custom settings](../../operations/settings/index.md#custom_settings). The prefixes must be separated with commas.
@ -98,7 +123,7 @@ Default value: `1073741824` (1 GB).
 ```xml
 <core_dump>
    <size_limit>1073741824</size_limit>
-</core_dump> 
+</core_dump>
 ```
 ## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec}

@ -439,8 +464,8 @@ The server will need access to the public Internet via IPv4 (at the time of writ

 Keys:

-   `enabled` – Boolean flag to enable the feature, `false` by default. Set to `true` to allow sending crash reports. 
-   `endpoint` – You can override the Sentry endpoint URL for sending crash reports. It can be either a separate Sentry account or your self-hosted Sentry instance. Use the [Sentry DSN](https://docs.sentry.io/error-reporting/quickstart/?platform=native#configure-the-sdk) syntax. 
+-   `enabled` – Boolean flag to enable the feature, `false` by default. Set to `true` to allow sending crash reports.
+-   `endpoint` – You can override the Sentry endpoint URL for sending crash reports. It can be either a separate Sentry account or your self-hosted Sentry instance. Use the [Sentry DSN](https://docs.sentry.io/error-reporting/quickstart/?platform=native#configure-the-sdk) syntax.
 -   `anonymize` - Avoid attaching the server hostname to the crash report.
 -   `http_proxy` - Configure HTTP proxy for sending crash reports.
 -   `debug` - Sets the Sentry client into debug mode.
@ -502,7 +527,7 @@ The default `max_server_memory_usage` value is calculated as `memory_amount * ma

 ## max_server_memory_usage_to_ram_ratio {#max_server_memory_usage_to_ram_ratio}

-Defines the fraction of total physical RAM amount, available to the ClickHouse server. If the server tries to utilize more, the memory is cut down to the appropriate amount. 
+Defines the fraction of total physical RAM amount, available to the ClickHouse server. If the server tries to utilize more, the memory is cut down to the appropriate amount.

 Possible values:

@ -713,7 +738,7 @@ Keys for server/client settings:
 -   extendedVerification – Automatically extended verification of certificates after the session ends. Acceptable values: `true`, `false`.
 -   requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`.
 -   requireTLSv1_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`.
-   requireTLSv1 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`.
+-   requireTLSv1_2 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`.
 -   fips – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS.
 -   privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: `<privateKeyPassphraseHandler>`, `<name>KeyFileHandler</name>`, `<options><password>test</password></options>`, `</privateKeyPassphraseHandler>`.
 -   invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: `<invalidCertificateHandler> <name>ConsoleCertificateHandler</name> </invalidCertificateHandler>` .
@ -880,7 +905,7 @@ Parameters:
 -   `flush_interval_milliseconds` — Interval for flushing data from the buffer in memory to the table.

 **Example**
-```xml 
+```xml
 <yandex>
    <text_log>
        <level>notice</level>
--- a/docs/en/operations/settings/index.md
+++ b/docs/en/operations/settings/index.md
@ -31,7 +31,7 @@ Settings that can only be made in the server config file are not covered in this

 ## Custom Settings {#custom_settings}

-In addition to the common [settings](../../operations/settings/settings.md), users can define custom settings. 
+In addition to the common [settings](../../operations/settings/settings.md), users can define custom settings.

 A custom setting name must begin with one of predefined prefixes. The list of these prefixes must be declared in the [custom_settings_prefixes](../../operations/server-configuration-parameters/settings.md#custom_settings_prefixes) parameter in the server configuration file.

@ -48,7 +48,7 @@ SET custom_a = 123;
 To get the current value of a custom setting use `getSetting()` function:

 ```sql
-SELECT getSetting('custom_a');    
+SELECT getSetting('custom_a');
 ```

 **See Also**
--- a/docs/en/operations/settings/merge-tree-settings.md
+++ b/docs/en/operations/settings/merge-tree-settings.md
@ -280,14 +280,13 @@ Default value: `0`.

 ## check_sample_column_is_correct {#check_sample_column_is_correct}

-Enables to check column for sampling or sampling expression is correct at table creation.
+Enables the check at table creation, that the data type of a column for sampling or sampling expression is correct. The data type must be one of unsigned [integer types](../../sql-reference/data-types/int-uint.md): `UInt8`, `UInt16`, `UInt32`, `UInt64`.

 Possible values:

-   true  — Check column or sampling expression is correct at table creation.
-   false — Do not check column or sampling expression is correct at table creation.
+-   true  — The check is enabled.
+-   false — The check is disabled at table creation.

 Default value: `true`.

-By default, the ClickHouse server check column for sampling or sampling expression at table creation. If you already had tables with incorrect sampling expression, set value `false` to make ClickHouse server do not raise exception when ClickHouse server is starting.
-[Original article](https://clickhouse.tech/docs/en/operations/settings/merge_tree_settings/) <!--hide-->
+By default, the ClickHouse server checks at table creation the data type of a column for sampling or sampling expression. If you already have tables with incorrect sampling expression and do not want the server to raise an exception during startup, set `check_sample_column_is_correct` to `false`.
--- a/docs/en/operations/settings/query-complexity.md
+++ b/docs/en/operations/settings/query-complexity.md
@ -65,20 +65,20 @@ What to do when the volume of data read exceeds one of the limits: ‘throw’ o
 The following restrictions can be checked on each block (instead of on each row). That is, the restrictions can be broken a little.

 A maximum number of rows that can be read from a local table on a leaf node when running a distributed query. While
-distributed queries can issue a multiple sub-queries to each shard (leaf) - this limit will be checked only on the read 
-stage on the leaf nodes and ignored on results merging stage on the root node. For example, cluster consists of 2 shards 
-and each shard contains a table with 100 rows. Then distributed query which suppose to read all the data from both 
-tables with setting `max_rows_to_read=150` will fail as in total it will be 200 rows. While query 
+distributed queries can issue a multiple sub-queries to each shard (leaf) - this limit will be checked only on the read
+stage on the leaf nodes and ignored on results merging stage on the root node. For example, cluster consists of 2 shards
+and each shard contains a table with 100 rows. Then distributed query which suppose to read all the data from both
+tables with setting `max_rows_to_read=150` will fail as in total it will be 200 rows. While query
 with `max_rows_to_read_leaf=150` will succeed since leaf nodes will read 100 rows at max.

 ## max_bytes_to_read_leaf {#max-bytes-to-read-leaf}

-A maximum number of bytes (uncompressed data) that can be read from a local table on a leaf node when running 
-a distributed query. While distributed queries can issue a multiple sub-queries to each shard (leaf) - this limit will 
-be checked only on the read stage on the leaf nodes and ignored on results merging stage on the root node. 
-For example, cluster consists of 2 shards and each shard contains a table with 100 bytes of data. 
-Then distributed query which suppose to read all the data from both tables with setting `max_bytes_to_read=150` will fail 
-as in total it will be 200 bytes. While query with `max_bytes_to_read_leaf=150` will succeed since leaf nodes will read 
+A maximum number of bytes (uncompressed data) that can be read from a local table on a leaf node when running
+a distributed query. While distributed queries can issue a multiple sub-queries to each shard (leaf) - this limit will
+be checked only on the read stage on the leaf nodes and ignored on results merging stage on the root node.
+For example, cluster consists of 2 shards and each shard contains a table with 100 bytes of data.
+Then distributed query which suppose to read all the data from both tables with setting `max_bytes_to_read=150` will fail
+as in total it will be 200 bytes. While query with `max_bytes_to_read_leaf=150` will succeed since leaf nodes will read
 100 bytes at max.

 ## read_overflow_mode_leaf {#read-overflow-mode-leaf}
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@ -20,6 +20,29 @@ Possible values:
 -   `global` — Replaces the `IN`/`JOIN` query with `GLOBAL IN`/`GLOBAL JOIN.`
 -   `allow` — Allows the use of these types of subqueries.

+## prefer_global_in_and_join {#prefer-global-in-and-join}
+
+Enables the replacement of `IN`/`JOIN` operators with `GLOBAL IN`/`GLOBAL JOIN`.
+
+Possible values:
+
+-   0 — Disabled. `IN`/`JOIN` operators are not replaced with `GLOBAL IN`/`GLOBAL JOIN`.
+-   1 — Enabled. `IN`/`JOIN` operators are replaced with `GLOBAL IN`/`GLOBAL JOIN`.
+
+Default value: `0`.
+
+**Usage**
+
+Although `SET distributed_product_mode=global` can change the queries behavior for the distributed tables, it's not suitable for local tables or tables from external resources. Here is when the `prefer_global_in_and_join` setting comes into play.
+
+For example, we have query serving nodes that contain local tables, which are not suitable for distribution. We need to scatter their data on the fly during distributed processing with the `GLOBAL` keyword — `GLOBAL IN`/`GLOBAL JOIN`.
+
+Another use case of `prefer_global_in_and_join` is accessing tables created by external engines. This setting helps to reduce the number of calls to external sources while joining such tables: only one call per query.
+
+**See also:**
+
+-   [Distributed subqueries](../../sql-reference/operators/in.md#select-distributed-subqueries) for more information on how to use `GLOBAL IN`/`GLOBAL JOIN`
+
 ## enable_optimize_predicate_expression {#enable-optimize-predicate-expression}

 Turns on predicate pushdown in `SELECT` queries.
@ -153,6 +176,26 @@ Possible values:

 Default value: 1048576.

+## table_function_remote_max_addresses {#table_function_remote_max_addresses}
+
+Sets the maximum number of addresses generated from patterns for the [remote](../../sql-reference/table-functions/remote.md) function.
+
+Possible values:
+
+-   Positive integer.
+
+Default value: `1000`.
+
+##  glob_expansion_max_elements  {#glob_expansion_max_elements }
+
+Sets the maximum number of addresses generated from patterns for external storages and table functions (like [url](../../sql-reference/table-functions/url.md)) except the `remote` function.
+
+Possible values:
+
+-   Positive integer.
+
+Default value: `1000`.
+
 ## send_progress_in_http_headers {#settings-send_progress_in_http_headers}

 Enables or disables `X-ClickHouse-Progress` HTTP response headers in `clickhouse-server` responses.
@ -509,6 +552,23 @@ Possible values:

 Default value: `ALL`.

+## join_algorithm {#settings-join_algorithm}
+
+Specifies [JOIN](../../sql-reference/statements/select/join.md) algorithm.
+
+Possible values:
+
+- `hash` — [Hash join algorithm](https://en.wikipedia.org/wiki/Hash_join) is used.
+- `partial_merge` — [Sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join) is used.
+- `prefer_partial_merge` — ClickHouse always tries to use `merge` join if possible.
+- `auto` — ClickHouse tries to change `hash` join to `merge` join on the fly to avoid out of memory.
+
+Default value: `hash`.
+
+When using `hash` algorithm the right part of `JOIN` is uploaded into RAM.
+
+When using `partial_merge` algorithm ClickHouse sorts the data and dumps it to the disk. The `merge` algorithm in ClickHouse differs a bit from the classic realization. First ClickHouse sorts the right table by [join key](../../sql-reference/statements/select/join.md#select-join) in blocks and creates min-max index for sorted blocks. Then it sorts parts of left table by `join key` and joins them over right table. The min-max index is also used to skip unneeded right table blocks.
+
 ## join_any_take_last_row {#settings-join_any_take_last_row}

 Changes behaviour of join operations with `ANY` strictness.
@ -1214,7 +1274,7 @@ Default value: `3`.
 ## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}

 Controls quoting of 64-bit or bigger [integers](../../sql-reference/data-types/int-uint.md) (like `UInt64` or `Int128`) when they are output in a [JSON](../../interfaces/formats.md#json) format.
-Such integers are enclosed in quotes by default. This behavior is compatible with most JavaScript implementations. 
+Such integers are enclosed in quotes by default. This behavior is compatible with most JavaScript implementations.

 Possible values:

@ -2007,13 +2067,13 @@ Default value: 16.

 ## merge_selecting_sleep_ms {#merge_selecting_sleep_ms}

-Sleep time for merge selecting when no part selected, a lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters
+Sleep time for merge selecting when no part is selected. A lower setting triggers selecting tasks in `background_schedule_pool` frequently, which results in a large number of requests to Zookeeper in large-scale clusters.

 Possible values:

 -   Any positive integer.

-Default value: 5000 
+Default value: `5000`.

 ## parallel_distributed_insert_select {#parallel_distributed_insert_select}

@ -2890,7 +2950,7 @@ Result:
 └─────────────┘
 ```

-Note that this setting influences [Materialized view](../../sql-reference/statements/create/view.md#materialized) and [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md) behaviour.
+Note that this setting influences [Materialized view](../../sql-reference/statements/create/view.md#materialized) and [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md) behaviour.

 ## engine_file_empty_if_not_exists {#engine-file-empty_if-not-exists}

@ -3172,7 +3232,7 @@ Default value: `300`.

 ## distributed_ddl_task_timeout {#distributed_ddl_task_timeout}

-Sets timeout for DDL query responses from all hosts in cluster. If a DDL request has not been performed on all hosts, a response will contain a timeout error and a request will be executed in an async mode. Negative value means infinite. 
+Sets timeout for DDL query responses from all hosts in cluster. If a DDL request has not been performed on all hosts, a response will contain a timeout error and a request will be executed in an async mode. Negative value means infinite.

 Possible values:

--- a/docs/en/operations/system-tables/asynchronous_metric_log.md
+++ b/docs/en/operations/system-tables/asynchronous_metric_log.md
@ -33,7 +33,7 @@ SELECT * FROM system.asynchronous_metric_log LIMIT 10

 **See Also**

- [system.asynchronous_metrics](../system-tables/asynchronous_metrics.md) — Contains metrics, calculated periodically in the background. 
+- [system.asynchronous_metrics](../system-tables/asynchronous_metrics.md) — Contains metrics, calculated periodically in the background.
 - [system.metric_log](../system-tables/metric_log.md) — Contains history of metrics values from tables `system.metrics` and `system.events`, periodically flushed to disk.

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/asynchronous_metric_log) <!--hide-->
--- a/docs/en/operations/system-tables/columns.md
+++ b/docs/en/operations/system-tables/columns.md
@ -4,7 +4,7 @@ Contains information about columns in all the tables.

 You can use this table to get information similar to the [DESCRIBE TABLE](../../sql-reference/statements/misc.md#misc-describe-table) query, but for multiple tables at once.

-Columns from [temporary tables](../../sql-reference/statements/create/table.md#temporary-tables) are visible in the `system.columns` only in those session where they have been created. They are shown with the empty `database` field. 
+Columns from [temporary tables](../../sql-reference/statements/create/table.md#temporary-tables) are visible in the `system.columns` only in those session where they have been created. They are shown with the empty `database` field.

 Columns:

@ -38,17 +38,17 @@ database:                system
 table:                   aggregate_function_combinators
 name:                    name
 type:                    String
-default_kind:            
-default_expression:      
+default_kind:
+default_expression:
 data_compressed_bytes:   0
 data_uncompressed_bytes: 0
 marks_bytes:             0
-comment:                 
+comment:
 is_in_partition_key:     0
 is_in_sorting_key:       0
 is_in_primary_key:       0
 is_in_sampling_key:      0
-compression_codec:       
+compression_codec:

 Row 2:
 ──────
@ -56,17 +56,17 @@ database:                system
 table:                   aggregate_function_combinators
 name:                    is_internal
 type:                    UInt8
-default_kind:            
-default_expression:      
+default_kind:
+default_expression:
 data_compressed_bytes:   0
 data_uncompressed_bytes: 0
 marks_bytes:             0
-comment:                 
+comment:
 is_in_partition_key:     0
 is_in_sorting_key:       0
 is_in_primary_key:       0
 is_in_sampling_key:      0
-compression_codec:       
+compression_codec:
 ```

 The `system.columns` table contains the following columns (the column type is shown in brackets):
--- a/docs/en/operations/system-tables/disks.md
+++ b/docs/en/operations/system-tables/disks.md
@ -21,7 +21,7 @@ Columns:
 │ default │ /var/lib/clickhouse/ │ 276392587264 │ 490652508160 │               0 │
 └─────────┴──────────────────────┴──────────────┴──────────────┴─────────────────┘

-1 rows in set. Elapsed: 0.001 sec. 
+1 rows in set. Elapsed: 0.001 sec.
 ```

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/disks) <!--hide-->
--- a/docs/en/operations/system-tables/distributed_ddl_queue.md
+++ b/docs/en/operations/system-tables/distributed_ddl_queue.md
@ -62,4 +62,3 @@ exception_code:    ZOK
 ```

 [Original article](https://clickhouse.tech/docs/en/operations/system_tables/distributed_ddl_queuedistributed_ddl_queue.md) <!--hide-->
- 
--- a/docs/en/operations/system-tables/distribution_queue.md
+++ b/docs/en/operations/system-tables/distribution_queue.md
@ -40,7 +40,7 @@ is_blocked:            1
 error_count:           0
 data_files:            1
 data_compressed_bytes: 499
-last_exception:        
+last_exception:
 ```

 **See Also**
--- a/docs/en/operations/system-tables/enabled-roles.md
+++ b/docs/en/operations/system-tables/enabled-roles.md
@ -1,6 +1,6 @@
 # system.enabled_roles {#system_tables-enabled_roles}

-Contains all active roles at the moment, including current role of the current user and granted roles for current role. 
+Contains all active roles at the moment, including current role of the current user and granted roles for current role.

 Columns:

--- a/docs/en/operations/system-tables/functions.md
+++ b/docs/en/operations/system-tables/functions.md
@ -27,7 +27,7 @@ Columns:
 │ JSONExtractInt           │            0 │                0 │          │
 └──────────────────────────┴──────────────┴──────────────────┴──────────┘

-10 rows in set. Elapsed: 0.002 sec. 
+10 rows in set. Elapsed: 0.002 sec.
 ```

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/functions) <!--hide-->
--- a/docs/en/operations/system-tables/licenses.md
+++ b/docs/en/operations/system-tables/licenses.md
@ -1,11 +1,11 @@
 # system.licenses {#system-tables_system.licenses}

-Сontains licenses of third-party libraries that are located in the [contrib](https://github.com/ClickHouse/ClickHouse/tree/master/contrib) directory of ClickHouse sources. 
+Сontains licenses of third-party libraries that are located in the [contrib](https://github.com/ClickHouse/ClickHouse/tree/master/contrib) directory of ClickHouse sources.

 Columns:

 - `library_name` ([String](../../sql-reference/data-types/string.md)) — Name of the library, which is license connected with.
- `license_type` ([String](../../sql-reference/data-types/string.md)) — License type — e.g. Apache, MIT. 
+- `license_type` ([String](../../sql-reference/data-types/string.md)) — License type — e.g. Apache, MIT.
 - `license_path` ([String](../../sql-reference/data-types/string.md)) — Path to the file with the license text.
 - `license_text` ([String](../../sql-reference/data-types/string.md)) — License text.

--- a/docs/en/operations/system-tables/merge_tree_settings.md
+++ b/docs/en/operations/system-tables/merge_tree_settings.md
@ -48,7 +48,7 @@ changed:     0
 description: How many rows in blocks should be formed for merge operations.
 type:        SettingUInt64

-4 rows in set. Elapsed: 0.001 sec. 
+4 rows in set. Elapsed: 0.001 sec.
 ```

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/merge_tree_settings) <!--hide-->
--- a/docs/en/operations/system-tables/mutations.md
+++ b/docs/en/operations/system-tables/mutations.md
@ -1,6 +1,6 @@
 # system.mutations {#system_tables-mutations}

-The table contains information about [mutations](../../sql-reference/statements/alter/index.md#mutations) of [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables and their progress. Each mutation command is represented by a single row. 
+The table contains information about [mutations](../../sql-reference/statements/alter/index.md#mutations) of [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables and their progress. Each mutation command is represented by a single row.

 Columns:

@ -16,17 +16,17 @@ Columns:

 -   `block_numbers.partition_id` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — For mutations of replicated tables, the array contains the partitions' IDs (one record for each partition). For mutations of non-replicated tables the array is empty.

-   `block_numbers.number` ([Array](../../sql-reference/data-types/array.md)([Int64](../../sql-reference/data-types/int-uint.md))) — For mutations of replicated tables, the array contains one record for each partition, with the block number that was acquired by the mutation. Only parts that contain blocks with numbers less than this number will be mutated in the partition. 
-    
+-   `block_numbers.number` ([Array](../../sql-reference/data-types/array.md)([Int64](../../sql-reference/data-types/int-uint.md))) — For mutations of replicated tables, the array contains one record for each partition, with the block number that was acquired by the mutation. Only parts that contain blocks with numbers less than this number will be mutated in the partition.
+
    In non-replicated tables, block numbers in all partitions form a single sequence. This means that for mutations of non-replicated tables, the column will contain one record with a single block number acquired by the mutation.

 -   `parts_to_do_names` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — An array of names of data parts that need to be mutated for the mutation to complete.

 -   `parts_to_do` ([Int64](../../sql-reference/data-types/int-uint.md)) — The number of data parts that need to be mutated for the mutation to complete.

-   `is_done` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The flag whether the mutation is done or not. Possible values: 
+-   `is_done` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The flag whether the mutation is done or not. Possible values:
    -   `1` if the mutation is completed,
-    -   `0` if the mutation is still in process. 
+    -   `0` if the mutation is still in process.

 !!! info "Note"
    Even if `parts_to_do = 0` it is possible that a mutation of a replicated table is not completed yet because of a long-running `INSERT` query, that will create a new data part needed to be mutated.
--- a/docs/en/operations/system-tables/numbers.md
+++ b/docs/en/operations/system-tables/numbers.md
@ -26,7 +26,7 @@ Reads from this table are not parallelized.
 │      9 │
 └────────┘

-10 rows in set. Elapsed: 0.001 sec. 
+10 rows in set. Elapsed: 0.001 sec.
 ```

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/numbers) <!--hide-->
--- a/docs/en/operations/system-tables/numbers_mt.md
+++ b/docs/en/operations/system-tables/numbers_mt.md
@ -24,7 +24,7 @@ Used for tests.
 │      9 │
 └────────┘

-10 rows in set. Elapsed: 0.001 sec. 
+10 rows in set. Elapsed: 0.001 sec.
 ```

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/numbers_mt) <!--hide-->
--- a/docs/en/operations/system-tables/one.md
+++ b/docs/en/operations/system-tables/one.md
@ -17,7 +17,7 @@ This is similar to the `DUAL` table found in other DBMSs.
 │     0 │
 └───────┘

-1 rows in set. Elapsed: 0.001 sec. 
+1 rows in set. Elapsed: 0.001 sec.
 ```

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/one) <!--hide-->
--- a/docs/en/operations/system-tables/part_log.md
+++ b/docs/en/operations/system-tables/part_log.md
@ -63,7 +63,7 @@ read_rows:                     0
 read_bytes:                    0
 peak_memory_usage:             0
 error:                         0
-exception:                   
+exception:
 ```

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/part_log) <!--hide-->
--- a/docs/en/operations/system-tables/parts.md
+++ b/docs/en/operations/system-tables/parts.md
@ -19,10 +19,10 @@ Columns:

    Possible Values:

-    -   `Wide` — Each column is stored in a separate file in a filesystem. 
-    -   `Compact` — All columns are stored in one file in a filesystem. 
+    -   `Wide` — Each column is stored in a separate file in a filesystem.
+    -   `Compact` — All columns are stored in one file in a filesystem.

-    Data storing format is controlled by the `min_bytes_for_wide_part` and `min_rows_for_wide_part` settings of the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table. 
+    Data storing format is controlled by the `min_bytes_for_wide_part` and `min_rows_for_wide_part` settings of the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table.

   -   `active` ([UInt8](../../sql-reference/data-types/int-uint.md)) – Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging.

@ -88,7 +88,7 @@ Columns:

 -   `delete_ttl_info_max` ([DateTime](../../sql-reference/data-types/datetime.md)) — The maximum value of the date and time key for [TTL DELETE rule](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl).

-   `move_ttl_info.expression` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Array of expressions. Each expression defines a [TTL MOVE rule](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl). 
+-   `move_ttl_info.expression` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Array of expressions. Each expression defines a [TTL MOVE rule](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl).

    !!! note "Warning"
        The `move_ttl_info.expression` array is kept mostly for backward compatibility, now the simpliest way to check `TTL MOVE` rule is to use the `move_ttl_info.min` and `move_ttl_info.max` fields.
--- a/docs/en/operations/system-tables/parts_columns.md
+++ b/docs/en/operations/system-tables/parts_columns.md
@ -19,10 +19,10 @@ Columns:

    Possible values:

-    -   `Wide` — Each column is stored in a separate file in a filesystem. 
-    -   `Compact` — All columns are stored in one file in a filesystem. 
+    -   `Wide` — Each column is stored in a separate file in a filesystem.
+    -   `Compact` — All columns are stored in one file in a filesystem.

-    Data storing format is controlled by the `min_bytes_for_wide_part` and `min_rows_for_wide_part` settings of the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table. 
+    Data storing format is controlled by the `min_bytes_for_wide_part` and `min_rows_for_wide_part` settings of the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table.

 -   `active` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging.

--- a/docs/en/operations/system-tables/query_log.md
+++ b/docs/en/operations/system-tables/query_log.md
@ -51,6 +51,7 @@ Columns:
 -   `databases` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the databases present in the query.
 -   `tables` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the tables present in the query.
 -   `columns` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the columns present in the query.
+-   `projections` ([String](../../sql-reference/data-types/string.md)) — Names of the projections used during the query execution.
 -   `exception_code` ([Int32](../../sql-reference/data-types/int-uint.md)) — Code of an exception.
 -   `exception` ([String](../../sql-reference/data-types/string.md)) — Exception message.
 -   `stack_trace` ([String](../../sql-reference/data-types/string.md)) — [Stack trace](https://en.wikipedia.org/wiki/Stack_trace). An empty string, if the query was completed successfully.
@ -65,6 +66,8 @@ Columns:
 -   `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
 -   `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
 -   `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the parent query.
+-   `initial_query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Initial query starting time (for distributed query execution).
+-   `initial_query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Initial query starting time with microseconds precision (for distributed query execution).
 -   `interface` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Interface that the query was initiated from. Possible values:
    -   1 — TCP.
    -   2 — HTTP.
@ -101,55 +104,77 @@ Columns:
 **Example**

 ``` sql
-SELECT * FROM system.query_log WHERE type = 'QueryFinish' AND (query LIKE '%toDate(\'2000-12-05\')%') ORDER BY query_start_time DESC LIMIT 1 FORMAT Vertical;
+SELECT * FROM system.query_log WHERE type = 'QueryFinish' ORDER BY query_start_time DESC LIMIT 1 FORMAT Vertical;
 ```

 ``` text
 Row 1:
 ──────
-type:                          QueryStart
-event_date:                    2020-09-11
-event_time:                    2020-09-11 10:08:17
-event_time_microseconds:       2020-09-11 10:08:17.063321
-query_start_time:              2020-09-11 10:08:17
-query_start_time_microseconds: 2020-09-11 10:08:17.063321
-query_duration_ms:             0
-read_rows:                     0
-read_bytes:                    0
-written_rows:                  0
-written_bytes:                 0
-result_rows:                   0
-result_bytes:                  0
-memory_usage:                  0
-current_database:              default
-query:                         INSERT INTO test1 VALUES
-exception_code:                0
+type:                                  QueryFinish
+event_date:                            2021-07-28
+event_time:                            2021-07-28 13:46:56
+event_time_microseconds:               2021-07-28 13:46:56.719791
+query_start_time:                      2021-07-28 13:46:56
+query_start_time_microseconds:         2021-07-28 13:46:56.704542
+query_duration_ms:                     14
+read_rows:                             8393
+read_bytes:                            374325
+written_rows:                          0
+written_bytes:                         0
+result_rows:                           4201
+result_bytes:                          153024
+memory_usage:                          4714038
+current_database:                      default
+query:                                 SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)
+normalized_query_hash:                 6666026786019643712
+query_kind:                            Select
+databases:                             ['system']
+tables:                                ['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']
+columns:                               ['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']
+projections:                           []
+exception_code:                        0
 exception:
 stack_trace:
-is_initial_query:              1
-user:                          default
-query_id:                      50a320fd-85a8-49b8-8761-98a86bcbacef
-address:                       ::ffff:127.0.0.1
-port:                          33452
-initial_user:                  default
-initial_query_id:              50a320fd-85a8-49b8-8761-98a86bcbacef
-initial_address:               ::ffff:127.0.0.1
-initial_port:                  33452
-interface:                     1
-os_user:                       bharatnc
-client_hostname:               tower
-client_name:                   ClickHouse
-client_revision:               54437
-client_version_major:          20
-client_version_minor:          7
-client_version_patch:          2
-http_method:                   0
+is_initial_query:                      1
+user:                                  default
+query_id:                              a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf
+address:                               ::ffff:127.0.0.1
+port:                                  51006
+initial_user:                          default
+initial_query_id:                      a3361f6e-a1fd-4d54-9f6f-f93a08bab0bf
+initial_address:                       ::ffff:127.0.0.1
+initial_port:                          51006
+initial_query_start_time:              2021-07-28 13:46:56
+initial_query_start_time_microseconds: 2021-07-28 13:46:56.704542
+interface:                             1
+os_user:
+client_hostname:
+client_name:                           ClickHouse client
+client_revision:                       54449
+client_version_major:                  21
+client_version_minor:                  8
+client_version_patch:                  0
+http_method:                           0
 http_user_agent:
+http_referer:
+forwarded_for:
 quota_key:
-revision:                      54440
-thread_ids:                    []
-ProfileEvents:        {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
-Settings:             {'background_pool_size':'32','load_balancing':'random','allow_suspicious_low_cardinality_types':'1','distributed_aggregation_memory_efficient':'1','skip_unavailable_shards':'1','log_queries':'1','max_bytes_before_external_group_by':'20000000000','max_bytes_before_external_sort':'20000000000','allow_introspection_functions':'1'}
+revision:                              54453
+log_comment:
+thread_ids:                            [5058,22097,22110,22094]
+ProfileEvents.Names:                   ['Query','SelectQuery','ArenaAllocChunks','ArenaAllocBytes','FunctionExecute','NetworkSendElapsedMicroseconds','SelectedRows','SelectedBytes','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SystemTimeMicroseconds','SoftPageFaults','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSWriteChars']
+ProfileEvents.Values:                  [1,1,39,352256,64,360,8393,374325,412,440,34480,13108,4723,671,19,17828,8192,10240]
+Settings.Names:                        ['load_balancing','max_memory_usage']
+Settings.Values:                       ['random','10000000000']
+used_aggregate_functions:              []
+used_aggregate_function_combinators:   []
+used_database_engines:                 []
+used_data_type_families:               ['UInt64','UInt8','Nullable','String','date']
+used_dictionaries:                     []
+used_formats:                          []
+used_functions:                        ['concat','notEmpty','extractAll']
+used_storages:                         []
+used_table_functions:                  []
 ```

 **See Also**
--- a/docs/en/operations/system-tables/role-grants.md
+++ b/docs/en/operations/system-tables/role-grants.md
@ -13,9 +13,9 @@ Columns:
 - `granted_role_is_default` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Flag that shows whether `granted_role` is a default role. Possible values:
    -   1 — `granted_role` is a default role.
    -   0 — `granted_role` is not a default role.
-    
+
 - `with_admin_option` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Flag that shows whether `granted_role` is a role with [ADMIN OPTION](../../sql-reference/statements/grant.md#admin-option-privilege) privilege. Possible values:
    -   1 — The role has `ADMIN OPTION` privilege.
-    -   0 — The role without `ADMIN OPTION` privilege. 
+    -   0 — The role without `ADMIN OPTION` privilege.

 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/role-grants) <!--hide-->
--- a/docs/en/operations/system-tables/row_policies.md
+++ b/docs/en/operations/system-tables/row_policies.md
@ -13,7 +13,7 @@ Columns:

 -    `id` ([UUID](../../sql-reference/data-types/uuid.md)) — Row policy ID.

-    `storage` ([String](../../sql-reference/data-types/string.md)) — Name of the directory where the row policy is stored. 
+-    `storage` ([String](../../sql-reference/data-types/string.md)) — Name of the directory where the row policy is stored.

 -    `select_filter` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Condition which is used to filter rows.

--- a/docs/en/operations/system-tables/settings_profiles.md
+++ b/docs/en/operations/system-tables/settings_profiles.md
@ -7,7 +7,7 @@ Columns:

 -    `id` ([UUID](../../sql-reference/data-types/uuid.md)) — Setting profile ID.

-    `storage` ([String](../../sql-reference/data-types/string.md)) — Path to the storage of setting profiles. Configured in the `access_control_path` parameter. 
+-    `storage` ([String](../../sql-reference/data-types/string.md)) — Path to the storage of setting profiles. Configured in the `access_control_path` parameter.

 -    `num_elements` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of elements for this profile in the `system.settings_profile_elements` table.

--- a/docs/en/operations/system-tables/tables.md
+++ b/docs/en/operations/system-tables/tables.md
@ -1,24 +1,24 @@
 # system.tables {#system-tables}

-Contains metadata of each table that the server knows about. 
+Contains metadata of each table that the server knows about.

 [Detached](../../sql-reference/statements/detach.md) tables are not shown in `system.tables`.

-[Temporary tables](../../sql-reference/statements/create/table.md#temporary-tables) are visible in the `system.tables` only in those session where they have been created. They are shown with the empty `database` field and with the `is_temporary` flag switched on. 
+[Temporary tables](../../sql-reference/statements/create/table.md#temporary-tables) are visible in the `system.tables` only in those session where they have been created. They are shown with the empty `database` field and with the `is_temporary` flag switched on.

 Columns:

-   `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in. 
+-   `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in.

-   `name` ([String](../../sql-reference/data-types/string.md)) — Table name. 
+-   `name` ([String](../../sql-reference/data-types/string.md)) — Table name.

-   `engine` ([String](../../sql-reference/data-types/string.md)) — Table engine name (without parameters). 
+-   `engine` ([String](../../sql-reference/data-types/string.md)) — Table engine name (without parameters).

-   `is_temporary` ([UInt8](../../sql-reference/data-types/int-uint.md)) - Flag that indicates whether the table is temporary. 
+-   `is_temporary` ([UInt8](../../sql-reference/data-types/int-uint.md)) - Flag that indicates whether the table is temporary.

-   `data_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table data in the file system. 
+-   `data_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table data in the file system.

-   `metadata_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table metadata in the file system. 
+-   `metadata_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table metadata in the file system.

 -   `metadata_modification_time` ([DateTime](../../sql-reference/data-types/datetime.md)) - Time of latest modification of the table metadata.

@ -28,33 +28,33 @@ Columns:

 -   `create_table_query` ([String](../../sql-reference/data-types/string.md)) - The query that was used to create the table.

-   `engine_full` ([String](../../sql-reference/data-types/string.md)) - Parameters of the table engine. 
+-   `engine_full` ([String](../../sql-reference/data-types/string.md)) - Parameters of the table engine.

-   `partition_key` ([String](../../sql-reference/data-types/string.md)) - The partition key expression specified in the table. 
+-   `partition_key` ([String](../../sql-reference/data-types/string.md)) - The partition key expression specified in the table.

-   `sorting_key` ([String](../../sql-reference/data-types/string.md)) - The sorting key expression specified in the table. 
+-   `sorting_key` ([String](../../sql-reference/data-types/string.md)) - The sorting key expression specified in the table.

-   `primary_key` ([String](../../sql-reference/data-types/string.md)) - The primary key expression specified in the table. 
+-   `primary_key` ([String](../../sql-reference/data-types/string.md)) - The primary key expression specified in the table.

-   `sampling_key` ([String](../../sql-reference/data-types/string.md)) - The sampling key expression specified in the table. 
+-   `sampling_key` ([String](../../sql-reference/data-types/string.md)) - The sampling key expression specified in the table.

 -   `storage_policy` ([String](../../sql-reference/data-types/string.md)) - The storage policy:

    -   [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes)
    -   [Distributed](../../engines/table-engines/special/distributed.md#distributed)

-   `total_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of rows, if it is possible to quickly determine exact number of rows in the table, otherwise `NULL` (including underying `Buffer` table). 
+-   `total_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of rows, if it is possible to quickly determine exact number of rows in the table, otherwise `NULL` (including underying `Buffer` table).

-   `total_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of bytes, if it is possible to quickly determine exact number of bytes for the table on storage, otherwise `NULL` (does not includes any underlying storage). 
+-   `total_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of bytes, if it is possible to quickly determine exact number of bytes for the table on storage, otherwise `NULL` (does not includes any underlying storage).

    -   If the table stores data on disk, returns used space on disk (i.e. compressed).
    -   If the table stores data in memory, returns approximated number of used bytes in memory.

-   `lifetime_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of rows INSERTed since server start (only for `Buffer` tables). 
+-   `lifetime_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of rows INSERTed since server start (only for `Buffer` tables).

-   `lifetime_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of bytes INSERTed since server start (only for `Buffer` tables). 
+-   `lifetime_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of bytes INSERTed since server start (only for `Buffer` tables).

-   `comment` ([String](../../sql-reference/data-types/string.md)) - The comment for the table. 
+-   `comment` ([String](../../sql-reference/data-types/string.md)) - The comment for the table.


 The `system.tables` table is used in `SHOW TABLES` query implementation.
--- a/docs/en/operations/system-tables/text_log.md
+++ b/docs/en/operations/system-tables/text_log.md
@ -42,12 +42,12 @@ microseconds:            871397
 thread_name:             clickhouse-serv
 thread_id:               564917
 level:                   Information
-query_id:                
+query_id:
 logger_name:             DNSCacheUpdater
 message:                 Update period 15 seconds
 revision:                54440
 source_file:             /ClickHouse/src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start()
 source_line:             45
 ```
- 
+
 [Original article](https://clickhouse.tech/docs/en/operations/system-tables/text_log) <!--hide-->
--- a/docs/en/operations/system-tables/trace_log.md
+++ b/docs/en/operations/system-tables/trace_log.md
@ -49,7 +49,7 @@ timestamp_ns:            1599762189872924510
 revision:                54440
 trace_type:              Memory
 thread_id:               564963
-query_id:                
+query_id:
 trace:                   [371912858,371912789,371798468,371799717,371801313,371790250,624462773,566365041,566440261,566445834,566460071,566459914,566459842,566459580,566459469,566459389,566459341,566455774,371993941,371988245,372158848,372187428,372187309,372187093,372185478,140222123165193,140222122205443]
 size:                    5244400
 ```
--- a/docs/en/operations/system-tables/users.md
+++ b/docs/en/operations/system-tables/users.md
@ -7,7 +7,7 @@ Columns:

 -    `id` ([UUID](../../sql-reference/data-types/uuid.md)) — User ID.

-    `storage` ([String](../../sql-reference/data-types/string.md)) — Path to the storage of users. Configured in the `access_control_path` parameter. 
+-    `storage` ([String](../../sql-reference/data-types/string.md)) — Path to the storage of users. Configured in the `access_control_path` parameter.

 -    `auth_type` ([Enum8](../../sql-reference/data-types/enum.md)('no_password' = 0,'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3)) — Shows the authentication type. There are multiple ways of user identification: with no password, with plain text password, with [SHA256](https://ru.wikipedia.org/wiki/SHA-2)-encoded password or with [double SHA-1](https://ru.wikipedia.org/wiki/SHA-1)-encoded password.

--- a/docs/en/operations/update.md
+++ b/docs/en/operations/update.md
@ -16,12 +16,12 @@ $ sudo service clickhouse-server restart
 If you installed ClickHouse using something other than the recommended `deb` packages, use the appropriate update method.

 !!! note "Note"
-    You can update multiple servers at once as soon as there is no moment when all replicas of one shard are offline.    
+    You can update multiple servers at once as soon as there is no moment when all replicas of one shard are offline.

 The upgrade of older version of ClickHouse to specific version:

 As an example:
- 
+
 `xx.yy.a.b` is a current stable version. The latest stable version could be found [here](https://github.com/ClickHouse/ClickHouse/releases)

 ```bash
--- a/docs/en/operations/utilities/clickhouse-benchmark.md
+++ b/docs/en/operations/utilities/clickhouse-benchmark.md
@ -40,7 +40,7 @@ clickhouse-benchmark [keys] < queries_file;

 ## Keys {#clickhouse-benchmark-keys}

-   `--query=QUERY` — Query to execute. If this parameter is not passed, `clickhouse-benchmark` will read queries from standard input. 
+-   `--query=QUERY` — Query to execute. If this parameter is not passed, `clickhouse-benchmark` will read queries from standard input.
 -   `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` sends simultaneously. Default value: 1.
 -   `-d N`, `--delay=N` — Interval in seconds between intermediate reports (to disable reports set 0). Default value: 1.
 -   `-h HOST`, `--host=HOST` — Server host. Default value: `localhost`. For the [comparison mode](#clickhouse-benchmark-comparison-mode) you can use multiple `-h` keys.
--- a/docs/en/operations/utilities/clickhouse-copier.md
+++ b/docs/en/operations/utilities/clickhouse-copier.md
@ -74,7 +74,7 @@ Parameters:
                source cluster & destination clusters accept exactly the same
                parameters as parameters for the usual Distributed table
                see https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/
-            --> 
+            -->
            <shard>
                <internal_replication>false</internal_replication>
                    <replica>
--- a/docs/en/operations/utilities/clickhouse-format.md
+++ b/docs/en/operations/utilities/clickhouse-format.md
@ -18,7 +18,7 @@ Keys:
 - `--seed <string>` — Seed arbitrary string that determines the result of obfuscation.
 - `--backslash` — Add a backslash at the end of each line of the formatted query. Can be useful when you copy a query from web or somewhere else with multiple lines, and want to execute it in command line.

-## Examples {#examples} 
+## Examples {#examples}

 1. Highlighting and single line:

@ -32,12 +32,12 @@ Result:
 SELECT sum(number) FROM numbers(5)
 ```

-2. Multiqueries: 
+2. Multiqueries:

 ```bash
 $ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNION DISTINCT SELECT 3);"
 ```
-    
+
 Result:

 ```text
@ -58,13 +58,13 @@ FROM
 ```bash
 $ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWEEN a AND b, CASE WHEN x >= 123 THEN y ELSE NULL END;"
 ```
-    
+
 Result:

 ```text
 SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END;
 ```
-   
+
 Same query and another seed string:

 ```bash
@ -95,4 +95,4 @@ FROM  \
    UNION DISTINCT \
    SELECT 3 \
 )
-``` 
+```
--- a/Show More
+++ b/Show More