Merge branch 'master' into LessReadInOrder

CHANGELOG.md

@@ -488,6 +488,7 @@
 * Remove `is_deterministic` field from the `system.functions` table. [#66630](https://github.com/ClickHouse/ClickHouse/pull/66630) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Function `tuple` will now try to construct named tuples in query (controlled by `enable_named_columns_in_function_tuple`). Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)).
 * Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
+* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)).
 
 #### New Feature
 * Add `ASOF JOIN` support for `full_sorting_join` algorithm. [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)).
@@ -599,7 +600,6 @@
 * Functions `bitTest`, `bitTestAll`, and `bitTestAny` now return an error if the specified bit index is out-of-bounds [#65818](https://github.com/ClickHouse/ClickHouse/pull/65818) ([Pablo Marcos](https://github.com/pamarcos)).
 * Setting `join_any_take_last_row` is supported in any query with hash join. [#65820](https://github.com/ClickHouse/ClickHouse/pull/65820) ([vdimir](https://github.com/vdimir)).
 * Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
-* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)).
 * Fix growing memory usage in S3Queue. [#65839](https://github.com/ClickHouse/ClickHouse/pull/65839) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix tie handling in `arrayAUC` to match sklearn. [#65840](https://github.com/ClickHouse/ClickHouse/pull/65840) ([gabrielmcg44](https://github.com/gabrielmcg44)).
 * Fix possible issues with MySQL server protocol TLS connections. [#65917](https://github.com/ClickHouse/ClickHouse/pull/65917) ([Azat Khuzhin](https://github.com/azat)).

CMakeLists.txt

@@ -88,6 +88,7 @@ string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
 list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
 
 option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF)
+option (ENABLE_FUZZER_TEST "Build testing fuzzers in order to test libFuzzer functionality" OFF)
 
 if (ENABLE_FUZZING)
     # Also set WITH_COVERAGE=1 for better fuzzing process

SECURITY.md

@@ -14,9 +14,10 @@ The following versions of ClickHouse server are currently supported with securit
 
 | Version | Supported |
 |:-|:-|
+| 24.10 | ✔️ |
 | 24.9 | ✔️ |
 | 24.8 | ✔️ |
-| 24.7 | ✔️ |
+| 24.7 | ❌ |
 | 24.6 | ❌ |
 | 24.5 | ❌ |
 | 24.4 | ❌ |

contrib/SimSIMD

@@ -1 +1 @@
-Subproject commit 935fef2964bc38e995c5f465b42259a35b8cf0d3
+Subproject commit ee3c9c9c00b51645f62a1a9e99611b78c0052a21

contrib/SimSIMD-cmake/CMakeLists.txt

@@ -1,4 +1,8 @@
-set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD")
+# See contrib/usearch-cmake/CMakeLists.txt, why only enabled on x86
-
+if (ARCH_AMD64)
-add_library(_simsimd INTERFACE)
+    set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD")
-target_include_directories(_simsimd SYSTEM INTERFACE "${SIMSIMD_PROJECT_DIR}/include")
+    set(SIMSIMD_SRCS ${SIMSIMD_PROJECT_DIR}/c/lib.c)
+    add_library(_simsimd ${SIMSIMD_SRCS})
+    target_include_directories(_simsimd SYSTEM PUBLIC "${SIMSIMD_PROJECT_DIR}/include")
+    target_compile_definitions(_simsimd PUBLIC SIMSIMD_DYNAMIC_DISPATCH)
+endif()

contrib/usearch

@@ -1 +1 @@
-Subproject commit 53799b84ca9ad708b060d0b1cfa5f039371721cd
+Subproject commit 7efe8b710c9831bfe06573b1df0fad001b04a2b5

contrib/usearch-cmake/CMakeLists.txt

@@ -6,12 +6,63 @@ target_include_directories(_usearch SYSTEM INTERFACE ${USEARCH_PROJECT_DIR}/incl
 target_link_libraries(_usearch INTERFACE _fp16)
 target_compile_definitions(_usearch INTERFACE USEARCH_USE_FP16LIB)
 
-# target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD)
+# Only x86 for now. On ARM, the linker goes down in flames. To make SimSIMD compile, I had to remove a macro checks in SimSIMD
-# ^^ simsimd is not enabled at the moment. Reasons:
+# for AVX512 (x86, worked nicely) and __ARM_BF16_FORMAT_ALTERNATIVE. It is probably because of that.
-# - Vectorization is important for raw scans but not so much for HNSW. We use usearch only for HNSW.
+if (ARCH_AMD64)
-# - Simsimd does compile-time dispatch (choice of SIMD kernels determined by capabilities of the build machine) or dynamic dispatch (SIMD
+    target_link_libraries(_usearch INTERFACE _simsimd)
-#   kernels chosen at runtime based on cpuid instruction). Since current builds are limited to SSE 4.2 (x86) and NEON (ARM), the speedup of
+    target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD)
-#   the former would be moderate compared to AVX-512 / SVE. The latter is at the moment too fragile with respect to portability across x86
+
-#   and ARM machines ... certain conbinations of quantizations / distance functions / SIMD instructions are not implemented at the moment.
+    target_compile_definitions(_usearch INTERFACE USEARCH_CAN_COMPILE_FLOAT16)
+    target_compile_definitions(_usearch INTERFACE USEARCH_CAN_COMPILE_BF16)
+endif ()
 
 add_library(ch_contrib::usearch ALIAS _usearch)
+
+
+# Cf. https://github.com/llvm/llvm-project/issues/107810 (though it is not 100% the same stack)
+#
+# LLVM ERROR: Cannot select: 0x7996e7a73150: f32,ch = load<(load (s16) from %ir.22, !tbaa !54231), anyext from bf16> 0x79961cb737c0, 0x7996e7a1a500, undef:i64, ./contrib/SimSIMD/include/simsimd/dot.h:215:1
+#   0x7996e7a1a500: i64 = add 0x79961e770d00, Constant:i64<-16>, ./contrib/SimSIMD/include/simsimd/dot.h:215:1
+#     0x79961e770d00: i64,ch = CopyFromReg 0x79961cb737c0, Register:i64 %4, ./contrib/SimSIMD/include/simsimd/dot.h:215:1
+#       0x7996e7a1ae10: i64 = Register %4
+#     0x7996e7a1b5f0: i64 = Constant<-16>
+#   0x7996e7a1a730: i64 = undef
+# In function: _ZL23simsimd_dot_bf16_serialPKu6__bf16S0_yPd
+# PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
+# Stack dump:
+# 0.      Running pass 'Function Pass Manager' on module 'src/libdbms.a(MergeTreeIndexVectorSimilarity.cpp.o at 2312737440)'.
+# 1.      Running pass 'AArch64 Instruction Selection' on function '@_ZL23simsimd_dot_bf16_serialPKu6__bf16S0_yPd'
+#  #0 0x00007999e83a63bf llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda63bf)
+#  #1 0x00007999e83a44f9 llvm::sys::RunSignalHandlers() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda44f9)
+#  #2 0x00007999e83a6b00 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xda6b00)
+#  #3 0x00007999e6e45320 (/lib/x86_64-linux-gnu/libc.so.6+0x45320)
+#  #4 0x00007999e6e9eb1c pthread_kill (/lib/x86_64-linux-gnu/libc.so.6+0x9eb1c)
+#  #5 0x00007999e6e4526e raise (/lib/x86_64-linux-gnu/libc.so.6+0x4526e)
+#  #6 0x00007999e6e288ff abort (/lib/x86_64-linux-gnu/libc.so.6+0x288ff)
+#  #7 0x00007999e82fe0c2 llvm::report_fatal_error(llvm::Twine const&, bool) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xcfe0c2)
+#  #8 0x00007999e8c2f8e3 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162f8e3)
+#  #9 0x00007999e8c2ed76 llvm::SelectionDAGISel::SelectCodeCommon(llvm::SDNode*, unsigned char const*, unsigned int) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162ed76)
+# #10 0x00007999ea1adbcb (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x2badbcb)
+# #11 0x00007999e8c2611f llvm::SelectionDAGISel::DoInstructionSelection() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x162611f)
+# #12 0x00007999e8c25790 llvm::SelectionDAGISel::CodeGenAndEmitDAG() (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x1625790)
+# #13 0x00007999e8c248de llvm::SelectionDAGISel::SelectAllBasicBlocks(llvm::Function const&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x16248de)
+# #14 0x00007999e8c22934 llvm::SelectionDAGISel::runOnMachineFunction(llvm::MachineFunction&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x1622934)
+# #15 0x00007999e87826b9 llvm::MachineFunctionPass::runOnFunction(llvm::Function&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x11826b9)
+# #16 0x00007999e84f7772 llvm::FPPassManager::runOnFunction(llvm::Function&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xef7772)
+# #17 0x00007999e84fd2f4 llvm::FPPassManager::runOnModule(llvm::Module&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xefd2f4)
+# #18 0x00007999e84f7e9f llvm::legacy::PassManagerImpl::run(llvm::Module&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xef7e9f)
+# #19 0x00007999e99f7d61 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f7d61)
+# #20 0x00007999e99f8c91 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f8c91)
+# #21 0x00007999e99f8b10 llvm::lto::thinBackend(llvm::lto::Config const&, unsigned int, std::function<llvm::Expected<std::unique_ptr<llvm::CachedFileStream, std::default_delete<llvm::CachedFileStream>>> (unsigned int, llvm::Twine const&)>, llvm::Module&, llvm::ModuleSummaryIndex const&, llvm::DenseMap<llvm::StringRef, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long>>, llvm::DenseMapInfo<llvm::StringRef, void
+# >, llvm::detail::DenseMapPair<llvm::StringRef, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long>>>> const&, llvm::DenseMap<unsigned long, llvm::GlobalValueSummary*, llvm::DenseMapInfo<unsigned long, void>, llvm::detail::DenseMapPair<unsigned long, llvm::GlobalValueSummary*>> const&, llvm::MapVector<llvm::StringRef, llvm::BitcodeModule, llvm::DenseMap<llvm::StringRef, unsigned int, llvm::DenseMapInfo<llvm::S
+# tringRef, void>, llvm::detail::DenseMapPair<llvm::StringRef, unsigned int>>, llvm::SmallVector<std::pair<llvm::StringRef, llvm::BitcodeModule>, 0u>>*, std::vector<unsigned char, std::allocator<unsigned char>> const&) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f8b10)
+# #22 0x00007999e99f248d (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f248d)
+# #23 0x00007999e99f1cd6 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0x23f1cd6)
+# #24 0x00007999e82c9beb (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xcc9beb)
+# #25 0x00007999e834ebe3 llvm::ThreadPool::processTasks(llvm::ThreadPoolTaskGroup*) (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xd4ebe3)
+# #26 0x00007999e834f704 (/usr/lib/llvm-18/bin/../lib/libLLVM.so.18.1+0xd4f704)
+# #27 0x00007999e6e9ca94 (/lib/x86_64-linux-gnu/libc.so.6+0x9ca94)
+# #28 0x00007999e6f29c3c (/lib/x86_64-linux-gnu/libc.so.6+0x129c3c)
+# clang++-18: error: unable to execute command: Aborted (core dumped)
+# clang++-18: error: linker command failed due to signal (use -v to see invocation)
+# ^[[A^Cninja: build stopped: interrupted by user.

docker/keeper/Dockerfile

@@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.9.2.42"
+ARG VERSION="24.10.1.2812"
 ARG PACKAGES="clickhouse-keeper"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/server/Dockerfile.alpine

@@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.9.2.42"
+ARG VERSION="24.10.1.2812"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/server/Dockerfile.ubuntu

@@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="24.9.2.42"
+ARG VERSION="24.10.1.2812"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 #docker-official-library:off

docker/test/libfuzzer/Dockerfile

@@ -33,8 +33,6 @@ RUN apt-get update \
 COPY requirements.txt /
 RUN pip3 install --no-cache-dir -r /requirements.txt
 
-ENV FUZZER_ARGS="-max_total_time=60"
-
 SHELL ["/bin/bash", "-c"]
 
 # docker run --network=host --volume <workspace>:/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer

docs/en/interfaces/prometheus.md

@@ -9,7 +9,7 @@ sidebar_label: Prometheus protocols
 ## Exposing metrics {#expose}
 
 :::note
-ClickHouse Cloud does not currently support connecting to Prometheus. To be notified when this feature is supported, please contact support@clickhouse.com.
+If you are using ClickHouse Cloud, you can expose metrics to Prometheus using the [Prometheus Integration](/en/integrations/prometheus).
 :::
 
 ClickHouse can expose its own metrics for scraping from Prometheus:

docs/en/operations/system-tables/grants.md

@@ -19,7 +19,7 @@ Columns:
 - `column` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Name of a column to which access is granted.
 
 - `is_partial_revoke` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Logical value. It shows whether some privileges have been revoked. Possible values:
-- `0` — The row describes a partial revoke.
+- `0` — The row describes a grant.
-- `1` — The row describes a grant.
+- `1` — The row describes a partial revoke.
 
 - `grant_option` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Permission is granted `WITH GRANT OPTION`, see [GRANT](../../sql-reference/statements/grant.md#granting-privilege-syntax).

programs/server/Server.cpp

@@ -1353,9 +1353,11 @@ try
     }
 
     FailPointInjection::enableFromGlobalConfig(config());
+#endif
 
     memory_worker.start();
 
+#if defined(OS_LINUX)
     int default_oom_score = 0;
 
 #if !defined(NDEBUG)

src/Access/AccessControl.cpp

@@ -608,7 +608,7 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po
     }
     catch (...)
     {
-        tryLogCurrentException(getLogger(), "from: " + address.toString() + ", user: " + credentials.getUserName()  + ": Authentication failed");
+        tryLogCurrentException(getLogger(), "from: " + address.toString() + ", user: " + credentials.getUserName()  + ": Authentication failed", LogsLevel::information);
 
         WriteBufferFromOwnString message;
         message << credentials.getUserName() << ": Authentication failed: password is incorrect, or there is no user with such name.";
@@ -622,8 +622,9 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po
                 << "and deleting this file will reset the password.\n"
                 << "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed.\n\n";
 
-        /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons,
+        /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons.
-        /// only the log will show the exact reason.
+        /// Only the log ((*), above) will show the exact reason. Note that (*) logs at information level instead of the default error level as
+        /// authentication failures are not an unusual event.
         throw Exception(PreformattedMessage{message.str(),
                                             "{}: Authentication failed: password is incorrect, or there is no user with such name",
                                             std::vector<std::string>{credentials.getUserName()}},

src/AggregateFunctions/fuzzers/CMakeLists.txt

@@ -1,2 +1,2 @@
 clickhouse_add_executable(aggregate_function_state_deserialization_fuzzer aggregate_function_state_deserialization_fuzzer.cpp ${SRCS})
-target_link_libraries(aggregate_function_state_deserialization_fuzzer PRIVATE clickhouse_aggregate_functions)
+target_link_libraries(aggregate_function_state_deserialization_fuzzer PRIVATE clickhouse_aggregate_functions dbms)

src/Common/Exception.cpp

@@ -251,7 +251,7 @@ void Exception::setThreadFramePointers(ThreadFramePointersBase frame_pointers)
         thread_frame_pointers.frame_pointers = std::move(frame_pointers);
 }
 
-static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message)
+static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message, LogsLevel level)
 {
     if (!isLoggingEnabled())
         return;
@@ -262,14 +262,25 @@ static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string
         if (!start_of_message.empty())
             message.text = fmt::format("{}: {}", start_of_message, message.text);
 
-        LOG_ERROR(logger, message);
+        switch (level)
+        {
+            case LogsLevel::none: break;
+            case LogsLevel::test: LOG_TEST(logger, message); break;
+            case LogsLevel::trace: LOG_TRACE(logger, message); break;
+            case LogsLevel::debug: LOG_DEBUG(logger, message); break;
+            case LogsLevel::information: LOG_INFO(logger, message); break;
+            case LogsLevel::warning: LOG_WARNING(logger, message); break;
+            case LogsLevel::error: LOG_ERROR(logger, message); break;
+            case LogsLevel::fatal: LOG_FATAL(logger, message); break;
+        }
+
     }
     catch (...) // NOLINT(bugprone-empty-catch)
     {
     }
 }
 
-void tryLogCurrentException(const char * log_name, const std::string & start_of_message)
+void tryLogCurrentException(const char * log_name, const std::string & start_of_message, LogsLevel level)
 {
     if (!isLoggingEnabled())
         return;
@@ -283,10 +294,10 @@ void tryLogCurrentException(const char * log_name, const std::string & start_of_
 
     /// getLogger can allocate memory too
     auto logger = getLogger(log_name);
-    tryLogCurrentExceptionImpl(logger.get(), start_of_message);
+    tryLogCurrentExceptionImpl(logger.get(), start_of_message, level);
 }
 
-void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message)
+void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message, LogsLevel level)
 {
     /// Under high memory pressure, new allocations throw a
     /// MEMORY_LIMIT_EXCEEDED exception.
@@ -295,17 +306,17 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_
     /// MemoryTracker until the exception will be logged.
     LockMemoryExceptionInThread lock_memory_tracker(VariableContext::Global);
 
-    tryLogCurrentExceptionImpl(logger, start_of_message);
+    tryLogCurrentExceptionImpl(logger, start_of_message, level);
 }
 
-void tryLogCurrentException(LoggerPtr logger, const std::string & start_of_message)
+void tryLogCurrentException(LoggerPtr logger, const std::string & start_of_message, LogsLevel level)
 {
-    tryLogCurrentException(logger.get(), start_of_message);
+    tryLogCurrentException(logger.get(), start_of_message, level);
 }
 
-void tryLogCurrentException(const AtomicLogger & logger, const std::string & start_of_message)
+void tryLogCurrentException(const AtomicLogger & logger, const std::string & start_of_message, LogsLevel level)
 {
-    tryLogCurrentException(logger.load(), start_of_message);
+    tryLogCurrentException(logger.load(), start_of_message, level);
 }
 
 static void getNoSpaceLeftInfoMessage(std::filesystem::path path, String & msg)

src/Common/Exception.h

@@ -7,6 +7,7 @@
 #include <Common/Logger.h>
 #include <Common/LoggingFormatStringHelpers.h>
 #include <Common/StackTrace.h>
+#include <Core/LogsLevel.h>
 
 #include <cerrno>
 #include <exception>
@@ -276,10 +277,10 @@ using Exceptions = std::vector<std::exception_ptr>;
   * Can be used in destructors in the catch-all block.
   */
 /// TODO: Logger leak constexpr overload
-void tryLogCurrentException(const char * log_name, const std::string & start_of_message = "");
+void tryLogCurrentException(const char * log_name, const std::string & start_of_message = "", LogsLevel level = LogsLevel::error);
-void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message = "");
+void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message = "", LogsLevel level = LogsLevel::error);
-void tryLogCurrentException(LoggerPtr logger, const std::string & start_of_message = "");
+void tryLogCurrentException(LoggerPtr logger, const std::string & start_of_message = "", LogsLevel level = LogsLevel::error);
-void tryLogCurrentException(const AtomicLogger & logger, const std::string & start_of_message = "");
+void tryLogCurrentException(const AtomicLogger & logger, const std::string & start_of_message = "", LogsLevel level = LogsLevel::error);
 
 
 /** Prints current exception in canonical format.

src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp

@@ -568,7 +568,7 @@ std::vector<std::string> NamedCollectionsMetadataStorage::listCollections() cons
     std::vector<std::string> collections;
     collections.reserve(paths.size());
     for (const auto & path : paths)
-        collections.push_back(std::filesystem::path(path).stem());
+        collections.push_back(unescapeForFileName(std::filesystem::path(path).stem()));
     return collections;
 }
 

src/Common/ProfileEvents.cpp

@@ -547,6 +547,7 @@ The server successfully detected this situation and will download merged part fr
     M(FilesystemCacheLoadMetadataMicroseconds, "Time spent loading filesystem cache metadata", ValueType::Microseconds) \
     M(FilesystemCacheEvictedBytes, "Number of bytes evicted from filesystem cache", ValueType::Bytes) \
     M(FilesystemCacheEvictedFileSegments, "Number of file segments evicted from filesystem cache", ValueType::Number) \
+    M(FilesystemCacheBackgroundDownloadQueuePush, "Number of file segments sent for background download in filesystem cache", ValueType::Number) \
     M(FilesystemCacheEvictionSkippedFileSegments, "Number of file segments skipped for eviction because of being in unreleasable state", ValueType::Number) \
     M(FilesystemCacheEvictionSkippedEvictingFileSegments, "Number of file segments skipped for eviction because of being in evicting state", ValueType::Number) \
     M(FilesystemCacheEvictionTries, "Number of filesystem cache eviction attempts", ValueType::Number) \
@@ -745,6 +746,12 @@ The server successfully detected this situation and will download merged part fr
     M(ReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the remote server side.", ValueType::Microseconds) \
     M(MergeTreeReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the remote server side.", ValueType::Microseconds) \
     M(MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds, "Time spent in sending the announcement from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.", ValueType::Microseconds) \
+    M(MergerMutatorsGetPartsForMergeElapsedMicroseconds, "Time spent to take data parts snapshot to build ranges from them.", ValueType::Microseconds) \
+    M(MergerMutatorPrepareRangesForMergeElapsedMicroseconds, "Time spent to prepare parts ranges which can be merged according to merge predicate.", ValueType::Microseconds) \
+    M(MergerMutatorSelectPartsForMergeElapsedMicroseconds, "Time spent to select parts from ranges which can be merged.", ValueType::Microseconds) \
+    M(MergerMutatorRangesForMergeCount, "Amount of candidate ranges for merge", ValueType::Number) \
+    M(MergerMutatorPartsInRangesForMergeCount, "Amount of candidate parts for merge", ValueType::Number) \
+    M(MergerMutatorSelectRangePartsCount, "Amount of parts in selected range for merge", ValueType::Number) \
     \
     M(ConnectionPoolIsFullMicroseconds, "Total time spent waiting for a slot in connection pool.", ValueType::Microseconds) \
     M(AsyncLoaderWaitMicroseconds, "Total time a query was waiting for async loader jobs.", ValueType::Microseconds) \

src/Core/Joins.h

@@ -119,15 +119,4 @@ enum class JoinTableSide : uint8_t
 
 const char * toString(JoinTableSide join_table_side);
 
-/// Setting to choose which table to use as the inner table in hash join
-enum class JoinInnerTableSelectionMode : uint8_t
-{
-    /// Use left table
-    Left,
-    /// Use right table
-    Right,
-    /// Use the table with the smallest number of rows
-    Auto,
-};
-
 }

src/Core/Settings.cpp

@@ -1912,9 +1912,6 @@ See also:
 For single JOIN in case of identifier ambiguity prefer left table
 )", IMPORTANT) \
     \
-    DECLARE(JoinInnerTableSelectionMode, query_plan_join_inner_table_selection, JoinInnerTableSelectionMode::Auto, R"(
-Select the side of the join to be the inner table in the query plan. Supported only for `ALL` join strictness with `JOIN ON` clause. Possible values: 'auto', 'left', 'right'.
-)", 0) \
     DECLARE(UInt64, preferred_block_size_bytes, 1000000, R"(
 This setting adjusts the data block size for query processing and represents additional fine-tuning to the more rough 'max_block_size' setting. If the columns are large and with 'max_block_size' rows the block size is likely to be larger than the specified amount of bytes, its size will be lowered for better CPU cache locality.
 )", 0) \
@@ -4242,7 +4239,7 @@ Rewrite aggregate functions with if expression as argument when logically equiva
 For example, `avg(if(cond, col, null))` can be rewritten to `avgOrNullIf(cond, col)`. It may improve performance.
 
 :::note
-Supported only with experimental analyzer (`enable_analyzer = 1`).
+Supported only with the analyzer (`enable_analyzer = 1`).
 :::
 )", 0) \
     DECLARE(Bool, optimize_rewrite_array_exists_to_has, false, R"(
@@ -5135,6 +5132,12 @@ Only in ClickHouse Cloud. A window for sending ACK for DataPacket sequence in a
 )", 0) \
     DECLARE(Bool, distributed_cache_discard_connection_if_unread_data, true, R"(
 Only in ClickHouse Cloud. Discard connection if some data is unread.
+)", 0) \
+    DECLARE(Bool, filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage, true, R"(
+Only in ClickHouse Cloud. Wait time to lock cache for space reservation in filesystem cache
+)", 0) \
+    DECLARE(Bool, filesystem_cache_enable_background_download_during_fetch, true, R"(
+Only in ClickHouse Cloud. Wait time to lock cache for space reservation in filesystem cache
 )", 0) \
     \
     DECLARE(Bool, parallelize_output_from_storages, true, R"(
@@ -5145,6 +5148,7 @@ The setting allows a user to provide own deduplication semantic in MergeTree/Rep
 For example, by providing a unique value for the setting in each INSERT statement,
 user can avoid the same inserted data being deduplicated.
 
+
 Possible values:
 
 - Any string
@@ -5619,7 +5623,7 @@ If true, and JOIN can be executed with parallel replicas algorithm, and all stor
     DECLARE(UInt64, parallel_replicas_mark_segment_size, 0, R"(
 Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing. Value should be in range [128; 16384]
 )", BETA) \
-    DECLARE(Bool, parallel_replicas_local_plan, false, R"(
+    DECLARE(Bool, parallel_replicas_local_plan, true, R"(
 Build local plan for local replica
 )", BETA) \
     \

src/Core/Settings.h

@@ -66,7 +66,6 @@ class WriteBuffer;
     M(CLASS_NAME, IntervalOutputFormat) \
     M(CLASS_NAME, JoinAlgorithm) \
     M(CLASS_NAME, JoinStrictness) \
-    M(CLASS_NAME, JoinInnerTableSelectionMode) \
     M(CLASS_NAME, LightweightMutationProjectionMode) \
     M(CLASS_NAME, LoadBalancing) \
     M(CLASS_NAME, LocalFSReadMethod) \

src/Core/SettingsChangesHistory.cpp

@@ -66,13 +66,15 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
         {
             {"read_in_order_use_virtual_row", false, false, "Use virtual row while reading in order of primary key or its monotonic function fashion. It is useful when searching over multiple parts as only relevant ones are touched."},
             {"distributed_cache_discard_connection_if_unread_data", true, true, "New setting"},
+            {"filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage", true, true, "New setting"},
+            {"filesystem_cache_enable_background_download_during_fetch", true, true, "New setting"},
             {"azure_check_objects_after_upload", false, false, "Check each uploaded object in azure blob storage to be sure that upload was successful"},
             {"backup_restore_keeper_max_retries", 20, 1000, "Should be big enough so the whole operation BACKUP or RESTORE operation won't fail because of a temporary [Zoo]Keeper failure in the middle of it."},
             {"backup_restore_failure_after_host_disconnected_for_seconds", 0, 3600, "New setting."},
             {"backup_restore_keeper_max_retries_while_initializing", 0, 20, "New setting."},
             {"backup_restore_keeper_max_retries_while_handling_error", 0, 20, "New setting."},
             {"backup_restore_finish_timeout_after_error_sec", 0, 180, "New setting."},
-            {"query_plan_join_inner_table_selection", "auto", "auto", "New setting."},
+            {"parallel_replicas_local_plan", false, true, "Use local plan for local replica in a query with parallel replicas"},
         }
     },
     {"24.10",

src/Core/SettingsEnums.cpp

@@ -55,10 +55,6 @@ IMPLEMENT_SETTING_MULTI_ENUM(JoinAlgorithm, ErrorCodes::UNKNOWN_JOIN,
      {"full_sorting_merge",   JoinAlgorithm::FULL_SORTING_MERGE},
      {"grace_hash",           JoinAlgorithm::GRACE_HASH}})
 
-IMPLEMENT_SETTING_ENUM(JoinInnerTableSelectionMode, ErrorCodes::BAD_ARGUMENTS,
-    {{"left",       JoinInnerTableSelectionMode::Left},
-     {"right",      JoinInnerTableSelectionMode::Right},
-     {"auto",       JoinInnerTableSelectionMode::Auto}})
 
 IMPLEMENT_SETTING_ENUM(TotalsMode, ErrorCodes::UNKNOWN_TOTALS_MODE,
     {{"before_having",          TotalsMode::BEFORE_HAVING},

src/Core/SettingsEnums.h

@@ -128,8 +128,8 @@ constexpr auto getEnumValues();
 DECLARE_SETTING_ENUM(LoadBalancing)
 
 DECLARE_SETTING_ENUM(JoinStrictness)
+
 DECLARE_SETTING_MULTI_ENUM(JoinAlgorithm)
-DECLARE_SETTING_ENUM(JoinInnerTableSelectionMode)
 
 
 /// Which rows should be included in TOTALS.

src/Core/fuzzers/CMakeLists.txt

@@ -1,2 +1,2 @@
 clickhouse_add_executable (names_and_types_fuzzer names_and_types_fuzzer.cpp)
-target_link_libraries (names_and_types_fuzzer PRIVATE)
+target_link_libraries (names_and_types_fuzzer PRIVATE dbms)

src/DataTypes/fuzzers/CMakeLists.txt

@@ -1,2 +1,3 @@
 clickhouse_add_executable(data_type_deserialization_fuzzer data_type_deserialization_fuzzer.cpp ${SRCS})
-target_link_libraries(data_type_deserialization_fuzzer PRIVATE clickhouse_aggregate_functions)
+
+target_link_libraries(data_type_deserialization_fuzzer PRIVATE clickhouse_aggregate_functions dbms)

src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp

@@ -3,6 +3,7 @@
 #include <IO/ReadBufferFromMemory.h>
 #include <IO/ReadHelpers.h>
 
+#include <DataTypes/IDataType.h>
 #include <DataTypes/DataTypeFactory.h>
 
 #include <Common/MemoryTracker.h>

src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp

@@ -535,7 +535,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext()
     chassert(file_offset_of_buffer_end > completed_range.right);
     cache_file_reader.reset();
 
-    file_segments->popFront();
+    file_segments->completeAndPopFront(settings.filesystem_cache_allow_background_download);
     if (file_segments->empty() && !nextFileSegmentsBatch())
         return false;
 
@@ -556,6 +556,12 @@ CachedOnDiskReadBufferFromFile::~CachedOnDiskReadBufferFromFile()
     {
         appendFilesystemCacheLog(file_segments->front(), read_type);
     }
+
+    if (file_segments && !file_segments->empty() && !file_segments->front().isCompleted())
+    {
+        file_segments->completeAndPopFront(settings.filesystem_cache_allow_background_download);
+        file_segments = {};
+    }
 }
 
 void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment)
@@ -784,6 +790,7 @@ bool CachedOnDiskReadBufferFromFile::writeCache(char * data, size_t size, size_t
             LOG_INFO(log, "Insert into cache is skipped due to insufficient disk space. ({})", e.displayText());
             return false;
         }
+        chassert(file_segment.state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION);
         throw;
     }
 

src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp

@@ -196,7 +196,7 @@ void FileSegmentRangeWriter::completeFileSegment()
     if (file_segment.isDetached() || file_segment.isCompleted())
         return;
 
-    file_segment.complete();
+    file_segment.complete(false);
     appendFilesystemCacheLog(file_segment);
 }
 
@@ -210,7 +210,7 @@ void FileSegmentRangeWriter::jumpToPosition(size_t position)
         if (position < current_write_offset)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot jump backwards: {} < {}", position, current_write_offset);
 
-        file_segment.complete();
+        file_segment.complete(false);
         file_segments.reset();
     }
     expected_write_offset = position;

src/Formats/fuzzers/CMakeLists.txt

@@ -1,2 +1,2 @@
 clickhouse_add_executable(format_fuzzer format_fuzzer.cpp ${SRCS})
-target_link_libraries(format_fuzzer PRIVATE clickhouse_aggregate_functions)
+target_link_libraries(format_fuzzer PRIVATE clickhouse_aggregate_functions dbms)

src/IO/ReadSettings.h

@@ -58,6 +58,9 @@ struct ReadSettings
     bool enable_filesystem_cache_log = false;
     size_t filesystem_cache_segments_batch_size = 20;
     size_t filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = 1000;
+    bool filesystem_cache_allow_background_download = true;
+    bool filesystem_cache_allow_background_download_for_metadata_files_in_packed_storage = true;
+    bool filesystem_cache_allow_background_download_during_fetch = true;
 
     bool use_page_cache_for_disks_without_file_cache = false;
     bool read_from_page_cache_if_exists_otherwise_bypass_cache = false;

src/Interpreters/Cache/EvictionCandidates.cpp

@@ -83,7 +83,8 @@ void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & loc
             queue_iterator->invalidate();
 
             chassert(candidate->releasable());
-            candidate->file_segment->resetQueueIterator();
+            candidate->file_segment->markDelayedRemovalAndResetQueueIterator();
+
             /// We need to set removed flag in file segment metadata,
             /// because in dynamic cache resize we first remove queue entries,
             /// then evict which also removes file segment metadata,

src/Interpreters/Cache/FileSegment.cpp

@@ -28,6 +28,7 @@ namespace ProfileEvents
     extern const Event FileSegmentFailToIncreasePriority;
     extern const Event FilesystemCacheHoldFileSegments;
     extern const Event FilesystemCacheUnusedHoldFileSegments;
+    extern const Event FilesystemCacheBackgroundDownloadQueuePush;
 }
 
 namespace CurrentMetrics
@@ -171,10 +172,11 @@ void FileSegment::setQueueIterator(Priority::IteratorPtr iterator)
     queue_iterator = iterator;
 }
 
-void FileSegment::resetQueueIterator()
+void FileSegment::markDelayedRemovalAndResetQueueIterator()
 {
     auto lk = lock();
-    queue_iterator.reset();
+    on_delayed_removal = true;
+    queue_iterator = {};
 }
 
 size_t FileSegment::getCurrentWriteOffset() const
@@ -627,7 +629,7 @@ void FileSegment::completePartAndResetDownloader()
     LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lk));
 }
 
-void FileSegment::complete()
+void FileSegment::complete(bool allow_background_download)
 {
     ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::FileSegmentCompleteMicroseconds);
 
@@ -700,12 +702,15 @@ void FileSegment::complete()
         case State::PARTIALLY_DOWNLOADED:
         {
             chassert(current_downloaded_size > 0);
+            chassert(fs::exists(getPath()));
+            chassert(fs::file_size(getPath()) > 0);
 
             if (is_last_holder)
             {
                 bool added_to_download_queue = false;
-                if (background_download_enabled && remote_file_reader)
+                if (allow_background_download && background_download_enabled && remote_file_reader)
                 {
+                    ProfileEvents::increment(ProfileEvents::FilesystemCacheBackgroundDownloadQueuePush);
                     added_to_download_queue = locked_key->addToDownloadQueue(offset(), segment_lock); /// Finish download in background.
                 }
 
@@ -841,29 +846,60 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock)
         }
     }
 
-    if (download_state == State::DOWNLOADED)
+    switch (download_state.load())
+    {
+        case State::EMPTY:
         {
             chassert(downloader_id.empty());
+            chassert(!fs::exists(getPath()));
+            chassert(!queue_iterator);
+            break;
+        }
+        case State::DOWNLOADED:
+        {
+            chassert(downloader_id.empty());
+
             chassert(downloaded_size == reserved_size);
             chassert(downloaded_size == range().size());
             chassert(downloaded_size > 0);
-        chassert(std::filesystem::file_size(getPath()) > 0);
+            chassert(fs::file_size(getPath()) > 0);
+
+            chassert(queue_iterator || on_delayed_removal);
             check_iterator(queue_iterator);
+            break;
         }
-    else
+        case State::DOWNLOADING:
-    {
-        if (download_state == State::DOWNLOADING)
         {
             chassert(!downloader_id.empty());
+            if (downloaded_size)
+            {
+                chassert(queue_iterator);
+                chassert(fs::file_size(getPath()) > 0);
             }
-        else if (download_state == State::PARTIALLY_DOWNLOADED
+            break;
-                 || download_state == State::EMPTY)
+        }
+        case State::PARTIALLY_DOWNLOADED:
         {
             chassert(downloader_id.empty());
-        }
 
             chassert(reserved_size >= downloaded_size);
+            chassert(downloaded_size > 0);
+            chassert(fs::file_size(getPath()) > 0);
+
+            chassert(queue_iterator);
             check_iterator(queue_iterator);
+            break;
+        }
+        case State::PARTIALLY_DOWNLOADED_NO_CONTINUATION:
+        {
+            chassert(reserved_size >= downloaded_size);
+            check_iterator(queue_iterator);
+            break;
+        }
+        case State::DETACHED:
+        {
+            break;
+        }
     }
 
     return true;
@@ -991,7 +1027,12 @@ FileSegmentsHolder::FileSegmentsHolder(FileSegments && file_segments_)
 FileSegmentPtr FileSegmentsHolder::getSingleFileSegment() const
 {
     if (file_segments.size() != 1)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected single file segment, got: {} in holder {}", file_segments.size(), toString());
+    {
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Expected single file segment, got: {} in holder {}",
+            file_segments.size(), toString());
+    }
     return file_segments.front();
 }
 
@@ -1001,7 +1042,23 @@ void FileSegmentsHolder::reset()
 
     ProfileEvents::increment(ProfileEvents::FilesystemCacheUnusedHoldFileSegments, file_segments.size());
     for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();)
-        file_segment_it = completeAndPopFrontImpl();
+    {
+        try
+        {
+            /// One might think it would have been more correct to do `false` here,
+            /// not to allow background download for file segments that we actually did not start reading.
+            /// But actually we would only do that, if those file segments were already read partially by some other thread/query
+            /// but they were not put to the download queue, because current thread was holding them in Holder.
+            /// So as a culprit, we need to allow to happen what would have happened if we did not exist.
+            file_segment_it = completeAndPopFrontImpl(true);
+        }
+        catch (...)
+        {
+            tryLogCurrentException(__PRETTY_FUNCTION__);
+            chassert(false);
+            continue;
+        }
+    }
     file_segments.clear();
 }
 
@@ -1010,9 +1067,9 @@ FileSegmentsHolder::~FileSegmentsHolder()
     reset();
 }
 
-FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl()
+FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl(bool allow_background_download)
 {
-    front().complete();
+    front().complete(allow_background_download);
     CurrentMetrics::sub(CurrentMetrics::FilesystemCacheHoldFileSegments);
     return file_segments.erase(file_segments.begin());
 }

src/Interpreters/Cache/FileSegment.h

@@ -177,7 +177,7 @@ public:
 
     void setQueueIterator(Priority::IteratorPtr iterator);
 
-    void resetQueueIterator();
+    void markDelayedRemovalAndResetQueueIterator();
 
     KeyMetadataPtr tryGetKeyMetadata() const;
 
@@ -189,7 +189,7 @@ public:
      * ========== Methods that must do cv.notify() ==================
      */
 
-    void complete();
+    void complete(bool allow_background_download);
 
     void completePartAndResetDownloader();
 
@@ -249,12 +249,13 @@ private:
 
     String tryGetPath() const;
 
-    Key file_key;
+    const Key file_key;
     Range segment_range;
     const FileSegmentKind segment_kind;
     /// Size of the segment is not known until it is downloaded and
     /// can be bigger than max_file_segment_size.
-    const bool is_unbound = false;
+    /// is_unbound == true for temporary data in cache.
+    const bool is_unbound;
     const bool background_download_enabled;
 
     std::atomic<State> download_state;
@@ -279,6 +280,8 @@ private:
     std::atomic<size_t> hits_count = 0; /// cache hits.
     std::atomic<size_t> ref_count = 0; /// Used for getting snapshot state
 
+    bool on_delayed_removal = false;
+
     CurrentMetrics::Increment metric_increment{CurrentMetrics::CacheFileSegments};
 };
 
@@ -297,7 +300,7 @@ struct FileSegmentsHolder final : private boost::noncopyable
 
     String toString(bool with_state = false) const;
 
-    void popFront() { completeAndPopFrontImpl(); }
+    void completeAndPopFront(bool allow_background_download) { completeAndPopFrontImpl(allow_background_download); }
 
     FileSegment & front() { return *file_segments.front(); }
     const FileSegment & front() const { return *file_segments.front(); }
@@ -319,7 +322,7 @@ struct FileSegmentsHolder final : private boost::noncopyable
 private:
     FileSegments file_segments{};
 
-    FileSegments::iterator completeAndPopFrontImpl();
+    FileSegments::iterator completeAndPopFrontImpl(bool allow_background_download);
 };
 
 using FileSegmentsHolderPtr = std::unique_ptr<FileSegmentsHolder>;

src/Interpreters/Cache/Metadata.cpp

@@ -940,7 +940,16 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl(
     if (file_segment->queue_iterator && invalidate_queue_entry)
         file_segment->queue_iterator->invalidate();
 
+    try
+    {
         file_segment->detach(segment_lock, *this);
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+        chassert(false);
+        /// Do not rethrow, we must delete the file below.
+    }
 
     try
     {
@@ -990,8 +999,8 @@ void LockedKey::shrinkFileSegmentToDownloadedSize(
      * because of no space left in cache, we need to be able to cut file segment's size to downloaded_size.
      */
 
-    auto metadata = getByOffset(offset);
+    auto file_segment_metadata = getByOffset(offset);
-    const auto & file_segment = metadata->file_segment;
+    const auto & file_segment = file_segment_metadata->file_segment;
     chassert(file_segment->assertCorrectnessUnlocked(segment_lock));
 
     const size_t downloaded_size = file_segment->getDownloadedSize();
@@ -1006,15 +1015,15 @@ void LockedKey::shrinkFileSegmentToDownloadedSize(
     chassert(file_segment->reserved_size >= downloaded_size);
     int64_t diff = file_segment->reserved_size - downloaded_size;
 
-    metadata->file_segment = std::make_shared<FileSegment>(
+    file_segment_metadata->file_segment = std::make_shared<FileSegment>(
         getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED,
         CreateFileSegmentSettings(file_segment->getKind()), false,
         file_segment->cache, key_metadata, file_segment->queue_iterator);
 
     if (diff)
-        metadata->getQueueIterator()->decrementSize(diff);
+        file_segment_metadata->getQueueIterator()->decrementSize(diff);
 
-    chassert(file_segment->assertCorrectnessUnlocked(segment_lock));
+    chassert(file_segment_metadata->file_segment->assertCorrectnessUnlocked(segment_lock));
 }
 
 bool LockedKey::addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &)

src/Interpreters/ConcurrentHashJoin.h

@@ -60,17 +60,6 @@ public:
     IBlocksStreamPtr
     getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override;
 
-
-    bool isCloneSupported() const override
-    {
-        return !getTotals() && getTotalRowCount() == 0;
-    }
-
-    std::shared_ptr<IJoin> clone(const std::shared_ptr<TableJoin> & table_join_, const Block &, const Block & right_sample_block_) const override
-    {
-        return std::make_shared<ConcurrentHashJoin>(context, table_join_, slots, right_sample_block_, stats_collecting_params);
-    }
-
 private:
     struct InternalHashJoin
     {

src/Interpreters/Context.cpp

@@ -194,6 +194,8 @@ namespace Setting
     extern const SettingsUInt64 filesystem_cache_max_download_size;
     extern const SettingsUInt64 filesystem_cache_reserve_space_wait_lock_timeout_milliseconds;
     extern const SettingsUInt64 filesystem_cache_segments_batch_size;
+    extern const SettingsBool filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage;
+    extern const SettingsBool filesystem_cache_enable_background_download_during_fetch;
     extern const SettingsBool http_make_head_request;
     extern const SettingsUInt64 http_max_fields;
     extern const SettingsUInt64 http_max_field_name_size;
@@ -5746,6 +5748,9 @@ ReadSettings Context::getReadSettings() const
     res.filesystem_cache_segments_batch_size = settings_ref[Setting::filesystem_cache_segments_batch_size];
     res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds
         = settings_ref[Setting::filesystem_cache_reserve_space_wait_lock_timeout_milliseconds];
+    res.filesystem_cache_allow_background_download_for_metadata_files_in_packed_storage
+        = settings_ref[Setting::filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage];
+    res.filesystem_cache_allow_background_download_during_fetch = settings_ref[Setting::filesystem_cache_enable_background_download_during_fetch];
 
     res.filesystem_cache_max_download_size = settings_ref[Setting::filesystem_cache_max_download_size];
     res.skip_download_if_exceeds_query_cache = settings_ref[Setting::skip_download_if_exceeds_query_cache];

src/Interpreters/ExpressionAnalyzer.cpp

@@ -1981,7 +1981,9 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                 Block before_prewhere_sample = source_header;
                 if (sanitizeBlock(before_prewhere_sample))
                 {
-                    before_prewhere_sample = prewhere_dag_and_flags->dag.updateHeader(before_prewhere_sample);
+                    ExpressionActions(
+                        prewhere_dag_and_flags->dag.clone(),
+                        ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_prewhere_sample);
                     auto & column_elem = before_prewhere_sample.getByName(query.prewhere()->getColumnName());
                     /// If the filter column is a constant, record it.
                     if (column_elem.column)
@@ -2013,7 +2015,9 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                     before_where_sample = source_header;
                 if (sanitizeBlock(before_where_sample))
                 {
-                    before_where_sample = before_where->dag.updateHeader(before_where_sample);
+                    ExpressionActions(
+                        before_where->dag.clone(),
+                        ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_where_sample);
 
                     auto & column_elem
                         = before_where_sample.getByName(query.where()->getColumnName());

src/Interpreters/FullSortingMergeJoin.h

@@ -36,7 +36,7 @@ public:
 
     bool isCloneSupported() const override
     {
-        return !getTotals();
+        return true;
     }
 
     std::shared_ptr<IJoin> clone(const std::shared_ptr<TableJoin> & table_join_,

src/Interpreters/HashJoin/HashJoin.cpp

@@ -383,16 +383,6 @@ size_t HashJoin::getTotalByteCount() const
     return res;
 }
 
-bool HashJoin::isUsedByAnotherAlgorithm() const
-{
-    return table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO) || table_join->isEnabledAlgorithm(JoinAlgorithm::GRACE_HASH);
-}
-
-bool HashJoin::canRemoveColumnsFromLeftBlock() const
-{
-    return table_join->enableEnalyzer() && !table_join->hasUsing() && !isUsedByAnotherAlgorithm();
-}
-
 void HashJoin::initRightBlockStructure(Block & saved_block_sample)
 {
     if (isCrossOrComma(kind))
@@ -404,7 +394,8 @@ void HashJoin::initRightBlockStructure(Block & saved_block_sample)
 
     bool multiple_disjuncts = !table_join->oneDisjunct();
     /// We could remove key columns for LEFT | INNER HashJoin but we should keep them for JoinSwitcher (if any).
-    bool save_key_columns = isUsedByAnotherAlgorithm() ||
+    bool save_key_columns = table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO) ||
+                            table_join->isEnabledAlgorithm(JoinAlgorithm::GRACE_HASH) ||
                             isRightOrFull(kind) ||
                             multiple_disjuncts ||
                             table_join->getMixedJoinExpression();
@@ -1237,10 +1228,7 @@ IBlocksStreamPtr HashJoin::getNonJoinedBlocks(const Block & left_sample_block,
 {
     if (!JoinCommon::hasNonJoinedBlocks(*table_join))
         return {};
-
     size_t left_columns_count = left_sample_block.columns();
-    if (canRemoveColumnsFromLeftBlock())
-        left_columns_count = table_join->getOutputColumns(JoinTableSide::Left).size();
 
     bool flag_per_row = needUsedFlagsForPerRightTableRow(table_join);
     if (!flag_per_row)

src/Interpreters/HashJoin/HashJoin.h

@@ -127,7 +127,7 @@ public:
 
     bool isCloneSupported() const override
     {
-        return !getTotals() && getTotalRowCount() == 0;
+        return true;
     }
 
     std::shared_ptr<IJoin> clone(const std::shared_ptr<TableJoin> & table_join_,
@@ -464,9 +464,6 @@ private:
 
     bool empty() const;
 
-    bool isUsedByAnotherAlgorithm() const;
-    bool canRemoveColumnsFromLeftBlock() const;
-
     void validateAdditionalFilterExpression(std::shared_ptr<ExpressionActions> additional_filter_expression);
     bool needUsedFlagsForPerRightTableRow(std::shared_ptr<TableJoin> table_join_) const;
 

src/Interpreters/HashJoin/HashJoinMethodsImpl.h

@@ -56,6 +56,7 @@ Block HashJoinMethods<KIND, STRICTNESS, MapsTemplate>::joinBlockImpl(
         const auto & key_names = !is_join_get ? onexprs[i].key_names_left : onexprs[i].key_names_right;
         join_on_keys.emplace_back(block, key_names, onexprs[i].condColumnNames().first, join.key_sizes[i]);
     }
+    size_t existing_columns = block.columns();
 
     /** If you use FULL or RIGHT JOIN, then the columns from the "left" table must be materialized.
       * Because if they are constants, then in the "not joined" rows, they may have different values
@@ -98,22 +99,6 @@ Block HashJoinMethods<KIND, STRICTNESS, MapsTemplate>::joinBlockImpl(
         added_columns.buildJoinGetOutput();
     else
         added_columns.buildOutput();
-
-    const auto & table_join = join.table_join;
-    std::set<size_t> block_columns_to_erase;
-    if (join.canRemoveColumnsFromLeftBlock())
-    {
-        std::unordered_set<String> left_output_columns;
-        for (const auto & out_column : table_join->getOutputColumns(JoinTableSide::Left))
-            left_output_columns.insert(out_column.name);
-        for (size_t i = 0; i < block.columns(); ++i)
-        {
-            if (!left_output_columns.contains(block.getByPosition(i).name))
-                block_columns_to_erase.insert(i);
-        }
-    }
-    size_t existing_columns = block.columns();
-
     for (size_t i = 0; i < added_columns.size(); ++i)
         block.insert(added_columns.moveColumn(i));
 
@@ -175,7 +160,6 @@ Block HashJoinMethods<KIND, STRICTNESS, MapsTemplate>::joinBlockImpl(
             block.safeGetByPosition(pos).column = block.safeGetByPosition(pos).column->replicate(*offsets_to_replicate);
         }
     }
-    block.erase(block_columns_to_erase);
     return remaining_block;
 }
 

src/Interpreters/InterpreterSelectQuery.cpp

@@ -1888,9 +1888,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
                         expressions.join,
                         settings[Setting::max_block_size],
                         max_streams,
-                        /* required_output_ = */ NameSet{},
+                        analysis_result.optimize_read_in_order);
-                        analysis_result.optimize_read_in_order,
-                        /* use_new_analyzer_ = */ false);
 
                     join_step->setStepDescription(fmt::format("JOIN {}", expressions.join->pipelineType()));
                     std::vector<QueryPlanPtr> plans;

src/Interpreters/TableJoin.cpp

@@ -41,7 +41,6 @@ namespace DB
 namespace Setting
 {
     extern const SettingsBool allow_experimental_join_right_table_sorting;
-    extern const SettingsBool allow_experimental_analyzer;
     extern const SettingsUInt64 cross_join_min_bytes_to_compress;
     extern const SettingsUInt64 cross_join_min_rows_to_compress;
     extern const SettingsUInt64 default_max_bytes_in_join;
@@ -144,7 +143,6 @@ TableJoin::TableJoin(const Settings & settings, VolumePtr tmp_volume_, Temporary
     , max_memory_usage(settings[Setting::max_memory_usage])
     , tmp_volume(tmp_volume_)
     , tmp_data(tmp_data_)
-    , enable_analyzer(settings[Setting::allow_experimental_analyzer])
 {
 }
 
@@ -163,8 +161,6 @@ void TableJoin::resetCollected()
     clauses.clear();
     columns_from_joined_table.clear();
     columns_added_by_join.clear();
-    columns_from_left_table.clear();
-    result_columns_from_left_table.clear();
     original_names.clear();
     renames.clear();
     left_type_map.clear();
@@ -207,20 +203,6 @@ size_t TableJoin::rightKeyInclusion(const String & name) const
     return count;
 }
 
-void TableJoin::setInputColumns(NamesAndTypesList left_output_columns, NamesAndTypesList right_output_columns)
-{
-    columns_from_left_table = std::move(left_output_columns);
-    columns_from_joined_table = std::move(right_output_columns);
-}
-
-
-const NamesAndTypesList & TableJoin::getOutputColumns(JoinTableSide side)
-{
-    if (side == JoinTableSide::Left)
-        return result_columns_from_left_table;
-    return columns_added_by_join;
-}
-
 void TableJoin::deduplicateAndQualifyColumnNames(const NameSet & left_table_columns, const String & right_table_prefix)
 {
     NameSet joined_columns;
@@ -369,18 +351,9 @@ bool TableJoin::rightBecomeNullable(const DataTypePtr & column_type) const
     return forceNullableRight() && JoinCommon::canBecomeNullable(column_type);
 }
 
-void TableJoin::setUsedColumn(const NameAndTypePair & joined_column, JoinTableSide side)
-{
-    if (side == JoinTableSide::Left)
-        result_columns_from_left_table.push_back(joined_column);
-    else
-        columns_added_by_join.push_back(joined_column);
-
-}
-
 void TableJoin::addJoinedColumn(const NameAndTypePair & joined_column)
 {
-    setUsedColumn(joined_column, JoinTableSide::Right);
+    columns_added_by_join.emplace_back(joined_column);
 }
 
 NamesAndTypesList TableJoin::correctedColumnsAddedByJoin() const
@@ -1022,32 +995,5 @@ size_t TableJoin::getMaxMemoryUsage() const
     return max_memory_usage;
 }
 
-void TableJoin::swapSides()
-{
-    assertEnableEnalyzer();
-
-    std::swap(key_asts_left, key_asts_right);
-    std::swap(left_type_map, right_type_map);
-    for (auto & clause : clauses)
-    {
-        std::swap(clause.key_names_left, clause.key_names_right);
-        std::swap(clause.on_filter_condition_left, clause.on_filter_condition_right);
-        std::swap(clause.analyzer_left_filter_condition_column_name, clause.analyzer_right_filter_condition_column_name);
-    }
-
-    std::swap(columns_from_left_table, columns_from_joined_table);
-    std::swap(result_columns_from_left_table, columns_added_by_join);
-
-    if (table_join.kind == JoinKind::Left)
-        table_join.kind = JoinKind::Right;
-    else if (table_join.kind == JoinKind::Right)
-        table_join.kind = JoinKind::Left;
-}
-
-void TableJoin::assertEnableEnalyzer() const
-{
-    if (!enable_analyzer)
-        throw DB::Exception(ErrorCodes::NOT_IMPLEMENTED, "TableJoin: analyzer is disabled");
-}
 
 }

src/Interpreters/TableJoin.h

@@ -167,9 +167,6 @@ private:
 
     ASOFJoinInequality asof_inequality = ASOFJoinInequality::GreaterOrEquals;
 
-    NamesAndTypesList columns_from_left_table;
-    NamesAndTypesList result_columns_from_left_table;
-
     /// All columns which can be read from joined table. Duplicating names are qualified.
     NamesAndTypesList columns_from_joined_table;
     /// Columns will be added to block by JOIN.
@@ -205,8 +202,6 @@ private:
 
     bool is_join_with_constant = false;
 
-    bool enable_analyzer = false;
-
     Names requiredJoinedNames() const;
 
     /// Create converting actions and change key column names if required
@@ -271,8 +266,6 @@ public:
     VolumePtr getGlobalTemporaryVolume() { return tmp_volume; }
 
     TemporaryDataOnDiskScopePtr getTempDataOnDisk() { return tmp_data; }
-    bool enableEnalyzer() const { return enable_analyzer; }
-    void assertEnableEnalyzer() const;
 
     ActionsDAG createJoinedBlockActions(ContextPtr context) const;
 
@@ -289,7 +282,6 @@ public:
     }
 
     bool allowParallelHashJoin() const;
-    void swapSides();
 
     bool joinUseNulls() const { return join_use_nulls; }
 
@@ -380,9 +372,6 @@ public:
     bool leftBecomeNullable(const DataTypePtr & column_type) const;
     bool rightBecomeNullable(const DataTypePtr & column_type) const;
     void addJoinedColumn(const NameAndTypePair & joined_column);
-
-    void setUsedColumn(const NameAndTypePair & joined_column, JoinTableSide side);
-
     void setColumnsAddedByJoin(const NamesAndTypesList & columns_added_by_join_value)
     {
         columns_added_by_join = columns_added_by_join_value;
@@ -408,17 +397,11 @@ public:
     ASTPtr leftKeysList() const;
     ASTPtr rightKeysList() const; /// For ON syntax only
 
-    void setColumnsFromJoinedTable(NamesAndTypesList columns_from_joined_table_value, const NameSet & left_table_columns, const String & right_table_prefix, const NamesAndTypesList & columns_from_left_table_)
+    void setColumnsFromJoinedTable(NamesAndTypesList columns_from_joined_table_value, const NameSet & left_table_columns, const String & right_table_prefix)
     {
         columns_from_joined_table = std::move(columns_from_joined_table_value);
         deduplicateAndQualifyColumnNames(left_table_columns, right_table_prefix);
-        result_columns_from_left_table = columns_from_left_table_;
-        columns_from_left_table = columns_from_left_table_;
     }
-
-    void setInputColumns(NamesAndTypesList left_output_columns, NamesAndTypesList right_output_columns);
-    const NamesAndTypesList & getOutputColumns(JoinTableSide side);
-
     const NamesAndTypesList & columnsFromJoinedTable() const { return columns_from_joined_table; }
     const NamesAndTypesList & columnsAddedByJoin() const { return columns_added_by_join; }
 

src/Interpreters/TreeRewriter.cpp

@@ -1353,15 +1353,12 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect(
 
     if (tables_with_columns.size() > 1)
     {
-        auto columns_from_left_table = tables_with_columns[0].columns;
         const auto & right_table = tables_with_columns[1];
         auto columns_from_joined_table = right_table.columns;
         /// query can use materialized or aliased columns from right joined table,
         /// we want to request it for right table
         columns_from_joined_table.insert(columns_from_joined_table.end(), right_table.hidden_columns.begin(), right_table.hidden_columns.end());
-        columns_from_left_table.insert(columns_from_left_table.end(), tables_with_columns[0].hidden_columns.begin(), tables_with_columns[0].hidden_columns.end());
+        result.analyzed_join->setColumnsFromJoinedTable(std::move(columns_from_joined_table), source_columns_set, right_table.table.getQualifiedNamePrefix());
-        result.analyzed_join->setColumnsFromJoinedTable(
-            std::move(columns_from_joined_table), source_columns_set, right_table.table.getQualifiedNamePrefix(), columns_from_left_table);
     }
 
     translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns);

src/Interpreters/fuzzers/CMakeLists.txt

@@ -3,5 +3,6 @@ target_link_libraries(execute_query_fuzzer PRIVATE
     dbms
     clickhouse_table_functions
     clickhouse_aggregate_functions
+    clickhouse_functions
     clickhouse_dictionaries
     clickhouse_dictionaries_embedded)

src/Interpreters/tests/gtest_filecache.cpp

@@ -253,7 +253,7 @@ void download(FileSegment & file_segment)
     download(cache_base_path, file_segment);
     ASSERT_EQ(file_segment.state(), State::DOWNLOADING);
 
-    file_segment.complete();
+    file_segment.complete(false);
     ASSERT_EQ(file_segment.state(), State::DOWNLOADED);
 }
 
@@ -263,7 +263,7 @@ void assertDownloadFails(FileSegment & file_segment)
     ASSERT_EQ(file_segment.getDownloadedSize(), 0);
     std::string failure_reason;
     ASSERT_FALSE(file_segment.reserve(file_segment.range().size(), 1000, failure_reason));
-    file_segment.complete();
+    file_segment.complete(false);
 }
 
 void download(const HolderPtr & holder)
@@ -971,7 +971,7 @@ TEST_F(FileCacheTest, temporaryData)
             ASSERT_TRUE(segment->getOrSetDownloader() == DB::FileSegment::getCallerId());
             ASSERT_TRUE(segment->reserve(segment->range().size(), 1000, failure_reason));
             download(*segment);
-            segment->complete();
+            segment->complete(false);
         }
     }
 

src/Parsers/ASTFunction.cpp

@@ -724,7 +724,10 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
                 {
                     if (secret_arguments.are_named)
                     {
-                        assert_cast<const ASTFunction *>(argument.get())->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens);
+                        if (const auto * func_ast = typeid_cast<const ASTFunction *>(argument.get()))
+                            func_ast->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens);
+                        else
+                            argument->formatImpl(settings, state, nested_dont_need_parens);
                         settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : "");
                     }
                     if (!secret_arguments.replacement.empty())

src/Parsers/CreateQueryUUIDs.cpp

@@ -31,7 +31,7 @@ CreateQueryUUIDs::CreateQueryUUIDs(const ASTCreateQuery & query, bool generate_r
         /// If we generate random UUIDs for already existing tables then those UUIDs will not be correct making those inner target table inaccessible.
         /// Thus it's not safe for example to replace
         /// "ATTACH MATERIALIZED VIEW mv AS SELECT a FROM b" with
-        /// "ATTACH MATERIALIZED VIEW mv TO INNER UUID '123e4567-e89b-12d3-a456-426614174000' AS SELECT a FROM b"
+        /// "ATTACH MATERIALIZED VIEW mv TO INNER UUID "XXXX" AS SELECT a FROM b"
         /// This replacement is safe only for CREATE queries when inner target tables don't exist yet.
         if (!query.attach)
         {

src/Parsers/fuzzers/CMakeLists.txt

@@ -2,10 +2,10 @@ clickhouse_add_executable(lexer_fuzzer lexer_fuzzer.cpp ${SRCS})
 target_link_libraries(lexer_fuzzer PRIVATE clickhouse_parsers)
 
 clickhouse_add_executable(select_parser_fuzzer select_parser_fuzzer.cpp ${SRCS})
-target_link_libraries(select_parser_fuzzer PRIVATE clickhouse_parsers dbms)
+target_link_libraries(select_parser_fuzzer PRIVATE clickhouse_parsers clickhouse_functions dbms)
 
 clickhouse_add_executable(create_parser_fuzzer create_parser_fuzzer.cpp ${SRCS})
-target_link_libraries(create_parser_fuzzer PRIVATE clickhouse_parsers dbms)
+target_link_libraries(create_parser_fuzzer PRIVATE clickhouse_parsers clickhouse_functions dbms)
 
 add_subdirectory(codegen_fuzzer)
 

src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt

@@ -47,4 +47,4 @@ target_compile_options (codegen_select_fuzzer PRIVATE -Wno-newline-eof)
 target_link_libraries(protoc ch_contrib::fuzzer)
 
 target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
-target_link_libraries(codegen_select_fuzzer PRIVATE ch_contrib::protobuf_mutator ch_contrib::protoc dbms)
+target_link_libraries(codegen_select_fuzzer PRIVATE ch_contrib::protobuf_mutator ch_contrib::protoc clickhouse_functions dbms)

src/Planner/CollectColumnIdentifiers.cpp

@@ -2,7 +2,6 @@
 
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/ColumnNode.h>
-#include <Analyzer/JoinNode.h>
 
 #include <Planner/PlannerContext.h>
 

src/Planner/PlannerJoinTree.cpp

@@ -104,7 +104,6 @@ namespace Setting
     extern const SettingsBool optimize_move_to_prewhere;
     extern const SettingsBool optimize_move_to_prewhere_if_final;
     extern const SettingsBool use_concurrency_control;
-    extern const SettingsJoinInnerTableSelectionMode query_plan_join_inner_table_selection;
 }
 
 namespace ErrorCodes
@@ -1242,55 +1241,6 @@ void joinCastPlanColumnsToNullable(QueryPlan & plan_to_add_cast, PlannerContextP
     plan_to_add_cast.addStep(std::move(cast_join_columns_step));
 }
 
-std::optional<ActionsDAG> createStepToDropColumns(
-    const Block & header,
-    const ColumnIdentifierSet & outer_scope_columns,
-    const PlannerContextPtr & planner_context)
-{
-    ActionsDAG drop_unused_columns_after_join_actions_dag(header.getColumnsWithTypeAndName());
-    ActionsDAG::NodeRawConstPtrs drop_unused_columns_after_join_actions_dag_updated_outputs;
-    std::unordered_set<std::string_view> drop_unused_columns_after_join_actions_dag_updated_outputs_names;
-    std::optional<size_t> first_skipped_column_node_index;
-
-    auto & drop_unused_columns_after_join_actions_dag_outputs = drop_unused_columns_after_join_actions_dag.getOutputs();
-    size_t drop_unused_columns_after_join_actions_dag_outputs_size = drop_unused_columns_after_join_actions_dag_outputs.size();
-
-    const auto & global_planner_context = planner_context->getGlobalPlannerContext();
-
-    for (size_t i = 0; i < drop_unused_columns_after_join_actions_dag_outputs_size; ++i)
-    {
-        const auto & output = drop_unused_columns_after_join_actions_dag_outputs[i];
-
-        if (drop_unused_columns_after_join_actions_dag_updated_outputs_names.contains(output->result_name)
-            || !global_planner_context->hasColumnIdentifier(output->result_name))
-            continue;
-
-        if (!outer_scope_columns.contains(output->result_name))
-        {
-            if (!first_skipped_column_node_index)
-                first_skipped_column_node_index = i;
-            continue;
-        }
-
-        drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(output);
-        drop_unused_columns_after_join_actions_dag_updated_outputs_names.insert(output->result_name);
-    }
-
-    if (!first_skipped_column_node_index)
-        return {};
-
-    /** It is expected that JOIN TREE query plan will contain at least 1 column, even if there are no columns in outer scope.
-      *
-      * Example: SELECT count() FROM test_table_1 AS t1, test_table_2 AS t2;
-      */
-    if (drop_unused_columns_after_join_actions_dag_updated_outputs.empty() && first_skipped_column_node_index)
-        drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(drop_unused_columns_after_join_actions_dag_outputs[*first_skipped_column_node_index]);
-
-    drop_unused_columns_after_join_actions_dag_outputs = std::move(drop_unused_columns_after_join_actions_dag_updated_outputs);
-
-    return drop_unused_columns_after_join_actions_dag;
-}
-
 JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_expression,
     JoinTreeQueryPlan left_join_tree_query_plan,
     JoinTreeQueryPlan right_join_tree_query_plan,
@@ -1563,37 +1513,21 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
     }
 
     const Block & left_header = left_plan.getCurrentHeader();
+    auto left_table_names = left_header.getNames();
+    NameSet left_table_names_set(left_table_names.begin(), left_table_names.end());
+
+    auto columns_from_joined_table = right_plan.getCurrentHeader().getNamesAndTypesList();
+    table_join->setColumnsFromJoinedTable(columns_from_joined_table, left_table_names_set, "");
+
+    for (auto & column_from_joined_table : columns_from_joined_table)
+    {
+        /// Add columns from joined table only if they are presented in outer scope, otherwise they can be dropped
+        if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name) &&
+            outer_scope_columns.contains(column_from_joined_table.name))
+            table_join->addJoinedColumn(column_from_joined_table);
+    }
+
     const Block & right_header = right_plan.getCurrentHeader();
-
-    auto columns_from_left_table = left_header.getNamesAndTypesList();
-    auto columns_from_right_table = right_header.getNamesAndTypesList();
-
-    table_join->setInputColumns(columns_from_left_table, columns_from_right_table);
-
-    for (auto & column_from_joined_table : columns_from_left_table)
-    {
-        /// Add columns to output only if they are presented in outer scope, otherwise they can be dropped
-        if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name) &&
-            outer_scope_columns.contains(column_from_joined_table.name))
-            table_join->setUsedColumn(column_from_joined_table, JoinTableSide::Left);
-    }
-
-    for (auto & column_from_joined_table : columns_from_right_table)
-    {
-        /// Add columns to output only if they are presented in outer scope, otherwise they can be dropped
-        if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name) &&
-            outer_scope_columns.contains(column_from_joined_table.name))
-            table_join->setUsedColumn(column_from_joined_table, JoinTableSide::Right);
-    }
-
-    if (table_join->getOutputColumns(JoinTableSide::Left).empty() && table_join->getOutputColumns(JoinTableSide::Right).empty())
-    {
-        if (!columns_from_left_table.empty())
-            table_join->setUsedColumn(columns_from_left_table.front(), JoinTableSide::Left);
-        else if (!columns_from_right_table.empty())
-            table_join->setUsedColumn(columns_from_right_table.front(), JoinTableSide::Right);
-    }
-
     auto join_algorithm = chooseJoinAlgorithm(table_join, join_node.getRightTableExpression(), left_header, right_header, planner_context);
 
     auto result_plan = QueryPlan();
@@ -1621,10 +1555,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
             SortingStep::Settings sort_settings(*query_context);
 
             auto sorting_step = std::make_unique<SortingStep>(
-                plan.getCurrentHeader(),
+                plan.getCurrentHeader(), std::move(sort_description), 0 /*limit*/, sort_settings, true /*is_sorting_for_merge_join*/);
-                std::move(sort_description),
-                0 /*limit*/,
-                sort_settings);
             sorting_step->setStepDescription(fmt::format("Sort {} before JOIN", join_table_side));
             plan.addStep(std::move(sorting_step));
         };
@@ -1684,26 +1615,13 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         }
 
         auto join_pipeline_type = join_algorithm->pipelineType();
-
-        ColumnIdentifierSet outer_scope_columns_nonempty;
-        if (outer_scope_columns.empty())
-        {
-            if (left_header.columns() > 1)
-                outer_scope_columns_nonempty.insert(left_header.getByPosition(0).name);
-            else if (right_header.columns() > 1)
-                outer_scope_columns_nonempty.insert(right_header.getByPosition(0).name);
-        }
-
         auto join_step = std::make_unique<JoinStep>(
             left_plan.getCurrentHeader(),
             right_plan.getCurrentHeader(),
             std::move(join_algorithm),
             settings[Setting::max_block_size],
             settings[Setting::max_threads],
-            outer_scope_columns.empty() ? outer_scope_columns_nonempty : outer_scope_columns,
+            false /*optimize_read_in_order*/);
-            false /*optimize_read_in_order*/,
-            true /*optimize_skip_unused_shards*/);
-        join_step->inner_table_selection_mode = settings[Setting::query_plan_join_inner_table_selection];
 
         join_step->setStepDescription(fmt::format("JOIN {}", join_pipeline_type));
 
@@ -1714,17 +1632,46 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         result_plan.unitePlans(std::move(join_step), {std::move(plans)});
     }
 
-    const auto & header_after_join = result_plan.getCurrentHeader();
+    ActionsDAG drop_unused_columns_after_join_actions_dag(result_plan.getCurrentHeader().getColumnsWithTypeAndName());
-    if (header_after_join.columns() > outer_scope_columns.size())
+    ActionsDAG::NodeRawConstPtrs drop_unused_columns_after_join_actions_dag_updated_outputs;
+    std::unordered_set<std::string_view> drop_unused_columns_after_join_actions_dag_updated_outputs_names;
+    std::optional<size_t> first_skipped_column_node_index;
+
+    auto & drop_unused_columns_after_join_actions_dag_outputs = drop_unused_columns_after_join_actions_dag.getOutputs();
+    size_t drop_unused_columns_after_join_actions_dag_outputs_size = drop_unused_columns_after_join_actions_dag_outputs.size();
+
+    for (size_t i = 0; i < drop_unused_columns_after_join_actions_dag_outputs_size; ++i)
     {
-        auto drop_unused_columns_after_join_actions_dag = createStepToDropColumns(header_after_join, outer_scope_columns, planner_context);
+        const auto & output = drop_unused_columns_after_join_actions_dag_outputs[i];
-        if (drop_unused_columns_after_join_actions_dag)
+
+        const auto & global_planner_context = planner_context->getGlobalPlannerContext();
+        if (drop_unused_columns_after_join_actions_dag_updated_outputs_names.contains(output->result_name)
+            || !global_planner_context->hasColumnIdentifier(output->result_name))
+            continue;
+
+        if (!outer_scope_columns.contains(output->result_name))
         {
-            auto drop_unused_columns_after_join_transform_step = std::make_unique<ExpressionStep>(result_plan.getCurrentHeader(), std::move(*drop_unused_columns_after_join_actions_dag));
+            if (!first_skipped_column_node_index)
-            drop_unused_columns_after_join_transform_step->setStepDescription("Drop unused columns after JOIN");
+                first_skipped_column_node_index = i;
+            continue;
+        }
+
+        drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(output);
+        drop_unused_columns_after_join_actions_dag_updated_outputs_names.insert(output->result_name);
+    }
+
+    /** It is expected that JOIN TREE query plan will contain at least 1 column, even if there are no columns in outer scope.
+      *
+      * Example: SELECT count() FROM test_table_1 AS t1, test_table_2 AS t2;
+      */
+    if (drop_unused_columns_after_join_actions_dag_updated_outputs.empty() && first_skipped_column_node_index)
+        drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(drop_unused_columns_after_join_actions_dag_outputs[*first_skipped_column_node_index]);
+
+    drop_unused_columns_after_join_actions_dag_outputs = std::move(drop_unused_columns_after_join_actions_dag_updated_outputs);
+
+    auto drop_unused_columns_after_join_transform_step = std::make_unique<ExpressionStep>(result_plan.getCurrentHeader(), std::move(drop_unused_columns_after_join_actions_dag));
+    drop_unused_columns_after_join_transform_step->setStepDescription("DROP unused columns after JOIN");
     result_plan.addStep(std::move(drop_unused_columns_after_join_transform_step));
-        }
-    }
 
     for (const auto & right_join_tree_query_plan_row_policy : right_join_tree_query_plan.used_row_policies)
         left_join_tree_query_plan.used_row_policies.insert(right_join_tree_query_plan_row_policy);

src/Planner/findParallelReplicasQuery.cpp

@@ -17,6 +17,7 @@
 #include <Processors/QueryPlan/ExpressionStep.h>
 #include <Processors/QueryPlan/FilterStep.h>
 #include <Processors/QueryPlan/JoinStep.h>
+#include <Processors/QueryPlan/SortingStep.h>
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/StorageDummy.h>
 #include <Storages/StorageMaterializedView.h>
@@ -170,12 +171,25 @@ const QueryNode * findQueryForParallelReplicas(
     const std::unordered_map<const QueryNode *, const QueryPlan::Node *> & mapping,
     const Settings & settings)
 {
-    const QueryPlan::Node * prev_checked_node = nullptr;
+    struct Frame
+    {
+        const QueryPlan::Node * node = nullptr;
+        /// Below we will check subqueries from `stack` to find outermost subquery that could be executed remotely.
+        /// Currently traversal algorithm considers only steps with 0 or 1 children and JOIN specifically.
+        /// When we found some step that requires finalization on the initiator (e.g. GROUP BY) there are two options:
+        /// 1. If plan looks like a single path (e.g. AggregatingStep -> ExpressionStep -> Reading) we can execute
+        /// current subquery as a whole with replicas.
+        /// 2. If we were inside JOIN we cannot offload the whole subquery to replicas because at least one side
+        /// of the JOIN needs to be finalized on the initiator.
+        /// So this flag is used to track what subquery to return once we hit a step that needs finalization.
+        bool inside_join = false;
+    };
+
     const QueryNode * res = nullptr;
 
     while (!stack.empty())
     {
-        const QueryNode * subquery_node = stack.top();
+        const QueryNode * const subquery_node = stack.top();
         stack.pop();
 
         auto it = mapping.find(subquery_node);
@@ -183,22 +197,21 @@ const QueryNode * findQueryForParallelReplicas(
         if (it == mapping.end())
             break;
 
-        const QueryPlan::Node * curr_node = it->second;
+        std::stack<Frame> nodes_to_check;
-        const QueryPlan::Node * next_node_to_check = curr_node;
+        nodes_to_check.push({.node = it->second, .inside_join = false});
         bool can_distribute_full_node = true;
+        bool currently_inside_join = false;
 
-        while (next_node_to_check && next_node_to_check != prev_checked_node)
+        while (!nodes_to_check.empty())
         {
+            const auto & [next_node_to_check, inside_join] = nodes_to_check.top();
+            nodes_to_check.pop();
             const auto & children = next_node_to_check->children;
             auto * step = next_node_to_check->step.get();
 
             if (children.empty())
             {
-                /// Found a source step. This should be possible only in the first iteration.
+                /// Found a source step.
-                if (prev_checked_node)
-                    return nullptr;
-
-                next_node_to_check = nullptr;
             }
             else if (children.size() == 1)
             {
@@ -206,12 +219,19 @@ const QueryNode * findQueryForParallelReplicas(
                 const auto * filter = typeid_cast<FilterStep *>(step);
 
                 const auto * creating_sets = typeid_cast<DelayedCreatingSetsStep *>(step);
-                bool allowed_creating_sets = settings[Setting::parallel_replicas_allow_in_with_subquery] && creating_sets;
+                const bool allowed_creating_sets = settings[Setting::parallel_replicas_allow_in_with_subquery] && creating_sets;
 
-                if (!expression && !filter && !allowed_creating_sets)
+                const auto * sorting = typeid_cast<SortingStep *>(step);
+                /// Sorting for merge join is supposed to be done locally before join itself, so it doesn't need finalization.
+                const bool allowed_sorting = sorting && sorting->isSortingForMergeJoin();
+
+                if (!expression && !filter && !allowed_creating_sets && !allowed_sorting)
+                {
                     can_distribute_full_node = false;
+                    currently_inside_join = inside_join;
+                }
 
-                next_node_to_check = children.front();
+                nodes_to_check.push({.node = children.front(), .inside_join = inside_join});
             }
             else
             {
@@ -221,12 +241,11 @@ const QueryNode * findQueryForParallelReplicas(
                 if (!join)
                     return res;
 
-                next_node_to_check = children.front();
+                for (const auto & child : children)
+                    nodes_to_check.push({.node = child, .inside_join = true});
             }
         }
 
-        /// Current node contains steps like GROUP BY / DISTINCT
-        /// Will try to execute query up to WithMergableStage
         if (!can_distribute_full_node)
         {
             /// Current query node does not contain subqueries.
@@ -234,12 +253,11 @@ const QueryNode * findQueryForParallelReplicas(
             if (!res)
                 return nullptr;
 
-            return subquery_node;
+            return currently_inside_join ? res : subquery_node;
         }
 
         /// Query is simple enough to be fully distributed.
         res = subquery_node;
-        prev_checked_node = curr_node;
     }
 
     return res;

src/Processors/QueryPlan/JoinStep.cpp

@@ -6,7 +6,6 @@
 #include <IO/Operators.h>
 #include <Common/JSONBuilder.h>
 #include <Common/typeid_cast.h>
-#include <Processors/Transforms/ColumnPermuteTransform.h>
 
 namespace DB
 {
@@ -37,37 +36,6 @@ std::vector<std::pair<String, String>> describeJoinActions(const JoinPtr & join)
     return description;
 }
 
-std::vector<size_t> getPermutationForBlock(
-    const Block & block,
-    const Block & lhs_block,
-    const Block & rhs_block,
-    const NameSet & name_filter)
-{
-    std::vector<size_t> permutation;
-    permutation.reserve(block.columns());
-    Block::NameMap name_map = block.getNamesToIndexesMap();
-
-    bool is_trivial = true;
-    for (const auto & other_block : {lhs_block, rhs_block})
-    {
-        for (const auto & col : other_block)
-        {
-            if (!name_filter.contains(col.name))
-                continue;
-            if (auto it = name_map.find(col.name); it != name_map.end())
-            {
-                is_trivial = is_trivial && it->second == permutation.size();
-                permutation.push_back(it->second);
-            }
-        }
-    }
-
-    if (is_trivial && permutation.size() == block.columns())
-        return {};
-
-    return permutation;
-}
-
 }
 
 JoinStep::JoinStep(
@@ -76,15 +44,8 @@ JoinStep::JoinStep(
     JoinPtr join_,
     size_t max_block_size_,
     size_t max_streams_,
-    NameSet required_output_,
+    bool keep_left_read_in_order_)
-    bool keep_left_read_in_order_,
+    : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_)
-    bool use_new_analyzer_)
-    : join(std::move(join_))
-    , max_block_size(max_block_size_)
-    , max_streams(max_streams_)
-    , required_output(std::move(required_output_))
-    , keep_left_read_in_order(keep_left_read_in_order_)
-    , use_new_analyzer(use_new_analyzer_)
 {
     updateInputHeaders({left_header_, right_header_});
 }
@@ -94,43 +55,23 @@ QueryPipelineBuilderPtr JoinStep::updatePipeline(QueryPipelineBuilders pipelines
     if (pipelines.size() != 2)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "JoinStep expect two input steps");
 
-    Block lhs_header = pipelines[0]->getHeader();
-    Block rhs_header = pipelines[1]->getHeader();
-
-    if (swap_streams)
-        std::swap(pipelines[0], pipelines[1]);
-
     if (join->pipelineType() == JoinPipelineType::YShaped)
     {
         auto joined_pipeline = QueryPipelineBuilder::joinPipelinesYShaped(
-            std::move(pipelines[0]), std::move(pipelines[1]), join, join_algorithm_header, max_block_size, &processors);
+            std::move(pipelines[0]), std::move(pipelines[1]), join, *output_header, max_block_size, &processors);
         joined_pipeline->resize(max_streams);
         return joined_pipeline;
     }
 
-    auto pipeline = QueryPipelineBuilder::joinPipelinesRightLeft(
+    return QueryPipelineBuilder::joinPipelinesRightLeft(
         std::move(pipelines[0]),
         std::move(pipelines[1]),
         join,
-        join_algorithm_header,
+        *output_header,
         max_block_size,
         max_streams,
         keep_left_read_in_order,
         &processors);
-
-    if (!use_new_analyzer)
-        return pipeline;
-
-    auto column_permutation = getPermutationForBlock(pipeline->getHeader(), lhs_header, rhs_header, required_output);
-    if (!column_permutation.empty())
-    {
-        pipeline->addSimpleTransform([&column_permutation](const Block & header)
-        {
-            return std::make_shared<ColumnPermuteTransform>(header, column_permutation);
-        });
-    }
-
-    return pipeline;
 }
 
 bool JoinStep::allowPushDownToRight() const
@@ -149,49 +90,17 @@ void JoinStep::describeActions(FormatSettings & settings) const
 
     for (const auto & [name, value] : describeJoinActions(join))
         settings.out << prefix << name << ": " << value << '\n';
-    if (swap_streams)
-        settings.out << prefix << "Swapped: true\n";
 }
 
 void JoinStep::describeActions(JSONBuilder::JSONMap & map) const
 {
     for (const auto & [name, value] : describeJoinActions(join))
         map.add(name, value);
-    if (swap_streams)
-        map.add("Swapped", true);
-}
-
-void JoinStep::setJoin(JoinPtr join_, bool swap_streams_)
-{
-    join_algorithm_header.clear();
-    swap_streams = swap_streams_;
-    join = std::move(join_);
-    updateOutputHeader();
 }
 
 void JoinStep::updateOutputHeader()
 {
-    if (join_algorithm_header)
+    output_header = JoiningTransform::transformHeader(input_headers.front(), join);
-        return;
-
-    const auto & header = swap_streams ? input_headers[1] : input_headers[0];
-
-    Block result_header = JoiningTransform::transformHeader(header, join);
-    join_algorithm_header = result_header;
-
-    if (!use_new_analyzer)
-    {
-        if (swap_streams)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot swap streams without new analyzer");
-        output_header = result_header;
-        return;
-    }
-
-    auto column_permutation = getPermutationForBlock(result_header, input_headers[0], input_headers[1], required_output);
-    if (!column_permutation.empty())
-        result_header = ColumnPermuteTransform::permute(result_header, column_permutation);
-
-    output_header = result_header;
 }
 
 static ITransformingStep::Traits getStorageJoinTraits()

src/Processors/QueryPlan/JoinStep.h

@@ -2,7 +2,6 @@
 
 #include <Processors/QueryPlan/IQueryPlanStep.h>
 #include <Processors/QueryPlan/ITransformingStep.h>
-#include <Core/Joins.h>
 
 namespace DB
 {
@@ -20,9 +19,7 @@ public:
         JoinPtr join_,
         size_t max_block_size_,
         size_t max_streams_,
-        NameSet required_output_,
+        bool keep_left_read_in_order_);
-        bool keep_left_read_in_order_,
-        bool use_new_analyzer_);
 
     String getName() const override { return "Join"; }
 
@@ -34,26 +31,16 @@ public:
     void describeActions(FormatSettings & settings) const override;
 
     const JoinPtr & getJoin() const { return join; }
-    void setJoin(JoinPtr join_, bool swap_streams_ = false);
+    void setJoin(JoinPtr join_) { join = std::move(join_); }
     bool allowPushDownToRight() const;
 
-    JoinInnerTableSelectionMode inner_table_selection_mode = JoinInnerTableSelectionMode::Right;
-
 private:
     void updateOutputHeader() override;
 
-    /// Header that expected to be returned from IJoin
-    Block join_algorithm_header;
-
     JoinPtr join;
     size_t max_block_size;
     size_t max_streams;
-
-    const NameSet required_output;
-    std::set<size_t> columns_to_remove;
     bool keep_left_read_in_order;
-    bool use_new_analyzer = false;
-    bool swap_streams = false;
 };
 
 /// Special step for the case when Join is already filled.

src/Processors/QueryPlan/Optimizations/Optimizations.h

@@ -113,7 +113,6 @@ void optimizePrimaryKeyConditionAndLimit(const Stack & stack);
 void optimizePrewhere(Stack & stack, QueryPlan::Nodes & nodes);
 void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes);
 void optimizeAggregationInOrder(QueryPlan::Node & node, QueryPlan::Nodes &);
-void optimizeJoin(QueryPlan::Node & node, QueryPlan::Nodes &);
 void optimizeDistinctInOrder(QueryPlan::Node & node, QueryPlan::Nodes &);
 
 /// A separate tree traverse to apply sorting properties after *InOrder optimizations.

src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp

@@ -1,102 +0,0 @@
-#include <Processors/QueryPlan/ExpressionStep.h>
-#include <Processors/QueryPlan/FilterStep.h>
-#include <Processors/QueryPlan/ITransformingStep.h>
-#include <Processors/QueryPlan/JoinStep.h>
-#include <Processors/QueryPlan/Optimizations/Optimizations.h>
-#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
-#include <Processors/QueryPlan/ReadFromMergeTree.h>
-#include <Processors/QueryPlan/SortingStep.h>
-#include <Storages/StorageMemory.h>
-#include <Processors/QueryPlan/ReadFromMemoryStorageStep.h>
-#include <Core/Settings.h>
-#include <Interpreters/IJoin.h>
-#include <Interpreters/HashJoin/HashJoin.h>
-
-#include <Interpreters/TableJoin.h>
-
-#include <Common/logger_useful.h>
-#include <Core/Joins.h>
-#include <ranges>
-
-namespace DB::QueryPlanOptimizations
-{
-
-static std::optional<UInt64> estimateReadRowsCount(QueryPlan::Node & node)
-{
-    IQueryPlanStep * step = node.step.get();
-    if (const auto * reading = typeid_cast<const ReadFromMergeTree *>(step))
-    {
-        if (auto analyzed_result = reading->getAnalyzedResult())
-            return analyzed_result->selected_rows;
-        if (auto analyzed_result = reading->selectRangesToRead())
-            return analyzed_result->selected_rows;
-        return {};
-    }
-
-    if (const auto * reading = typeid_cast<const ReadFromMemoryStorageStep *>(step))
-        return reading->getStorage()->totalRows(Settings{});
-
-    if (node.children.size() != 1)
-        return {};
-
-    if (typeid_cast<ExpressionStep *>(step) || typeid_cast<FilterStep *>(step))
-        return estimateReadRowsCount(*node.children.front());
-
-    return {};
-}
-
-void optimizeJoin(QueryPlan::Node & node, QueryPlan::Nodes &)
-{
-    auto * join_step = typeid_cast<JoinStep *>(node.step.get());
-    if (!join_step || node.children.size() != 2)
-        return;
-
-    const auto & join = join_step->getJoin();
-    if (join->pipelineType() != JoinPipelineType::FillRightFirst || !join->isCloneSupported())
-        return;
-
-    const auto & table_join = join->getTableJoin();
-
-    /// Algorithms other than HashJoin may not support OUTER JOINs
-    if (table_join.kind() != JoinKind::Inner && !typeid_cast<const HashJoin *>(join.get()))
-        return;
-
-    /// fixme: USING clause handled specially in join algorithm, so swap breaks it
-    /// fixme: Swapping for SEMI and ANTI joins should be alright, need to try to enable it and test
-    if (table_join.hasUsing() || table_join.strictness() != JoinStrictness::All)
-        return;
-
-    bool need_swap = false;
-    if (join_step->inner_table_selection_mode == JoinInnerTableSelectionMode::Auto)
-    {
-        auto lhs_extimation = estimateReadRowsCount(*node.children[0]);
-        auto rhs_extimation = estimateReadRowsCount(*node.children[1]);
-        LOG_TRACE(getLogger("optimizeJoin"), "Left table estimation: {}, right table estimation: {}",
-            lhs_extimation.transform(toString<UInt64>).value_or("unknown"),
-            rhs_extimation.transform(toString<UInt64>).value_or("unknown"));
-
-        if (lhs_extimation && rhs_extimation && *lhs_extimation < *rhs_extimation)
-            need_swap = true;
-    }
-    else if (join_step->inner_table_selection_mode == JoinInnerTableSelectionMode::Left)
-    {
-        need_swap = true;
-    }
-
-    if (!need_swap)
-        return;
-
-    const auto & headers = join_step->getInputHeaders();
-    if (headers.size() != 2)
-        return;
-
-    const auto & left_stream_input_header = headers.front();
-    const auto & right_stream_input_header = headers.back();
-
-    auto updated_table_join = std::make_shared<TableJoin>(table_join);
-    updated_table_join->swapSides();
-    auto updated_join = join->clone(updated_table_join, right_stream_input_header, left_stream_input_header);
-    join_step->setJoin(std::move(updated_join), /* swap_streams= */ true);
-}
-
-}

src/Processors/QueryPlan/Optimizations/optimizeTree.cpp

@@ -227,9 +227,6 @@ void addStepsToBuildSets(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::No
         /// NOTE: frame cannot be safely used after stack was modified.
         auto & frame = stack.back();
 
-        if (frame.next_child == 0)
-            optimizeJoin(*frame.node, nodes);
-
         /// Traverse all children first.
         if (frame.next_child < frame.node->children.size())
         {

src/Processors/QueryPlan/ReadFromMemoryStorageStep.h

@@ -35,8 +35,6 @@ public:
 
     void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
 
-    const StoragePtr & getStorage() const { return storage; }
-
 private:
     static constexpr auto name = "ReadFromMemoryStorage";
 

src/Processors/QueryPlan/SortingStep.cpp

@@ -77,13 +77,11 @@ static ITransformingStep::Traits getTraits(size_t limit)
 }
 
 SortingStep::SortingStep(
-    const Header & input_header,
+    const Header & input_header, SortDescription description_, UInt64 limit_, const Settings & settings_, bool is_sorting_for_merge_join_)
-    SortDescription description_,
-    UInt64 limit_,
-    const Settings & settings_)
     : ITransformingStep(input_header, input_header, getTraits(limit_))
     , type(Type::Full)
     , result_description(std::move(description_))
+    , is_sorting_for_merge_join(is_sorting_for_merge_join_)
     , limit(limit_)
     , sort_settings(settings_)
 {

src/Processors/QueryPlan/SortingStep.h

@@ -39,7 +39,8 @@ public:
         const Header & input_header,
         SortDescription description_,
         UInt64 limit_,
-        const Settings & settings_);
+        const Settings & settings_,
+        bool is_sorting_for_merge_join_ = false);
 
     /// Full with partitioning
     SortingStep(
@@ -81,6 +82,8 @@ public:
 
     bool hasPartitions() const { return !partition_by_description.empty(); }
 
+    bool isSortingForMergeJoin() const { return is_sorting_for_merge_join; }
+
     void convertToFinishSorting(SortDescription prefix_description, bool use_buffering_, bool apply_virtual_row_conversions_);
 
     Type getType() const { return type; }
@@ -125,6 +128,9 @@ private:
 
     SortDescription partition_by_description;
 
+    /// See `findQueryForParallelReplicas`
+    bool is_sorting_for_merge_join = false;
+
     UInt64 limit;
     bool always_read_till_end = false;
     bool use_buffering = false;

src/Processors/Transforms/ColumnPermuteTransform.cpp

@@ -1,49 +0,0 @@
-#include <Processors/Transforms/ColumnPermuteTransform.h>
-
-namespace DB
-{
-
-namespace
-{
-
-template <typename T>
-void applyPermutation(std::vector<T> & data, const std::vector<size_t> & permutation)
-{
-    std::vector<T> res;
-    res.reserve(permutation.size());
-    for (size_t i : permutation)
-        res.push_back(data[i]);
-    data = std::move(res);
-}
-
-void permuteChunk(Chunk & chunk, const std::vector<size_t> & permutation)
-{
-    size_t num_rows = chunk.getNumRows();
-    auto columns = chunk.detachColumns();
-    applyPermutation(columns, permutation);
-    chunk.setColumns(std::move(columns), num_rows);
-}
-
-}
-
-Block ColumnPermuteTransform::permute(const Block & block, const std::vector<size_t> & permutation)
-{
-    auto columns = block.getColumnsWithTypeAndName();
-    applyPermutation(columns, permutation);
-    return Block(columns);
-}
-
-ColumnPermuteTransform::ColumnPermuteTransform(const Block & header_, const std::vector<size_t> & permutation_)
-    : ISimpleTransform(header_, permute(header_, permutation_), false)
-    , permutation(permutation_)
-{
-}
-
-
-void ColumnPermuteTransform::transform(Chunk & chunk)
-{
-    permuteChunk(chunk, permutation);
-}
-
-
-}

src/Processors/Transforms/ColumnPermuteTransform.h

@@ -1,30 +0,0 @@
-#pragma once
-
-#include <atomic>
-#include <mutex>
-#include <vector>
-#include <Processors/ISimpleTransform.h>
-#include <Poco/Logger.h>
-#include <Interpreters/Set.h>
-
-namespace DB
-{
-
-class ColumnPermuteTransform : public ISimpleTransform
-{
-public:
-    ColumnPermuteTransform(const Block & header_, const std::vector<size_t> & permutation_);
-
-    String getName() const override { return "ColumnPermuteTransform"; }
-
-    void transform(Chunk & chunk) override;
-
-    static Block permute(const Block & block, const std::vector<size_t> & permutation);
-
-private:
-    Names column_names;
-    std::vector<size_t> permutation;
-};
-
-
-}

src/Processors/Transforms/JoiningTransform.cpp

@@ -19,7 +19,6 @@ Block JoiningTransform::transformHeader(Block header, const JoinPtr & join)
     join->initialize(header);
     ExtraBlockPtr tmp;
     join->joinBlock(header, tmp);
-    materializeBlockInplace(header);
     LOG_TEST(getLogger("JoiningTransform"), "After join block: '{}'", header.dumpStructure());
     return header;
 }

src/QueryPipeline/ExecutionSpeedLimits.cpp

@@ -86,10 +86,12 @@ void ExecutionSpeedLimits::throttle(
                 if (timeout_overflow_mode == OverflowMode::THROW && estimated_execution_time_seconds > max_estimated_execution_time.totalSeconds())
                     throw Exception(
                         ErrorCodes::TOO_SLOW,
-                        "Estimated query execution time ({} seconds) is too long. Maximum: {}. Estimated rows to process: {}",
+                        "Estimated query execution time ({:.5f} seconds) is too long. Maximum: {}. Estimated rows to process: {} ({} read in {:.5f} seconds).",
                         estimated_execution_time_seconds,
                         max_estimated_execution_time.totalSeconds(),
-                        total_rows_to_read);
+                        total_rows_to_read,
+                        read_rows,
+                        elapsed_seconds);
             }
 
             if (max_execution_rps && rows_per_second >= max_execution_rps)

src/Server/TCPHandler.cpp

@@ -1614,7 +1614,8 @@ void TCPHandler::receiveHello()
                 if (e.code() != DB::ErrorCodes::AUTHENTICATION_FAILED)
                     throw;
 
-                tryLogCurrentException(log, "SSL authentication failed, falling back to password authentication");
+                tryLogCurrentException(log, "SSL authentication failed, falling back to password authentication", LogsLevel::information);
+                /// ^^ Log at debug level instead of default error level as authentication failures are not an unusual event.
             }
         }
     }

src/Storages/MergeTree/DataPartsExchange.cpp

@@ -908,7 +908,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk(
     {
         part_storage_for_loading->commitTransaction();
 
-        MergeTreeDataPartBuilder builder(data, part_name, volume, part_relative_path, part_dir);
+        MergeTreeDataPartBuilder builder(data, part_name, volume, part_relative_path, part_dir, getReadSettings());
         new_data_part = builder.withPartFormatFromDisk().build();
 
         new_data_part->version.setCreationTID(Tx::PrehistoricTID, nullptr);

src/Storages/MergeTree/IMergeTreeDataPart.cpp

@@ -833,7 +833,7 @@ MergeTreeDataPartBuilder IMergeTreeDataPart::getProjectionPartBuilder(const Stri
 {
     const char * projection_extension = is_temp_projection ? ".tmp_proj" : ".proj";
     auto projection_storage = getDataPartStorage().getProjection(projection_name + projection_extension, !is_temp_projection);
-    MergeTreeDataPartBuilder builder(storage, projection_name, projection_storage);
+    MergeTreeDataPartBuilder builder(storage, projection_name, projection_storage, getReadSettings());
     return builder.withPartInfo(MergeListElement::FAKE_RESULT_PART_FOR_PROJECTION).withParentPart(this);
 }
 

src/Storages/MergeTree/IMergeTreeDataPartWriter.h

@@ -46,6 +46,8 @@ public:
 
     virtual void finish(bool sync) = 0;
 
+    virtual size_t getNumberOfOpenStreams() const = 0;
+
     Columns releaseIndexColumns();
 
     PlainMarksByName releaseCachedMarks();

src/Storages/MergeTree/IMergedBlockOutputStream.h

@@ -39,6 +39,11 @@ public:
         return writer->releaseCachedMarks();
     }
 
+    size_t getNumberOfOpenStreams() const
+    {
+        return writer->getNumberOfOpenStreams();
+    }
+
 protected:
 
     /// Remove all columns marked expired in data_part. Also, clears checksums

src/Storages/MergeTree/MergeTask.cpp

@@ -349,13 +349,13 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const
     if (global_ctx->parent_part)
     {
         auto data_part_storage = global_ctx->parent_part->getDataPartStorage().getProjection(local_tmp_part_basename,  /* use parent transaction */ false);
-        builder.emplace(*global_ctx->data, global_ctx->future_part->name, data_part_storage);
+        builder.emplace(*global_ctx->data, global_ctx->future_part->name, data_part_storage, getReadSettings());
         builder->withParentPart(global_ctx->parent_part);
     }
     else
     {
         auto local_single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + global_ctx->future_part->name, global_ctx->disk, 0);
-        builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename));
+        builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename, getReadSettings()));
         builder->withPartStorageType(global_ctx->future_part->part_format.storage_type);
     }
 

src/Storages/MergeTree/MergeTreeData.cpp

@@ -1442,7 +1442,7 @@ void MergeTreeData::loadUnexpectedDataPart(UnexpectedPartLoadState & state)
 
     try
     {
-        state.part = getDataPartBuilder(part_name, single_disk_volume, part_name)
+        state.part = getDataPartBuilder(part_name, single_disk_volume, part_name, getReadSettings())
             .withPartInfo(part_info)
             .withPartFormatFromDisk()
             .build();
@@ -1457,7 +1457,7 @@ void MergeTreeData::loadUnexpectedDataPart(UnexpectedPartLoadState & state)
             /// Build a fake part and mark it as broken in case of filesystem error.
             /// If the error impacts part directory instead of single files,
             /// an exception will be thrown during detach and silently ignored.
-            state.part = getDataPartBuilder(part_name, single_disk_volume, part_name)
+            state.part = getDataPartBuilder(part_name, single_disk_volume, part_name, getReadSettings())
                 .withPartStorageType(MergeTreeDataPartStorageType::Full)
                 .withPartType(MergeTreeDataPartType::Wide)
                 .build();
@@ -1491,7 +1491,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart(
             /// Build a fake part and mark it as broken in case of filesystem error.
             /// If the error impacts part directory instead of single files,
             /// an exception will be thrown during detach and silently ignored.
-            res.part = getDataPartBuilder(part_name, single_disk_volume, part_name)
+            res.part = getDataPartBuilder(part_name, single_disk_volume, part_name, getReadSettings())
                 .withPartStorageType(MergeTreeDataPartStorageType::Full)
                 .withPartType(MergeTreeDataPartType::Wide)
                 .build();
@@ -1512,7 +1512,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart(
 
     try
     {
-        res.part = getDataPartBuilder(part_name, single_disk_volume, part_name)
+        res.part = getDataPartBuilder(part_name, single_disk_volume, part_name, getReadSettings())
             .withPartInfo(part_info)
             .withPartFormatFromDisk()
             .build();
@@ -3835,9 +3835,9 @@ MergeTreeDataPartFormat MergeTreeData::choosePartFormatOnDisk(size_t bytes_uncom
 }
 
 MergeTreeDataPartBuilder MergeTreeData::getDataPartBuilder(
-    const String & name, const VolumePtr & volume, const String & part_dir) const
+    const String & name, const VolumePtr & volume, const String & part_dir, const ReadSettings & read_settings_) const
 {
-    return MergeTreeDataPartBuilder(*this, name, volume, relative_data_path, part_dir);
+    return MergeTreeDataPartBuilder(*this, name, volume, relative_data_path, part_dir, read_settings_);
 }
 
 void MergeTreeData::changeSettings(
@@ -5919,7 +5919,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(cons
     /// Load this part from the directory `temp_part_dir`.
     auto load_part = [&]
     {
-        MergeTreeDataPartBuilder builder(*this, part_name, single_disk_volume, parent_part_dir, part_dir_name);
+        MergeTreeDataPartBuilder builder(*this, part_name, single_disk_volume, parent_part_dir, part_dir_name, getReadSettings());
         builder.withPartFormatFromDisk();
         part = std::move(builder).build();
         part->version.setCreationTID(Tx::PrehistoricTID, nullptr);
@@ -5934,7 +5934,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(cons
         if (!part)
         {
             /// Make a fake data part only to copy its files to /detached/.
-            part = MergeTreeDataPartBuilder{*this, part_name, single_disk_volume, parent_part_dir, part_dir_name}
+            part = MergeTreeDataPartBuilder{*this, part_name, single_disk_volume, parent_part_dir, part_dir_name, getReadSettings()}
                        .withPartStorageType(MergeTreeDataPartStorageType::Full)
                        .withPartType(MergeTreeDataPartType::Wide)
                        .build();
@@ -6586,7 +6586,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const
         LOG_DEBUG(log, "Checking part {}", new_name);
 
         auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + old_name, disk);
-        auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir / new_name)
+        auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir / new_name, getReadSettings())
             .withPartFormatFromDisk()
             .build();
 
@@ -7641,7 +7641,7 @@ std::pair<MergeTreeData::MutableDataPartPtr, scope_guard> MergeTreeData::cloneAn
               std::string(fs::path(dst_part_storage->getFullRootPath()) / tmp_dst_part_name),
               with_copy);
 
-    auto dst_data_part = MergeTreeDataPartBuilder(*this, dst_part_name, dst_part_storage)
+    auto dst_data_part = MergeTreeDataPartBuilder(*this, dst_part_name, dst_part_storage, getReadSettings())
         .withPartFormatFromDisk()
         .build();
 
@@ -8910,7 +8910,7 @@ std::pair<MergeTreeData::MutableDataPartPtr, scope_guard> MergeTreeData::createE
     VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume);
 
     auto tmp_dir_holder = getTemporaryPartDirectoryHolder(EMPTY_PART_TMP_PREFIX + new_part_name);
-    auto new_data_part = getDataPartBuilder(new_part_name, data_part_volume, EMPTY_PART_TMP_PREFIX + new_part_name)
+    auto new_data_part = getDataPartBuilder(new_part_name, data_part_volume, EMPTY_PART_TMP_PREFIX + new_part_name, getReadSettings())
         .withBytesAndRowsOnDisk(0, 0)
         .withPartInfo(new_part_info)
         .build();

src/Storages/MergeTree/MergeTreeData.h

@@ -241,7 +241,7 @@ public:
 
     MergeTreeDataPartFormat choosePartFormat(size_t bytes_uncompressed, size_t rows_count) const;
     MergeTreeDataPartFormat choosePartFormatOnDisk(size_t bytes_uncompressed, size_t rows_count) const;
-    MergeTreeDataPartBuilder getDataPartBuilder(const String & name, const VolumePtr & volume, const String & part_dir) const;
+    MergeTreeDataPartBuilder getDataPartBuilder(const String & name, const VolumePtr & volume, const String & part_dir, const ReadSettings & read_settings_) const;
 
     /// Auxiliary object to add a set of parts into the working set in two steps:
     /// * First, as PreActive parts (the parts are ready, but not yet in the active set).

src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp

@@ -48,6 +48,16 @@ namespace CurrentMetrics
 {
     extern const Metric BackgroundMergesAndMutationsPoolTask;
 }
+namespace ProfileEvents
+{
+
+    extern const Event MergerMutatorsGetPartsForMergeElapsedMicroseconds;
+    extern const Event MergerMutatorPrepareRangesForMergeElapsedMicroseconds;
+    extern const Event MergerMutatorSelectPartsForMergeElapsedMicroseconds;
+    extern const Event MergerMutatorRangesForMergeCount;
+    extern const Event MergerMutatorPartsInRangesForMergeCount;
+    extern const Event MergerMutatorSelectRangePartsCount;
+}
 
 namespace DB
 {
@@ -71,6 +81,7 @@ namespace MergeTreeSetting
     extern const MergeTreeSettingsUInt64 parts_to_throw_insert;
     extern const MergeTreeSettingsMergeSelectorAlgorithm merge_selector_algorithm;
     extern const MergeTreeSettingsBool merge_selector_enable_heuristic_to_remove_small_parts_at_right;
+    extern const MergeTreeSettingsFloat merge_selector_base;
 }
 
 namespace ErrorCodes
@@ -214,6 +225,7 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
 {
     PartitionIdsHint res;
     MergeTreeData::DataPartsVector data_parts = getDataPartsToSelectMergeFrom(txn);
+
     if (data_parts.empty())
         return res;
 
@@ -271,6 +283,8 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
 MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::getDataPartsToSelectMergeFrom(
     const MergeTreeTransactionPtr & txn, const PartitionIdsHint * partitions_hint) const
 {
+
+    Stopwatch get_data_parts_for_merge_timer;
     auto res = getDataPartsToSelectMergeFrom(txn);
     if (!partitions_hint)
         return res;
@@ -279,6 +293,8 @@ MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::getDataPartsToSelectM
     {
         return !partitions_hint->contains(part->info.partition_id);
     });
+
+    ProfileEvents::increment(ProfileEvents::MergerMutatorsGetPartsForMergeElapsedMicroseconds, get_data_parts_for_merge_timer.elapsedMicroseconds());
     return res;
 }
 
@@ -356,6 +372,7 @@ MergeTreeDataMergerMutator::MergeSelectingInfo MergeTreeDataMergerMutator::getPo
     const MergeTreeTransactionPtr & txn,
     PreformattedMessage & out_disable_reason) const
 {
+    Stopwatch ranges_for_merge_timer;
     MergeSelectingInfo res;
 
     res.current_time = std::time(nullptr);
@@ -456,6 +473,10 @@ MergeTreeDataMergerMutator::MergeSelectingInfo MergeTreeDataMergerMutator::getPo
         prev_part = ∂
     }
 
+    ProfileEvents::increment(ProfileEvents::MergerMutatorPartsInRangesForMergeCount, res.parts_selected_precondition);
+    ProfileEvents::increment(ProfileEvents::MergerMutatorRangesForMergeCount, res.parts_ranges.size());
+    ProfileEvents::increment(ProfileEvents::MergerMutatorPrepareRangesForMergeElapsedMicroseconds, ranges_for_merge_timer.elapsedMicroseconds());
+
     return res;
 }
 
@@ -470,6 +491,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
     PreformattedMessage & out_disable_reason,
     bool dry_run)
 {
+    Stopwatch select_parts_from_ranges_timer;
     const auto data_settings = data.getSettings();
     IMergeSelector::PartsRange parts_to_merge;
 
@@ -542,6 +564,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
             simple_merge_settings.window_size = (*data_settings)[MergeTreeSetting::merge_selector_window_size];
             simple_merge_settings.max_parts_to_merge_at_once = (*data_settings)[MergeTreeSetting::max_parts_to_merge_at_once];
             simple_merge_settings.enable_heuristic_to_remove_small_parts_at_right = (*data_settings)[MergeTreeSetting::merge_selector_enable_heuristic_to_remove_small_parts_at_right];
+            simple_merge_settings.base = (*data_settings)[MergeTreeSetting::merge_selector_base];
 
             if (!(*data_settings)[MergeTreeSetting::min_age_to_force_merge_on_partition_only])
                 simple_merge_settings.min_age_to_force_merge = (*data_settings)[MergeTreeSetting::min_age_to_force_merge_seconds];
@@ -568,7 +591,8 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
 
         if (parts_to_merge.empty())
         {
-            out_disable_reason = PreformattedMessage::create("Did not find any parts to merge (with usual merge selectors)");
+            ProfileEvents::increment(ProfileEvents::MergerMutatorSelectPartsForMergeElapsedMicroseconds, select_parts_from_ranges_timer.elapsedMicroseconds());
+            out_disable_reason = PreformattedMessage::create("Did not find any parts to merge (with usual merge selectors) in {}ms", select_parts_from_ranges_timer.elapsedMicroseconds() / 1000);
             return SelectPartsDecision::CANNOT_SELECT;
         }
     }
@@ -581,8 +605,11 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
         parts.push_back(part);
     }
 
-    LOG_DEBUG(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name);
+    LOG_DEBUG(log, "Selected {} parts from {} to {} in {}ms", parts.size(), parts.front()->name, parts.back()->name, select_parts_from_ranges_timer.elapsedMicroseconds() / 1000);
+    ProfileEvents::increment(ProfileEvents::MergerMutatorSelectRangePartsCount, parts.size());
+
     future_part->assign(std::move(parts));
+    ProfileEvents::increment(ProfileEvents::MergerMutatorSelectPartsForMergeElapsedMicroseconds, select_parts_from_ranges_timer.elapsedMicroseconds());
     return SelectPartsDecision::SELECTED;
 }
 

src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp

@@ -14,20 +14,22 @@ namespace ErrorCodes
 }
 
 MergeTreeDataPartBuilder::MergeTreeDataPartBuilder(
-    const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_)
+    const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_, const ReadSettings & read_settings_)
     : data(data_)
     , name(std::move(name_))
     , volume(std::move(volume_))
     , root_path(std::move(root_path_))
     , part_dir(std::move(part_dir_))
+    , read_settings(read_settings_)
 {
 }
 
 MergeTreeDataPartBuilder::MergeTreeDataPartBuilder(
-    const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_)
+    const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_, const ReadSettings & read_settings_)
     : data(data_)
     , name(std::move(name_))
     , part_storage(std::move(part_storage_))
+    , read_settings(read_settings_)
 {
 }
 
@@ -73,7 +75,8 @@ MutableDataPartStoragePtr MergeTreeDataPartBuilder::getPartStorageByType(
     MergeTreeDataPartStorageType storage_type_,
     const VolumePtr & volume_,
     const String & root_path_,
-    const String & part_dir_)
+    const String & part_dir_,
+    const ReadSettings &) /// Unused here, but used in private repo.
 {
     if (!volume_)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create part storage, because volume is not specified");
@@ -112,7 +115,7 @@ MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartType(MergeTreeDataP
 
 MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartStorageType(MergeTreeDataPartStorageType storage_type_)
 {
-    part_storage = getPartStorageByType(storage_type_, volume, root_path, part_dir);
+    part_storage = getPartStorageByType(storage_type_, volume, root_path, part_dir, read_settings);
     return *this;
 }
 
@@ -126,7 +129,8 @@ MergeTreeDataPartBuilder::PartStorageAndMarkType
 MergeTreeDataPartBuilder::getPartStorageAndMarkType(
     const VolumePtr & volume_,
     const String & root_path_,
-    const String & part_dir_)
+    const String & part_dir_,
+    const ReadSettings & read_settings_)
 {
     auto disk = volume_->getDisk();
     auto part_relative_path = fs::path(root_path_) / part_dir_;
@@ -138,7 +142,7 @@ MergeTreeDataPartBuilder::getPartStorageAndMarkType(
 
         if (MarkType::isMarkFileExtension(ext))
         {
-            auto storage = getPartStorageByType(MergeTreeDataPartStorageType::Full, volume_, root_path_, part_dir_);
+            auto storage = getPartStorageByType(MergeTreeDataPartStorageType::Full, volume_, root_path_, part_dir_, read_settings_);
             return {std::move(storage), MarkType(ext)};
         }
     }
@@ -156,7 +160,7 @@ MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromDisk()
 MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromVolume()
 {
     assert(volume);
-    auto [storage, mark_type] = getPartStorageAndMarkType(volume, root_path, part_dir);
+    auto [storage, mark_type] = getPartStorageAndMarkType(volume, root_path, part_dir, read_settings);
 
     if (!storage || !mark_type)
     {

src/Storages/MergeTree/MergeTreeDataPartBuilder.h

@@ -21,8 +21,8 @@ using VolumePtr = std::shared_ptr<IVolume>;
 class MergeTreeDataPartBuilder
 {
 public:
-    MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_);
+    MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_, const ReadSettings & read_settings_);
-    MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_);
+    MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_, const ReadSettings & read_settings_);
 
     std::shared_ptr<IMergeTreeDataPart> build();
 
@@ -42,7 +42,8 @@ public:
     static PartStorageAndMarkType getPartStorageAndMarkType(
         const VolumePtr & volume_,
         const String & root_path_,
-        const String & part_dir_);
+        const String & part_dir_,
+        const ReadSettings & read_settings);
 
 private:
     Self & withPartFormatFromVolume();
@@ -52,7 +53,8 @@ private:
         MergeTreeDataPartStorageType storage_type_,
         const VolumePtr & volume_,
         const String & root_path_,
-        const String & part_dir_);
+        const String & part_dir_,
+        const ReadSettings & read_settings);
 
     const MergeTreeData & data;
     const String name;
@@ -64,6 +66,8 @@ private:
     std::optional<MergeTreeDataPartType> part_type;
     MutableDataPartStoragePtr part_storage;
     const IMergeTreeDataPart * parent_part = nullptr;
+
+    const ReadSettings read_settings;
 };
 
 }

src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h

@@ -32,6 +32,8 @@ public:
     void fillChecksums(MergeTreeDataPartChecksums & checksums, NameSet & checksums_to_remove) override;
     void finish(bool sync) override;
 
+    size_t getNumberOfOpenStreams() const override { return 1; }
+
 private:
     /// Finish serialization of the data. Flush rows in buffer to disk, compute checksums.
     void fillDataChecksums(MergeTreeDataPartChecksums & checksums);

src/Storages/MergeTree/MergeTreeDataPartWriterWide.h

@@ -43,6 +43,8 @@ public:
 
     void finish(bool sync) final;
 
+    size_t getNumberOfOpenStreams() const override { return column_streams.size(); }
+
 private:
     /// Finish serialization of data: write final mark if required and compute checksums
     /// Also validate written data in debug mode

src/Storages/MergeTree/MergeTreeDataWriter.cpp

@@ -610,7 +610,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl(
         }
     }
 
-    auto new_data_part = data.getDataPartBuilder(part_name, data_part_volume, part_dir)
+    auto new_data_part = data.getDataPartBuilder(part_name, data_part_volume, part_dir, getReadSettings())
         .withPartFormat(data.choosePartFormat(expected_size, block.rows()))
         .withPartInfo(new_part_info)
         .build();

src/Storages/MergeTree/MergeTreePartsMover.cpp

@@ -280,7 +280,7 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me
         cloned_part_storage = part->makeCloneOnDisk(disk, MergeTreeData::MOVING_DIR_NAME, read_settings, write_settings, cancellation_hook);
     }
 
-    MergeTreeDataPartBuilder builder(*data, part->name, cloned_part_storage);
+    MergeTreeDataPartBuilder builder(*data, part->name, cloned_part_storage, getReadSettings());
     cloned_part.part = std::move(builder).withPartFormatFromDisk().build();
     LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part.part->getDataPartStorage().getFullPath());
 

src/Storages/MergeTree/MergeTreeSettings.cpp

@@ -101,6 +101,7 @@ namespace ErrorCodes
     DECLARE(Milliseconds, background_task_preferred_step_execution_time_ms, 50, "Target time to execution of one step of merge or mutation. Can be exceeded if one step takes longer time", 0) \
     DECLARE(MergeSelectorAlgorithm, merge_selector_algorithm, MergeSelectorAlgorithm::SIMPLE, "The algorithm to select parts for merges assignment", EXPERIMENTAL) \
     DECLARE(Bool, merge_selector_enable_heuristic_to_remove_small_parts_at_right, true, "Enable heuristic for selecting parts for merge which removes parts from right side of range, if their size is less than specified ratio (0.01) of sum_size. Works for Simple and StochasticSimple merge selectors", 0) \
+    DECLARE(Float, merge_selector_base, 5.0, "Affects write amplification of assigned merges (expert level setting, don't change if you don't understand what it is doing). Works for Simple and StochasticSimple merge selectors", 0) \
     \
     /** Inserts settings. */ \
     DECLARE(UInt64, parts_to_delay_insert, 1000, "If table contains at least that many active parts in single partition, artificially slow down insert into table. Disabled if set to 0", 0) \

src/Storages/MergeTree/MergeTreeSink.cpp

@@ -94,7 +94,7 @@ void MergeTreeSink::consume(Chunk & chunk)
     DelayedPartitions partitions;
 
     const Settings & settings = context->getSettingsRef();
-    size_t streams = 0;
+    size_t total_streams = 0;
     bool support_parallel_write = false;
 
     auto token_info = chunk.getChunkInfos().get<DeduplicationToken::TokenInfo>();
@@ -153,16 +153,18 @@ void MergeTreeSink::consume(Chunk & chunk)
             max_insert_delayed_streams_for_parallel_write = 0;
 
         /// In case of too much columns/parts in block, flush explicitly.
-        streams += temp_part.streams.size();
+        size_t current_streams = 0;
+        for (const auto & stream : temp_part.streams)
+            current_streams += stream.stream->getNumberOfOpenStreams();
 
-        if (streams > max_insert_delayed_streams_for_parallel_write)
+        if (total_streams + current_streams > max_insert_delayed_streams_for_parallel_write)
         {
             finishDelayedChunk();
             delayed_chunk = std::make_unique<MergeTreeSink::DelayedChunk>();
             delayed_chunk->partitions = std::move(partitions);
             finishDelayedChunk();
 
-            streams = 0;
+            total_streams = 0;
             support_parallel_write = false;
             partitions = DelayedPartitions{};
         }
@@ -174,6 +176,8 @@ void MergeTreeSink::consume(Chunk & chunk)
             .block_dedup_token = block_dedup_token,
             .part_counters = std::move(part_counters),
         });
+
+        total_streams += current_streams;
     }
 
     if (need_to_define_dedup_token)

src/Storages/MergeTree/MutateTask.cpp

@@ -2289,7 +2289,7 @@ bool MutateTask::prepare()
     String tmp_part_dir_name = prefix + ctx->future_part->name;
     ctx->temporary_directory_lock = ctx->data->getTemporaryPartDirectoryHolder(tmp_part_dir_name);
 
-    auto builder = ctx->data->getDataPartBuilder(ctx->future_part->name, single_disk_volume, tmp_part_dir_name);
+    auto builder = ctx->data->getDataPartBuilder(ctx->future_part->name, single_disk_volume, tmp_part_dir_name, getReadSettings());
     builder.withPartFormat(ctx->future_part->part_format);
     builder.withPartInfo(ctx->future_part->part_info);
 

src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp

@@ -341,7 +341,7 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk & chunk)
     using DelayedPartitions = std::vector<DelayedPartition>;
     DelayedPartitions partitions;
 
-    size_t streams = 0;
+    size_t total_streams = 0;
     bool support_parallel_write = false;
 
     for (auto & current_block : part_blocks)
@@ -418,15 +418,18 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk & chunk)
             max_insert_delayed_streams_for_parallel_write = 0;
 
         /// In case of too much columns/parts in block, flush explicitly.
-        streams += temp_part.streams.size();
+        size_t current_streams = 0;
-        if (streams > max_insert_delayed_streams_for_parallel_write)
+        for (const auto & stream : temp_part.streams)
+            current_streams += stream.stream->getNumberOfOpenStreams();
+
+        if (total_streams + current_streams > max_insert_delayed_streams_for_parallel_write)
         {
             finishDelayedChunk(zookeeper);
             delayed_chunk = std::make_unique<ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk>(replicas_num);
             delayed_chunk->partitions = std::move(partitions);
             finishDelayedChunk(zookeeper);
 
-            streams = 0;
+            total_streams = 0;
             support_parallel_write = false;
             partitions = DelayedPartitions{};
         }
@@ -447,6 +450,8 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk & chunk)
             std::move(unmerged_block),
             std::move(part_counters) /// profile_events_scope must be reset here.
         ));
+
+        total_streams += current_streams;
     }
 
     if (need_to_define_dedup_token)

src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h

@@ -0,0 +1,112 @@
+#pragma once
+
+#include <Storages/IStorage.h>
+#include <Storages/ObjectStorage/Azure/Configuration.h>
+#include <Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h>
+#include <Storages/ObjectStorage/DataLakes/HudiMetadata.h>
+#include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
+#include <Storages/ObjectStorage/DataLakes/IcebergMetadata.h>
+#include <Storages/ObjectStorage/HDFS/Configuration.h>
+#include <Storages/ObjectStorage/Local/Configuration.h>
+#include <Storages/ObjectStorage/S3/Configuration.h>
+#include <Storages/ObjectStorage/StorageObjectStorage.h>
+#include <Storages/StorageFactory.h>
+#include <Common/logger_useful.h>
+
+#include <memory>
+
+
+namespace DB
+{
+
+template <typename T>
+concept StorageConfiguration = std::derived_from<T, StorageObjectStorage::Configuration>;
+
+template <StorageConfiguration BaseStorageConfiguration, typename DataLakeMetadata>
+class DataLakeConfiguration : public BaseStorageConfiguration, public std::enable_shared_from_this<StorageObjectStorage::Configuration>
+{
+public:
+    using Configuration = StorageObjectStorage::Configuration;
+
+    bool isDataLakeConfiguration() const override { return true; }
+
+    std::string getEngineName() const override { return DataLakeMetadata::name; }
+
+    void update(ObjectStoragePtr object_storage, ContextPtr local_context) override
+    {
+        BaseStorageConfiguration::update(object_storage, local_context);
+        auto new_metadata = DataLakeMetadata::create(object_storage, weak_from_this(), local_context);
+        if (current_metadata && *current_metadata == *new_metadata)
+            return;
+
+        current_metadata = std::move(new_metadata);
+        BaseStorageConfiguration::setPaths(current_metadata->getDataFiles());
+        BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns());
+    }
+
+    std::optional<ColumnsDescription> tryGetTableStructureFromMetadata() const override
+    {
+        if (!current_metadata)
+            return std::nullopt;
+        auto schema_from_metadata = current_metadata->getTableSchema();
+        if (!schema_from_metadata.empty())
+        {
+            return ColumnsDescription(std::move(schema_from_metadata));
+        }
+        return std::nullopt;
+    }
+
+private:
+    DataLakeMetadataPtr current_metadata;
+
+    ReadFromFormatInfo prepareReadingFromFormat(
+        ObjectStoragePtr object_storage,
+        const Strings & requested_columns,
+        const StorageSnapshotPtr & storage_snapshot,
+        bool supports_subset_of_columns,
+        ContextPtr local_context) override
+    {
+        auto info = DB::prepareReadingFromFormat(requested_columns, storage_snapshot, local_context, supports_subset_of_columns);
+        if (!current_metadata)
+        {
+            current_metadata = DataLakeMetadata::create(object_storage, weak_from_this(), local_context);
+        }
+        auto column_mapping = current_metadata->getColumnNameToPhysicalNameMapping();
+        if (!column_mapping.empty())
+        {
+            for (const auto & [column_name, physical_name] : column_mapping)
+            {
+                auto & column = info.format_header.getByName(column_name);
+                column.name = physical_name;
+            }
+        }
+        return info;
+    }
+};
+
+#if USE_AVRO
+#if USE_AWS_S3
+using StorageS3IcebergConfiguration = DataLakeConfiguration<StorageS3Configuration, IcebergMetadata>;
+#    endif
+
+#if USE_AZURE_BLOB_STORAGE
+using StorageAzureIcebergConfiguration = DataLakeConfiguration<StorageAzureConfiguration, IcebergMetadata>;
+#    endif
+
+#if USE_HDFS
+using StorageHDFSIcebergConfiguration = DataLakeConfiguration<StorageHDFSConfiguration, IcebergMetadata>;
+#    endif
+
+using StorageLocalIcebergConfiguration = DataLakeConfiguration<StorageLocalConfiguration, IcebergMetadata>;
+#endif
+
+#if USE_PARQUET
+#if USE_AWS_S3
+using StorageS3DeltaLakeConfiguration = DataLakeConfiguration<StorageS3Configuration, DeltaLakeMetadata>;
+#    endif
+#endif
+
+#if USE_AWS_S3
+using StorageS3HudiConfiguration = DataLakeConfiguration<StorageS3Configuration, HudiMetadata>;
+#endif
+}

src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp

@@ -56,22 +56,18 @@ namespace ErrorCodes
 
 struct DeltaLakeMetadataImpl
 {
-    using ConfigurationPtr = DeltaLakeMetadata::ConfigurationPtr;
+    using ConfigurationObserverPtr = DeltaLakeMetadata::ConfigurationObserverPtr;
 
     ObjectStoragePtr object_storage;
-    ConfigurationPtr configuration;
+    ConfigurationObserverPtr configuration;
     ContextPtr context;
 
     /**
      * Useful links:
      *  - https://github.com/delta-io/delta/blob/master/PROTOCOL.md#data-files
      */
-     DeltaLakeMetadataImpl(ObjectStoragePtr object_storage_,
+    DeltaLakeMetadataImpl(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_)
-          ConfigurationPtr configuration_,
+        : object_storage(object_storage_), configuration(configuration_), context(context_)
-          ContextPtr context_)
-        : object_storage(object_storage_)
-        , configuration(configuration_)
-        , context(context_)
     {
     }
 
@@ -111,6 +107,7 @@ struct DeltaLakeMetadataImpl
     };
     DeltaLakeMetadata processMetadataFiles()
     {
+        auto configuration_ptr = configuration.lock();
         std::set<String> result_files;
         NamesAndTypesList current_schema;
         DataLakePartitionColumns current_partition_columns;
@@ -122,7 +119,7 @@ struct DeltaLakeMetadataImpl
             while (true)
             {
                 const auto filename = withPadding(++current_version) + metadata_file_suffix;
-                const auto file_path = std::filesystem::path(configuration->getPath()) / deltalake_metadata_directory / filename;
+                const auto file_path = std::filesystem::path(configuration_ptr->getPath()) / deltalake_metadata_directory / filename;
 
                 if (!object_storage->exists(StoredObject(file_path)))
                     break;
@@ -136,7 +133,7 @@ struct DeltaLakeMetadataImpl
         }
         else
         {
-            const auto keys = listFiles(*object_storage, *configuration, deltalake_metadata_directory, metadata_file_suffix);
+            const auto keys = listFiles(*object_storage, *configuration_ptr, deltalake_metadata_directory, metadata_file_suffix);
             for (const String & key : keys)
                 processMetadataFile(key, current_schema, current_partition_columns, result_files);
         }
@@ -246,6 +243,8 @@ struct DeltaLakeMetadataImpl
                 }
             }
 
+            auto configuration_ptr = configuration.lock();
+
             if (object->has("add"))
             {
                 auto add_object = object->get("add").extract<Poco::JSON::Object::Ptr>();
@@ -253,7 +252,7 @@ struct DeltaLakeMetadataImpl
                     throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `add` field");
 
                 auto path = add_object->getValue<String>("path");
-                result.insert(fs::path(configuration->getPath()) / path);
+                result.insert(fs::path(configuration_ptr->getPath()) / path);
 
                 auto filename = fs::path(path).filename().string();
                 auto it = file_partition_columns.find(filename);
@@ -297,7 +296,7 @@ struct DeltaLakeMetadataImpl
                     throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `remove` field");
 
                 auto path = remove_object->getValue<String>("path");
-                result.erase(fs::path(configuration->getPath()) / path);
+                result.erase(fs::path(configuration_ptr->getPath()) / path);
             }
         }
     }
@@ -488,7 +487,9 @@ struct DeltaLakeMetadataImpl
      */
     size_t readLastCheckpointIfExists() const
     {
-        const auto last_checkpoint_file = std::filesystem::path(configuration->getPath()) / deltalake_metadata_directory / "_last_checkpoint";
+        auto configuration_ptr = configuration.lock();
+        const auto last_checkpoint_file
+            = std::filesystem::path(configuration_ptr->getPath()) / deltalake_metadata_directory / "_last_checkpoint";
         if (!object_storage->exists(StoredObject(last_checkpoint_file)))
             return 0;
 
@@ -555,7 +556,11 @@ struct DeltaLakeMetadataImpl
             return 0;
 
         const auto checkpoint_filename = withPadding(version) + ".checkpoint.parquet";
-        const auto checkpoint_path = std::filesystem::path(configuration->getPath()) / deltalake_metadata_directory / checkpoint_filename;
+
+        auto configuration_ptr = configuration.lock();
+
+        const auto checkpoint_path
+            = std::filesystem::path(configuration_ptr->getPath()) / deltalake_metadata_directory / checkpoint_filename;
 
         LOG_TRACE(log, "Using checkpoint file: {}", checkpoint_path.string());
 
@@ -671,7 +676,7 @@ struct DeltaLakeMetadataImpl
             }
 
             LOG_TEST(log, "Adding {}", path);
-            const auto [_, inserted] = result.insert(std::filesystem::path(configuration->getPath()) / path);
+            const auto [_, inserted] = result.insert(std::filesystem::path(configuration_ptr->getPath()) / path);
             if (!inserted)
                 throw Exception(ErrorCodes::INCORRECT_DATA, "File already exists {}", path);
         }
@@ -682,10 +687,7 @@ struct DeltaLakeMetadataImpl
     LoggerPtr log = getLogger("DeltaLakeMetadataParser");
 };
 
-DeltaLakeMetadata::DeltaLakeMetadata(
+DeltaLakeMetadata::DeltaLakeMetadata(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_)
-    ObjectStoragePtr object_storage_,
-    ConfigurationPtr configuration_,
-    ContextPtr context_)
 {
     auto impl = DeltaLakeMetadataImpl(object_storage_, configuration_, context_);
     auto result = impl.processMetadataFiles();

src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h

@@ -1,5 +1,9 @@
 #pragma once
 
+#include "config.h"
+
+#if USE_PARQUET
+
 #include <Interpreters/Context_fwd.h>
 #include <Core/Types.h>
 #include <Storages/ObjectStorage/StorageObjectStorage.h>
@@ -12,13 +16,10 @@ namespace DB
 class DeltaLakeMetadata final : public IDataLakeMetadata
 {
 public:
-    using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr;
+    using ConfigurationObserverPtr = StorageObjectStorage::ConfigurationObserverPtr;
     static constexpr auto name = "DeltaLake";
 
-    DeltaLakeMetadata(
+    DeltaLakeMetadata(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_);
-        ObjectStoragePtr object_storage_,
-        ConfigurationPtr configuration_,
-        ContextPtr context_);
 
     Strings getDataFiles() const override { return data_files; }
 
@@ -36,10 +37,7 @@ public:
             && data_files == deltalake_metadata->data_files;
     }
 
-    static DataLakeMetadataPtr create(
+    static DataLakeMetadataPtr create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context)
-        ObjectStoragePtr object_storage,
-        ConfigurationPtr configuration,
-        ContextPtr local_context)
     {
         return std::make_unique<DeltaLakeMetadata>(object_storage, configuration, local_context);
     }
@@ -52,3 +50,5 @@ private:
 };
 
 }
+
+#endif

src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp

@@ -1,11 +1,10 @@
-#include <Storages/ObjectStorage/DataLakes/HudiMetadata.h>
-#include <Storages/ObjectStorage/DataLakes/Common.h>
 #include <Disks/ObjectStorages/IObjectStorage.h>
-#include <Common/logger_useful.h>
+#include <IO/ReadHelpers.h>
+#include <Storages/ObjectStorage/DataLakes/Common.h>
+#include <Storages/ObjectStorage/DataLakes/HudiMetadata.h>
 #include <base/find_symbols.h>
 #include <Poco/String.h>
-#include "config.h"
+#include <Common/logger_useful.h>
-#include <IO/ReadHelpers.h>
 
 namespace DB
 {
@@ -43,8 +42,9 @@ namespace ErrorCodes
     */
 Strings HudiMetadata::getDataFilesImpl() const
 {
+    auto configuration_ptr = configuration.lock();
     auto log = getLogger("HudiMetadata");
-    const auto keys = listFiles(*object_storage, *configuration, "", Poco::toLower(configuration->format));
+    const auto keys = listFiles(*object_storage, *configuration_ptr, "", Poco::toLower(configuration_ptr->format));
 
     using Partition = std::string;
     using FileID = std::string;
@@ -86,13 +86,8 @@ Strings HudiMetadata::getDataFilesImpl() const
     return result;
 }
 
-HudiMetadata::HudiMetadata(
+HudiMetadata::HudiMetadata(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_)
-    ObjectStoragePtr object_storage_,
+    : WithContext(context_), object_storage(object_storage_), configuration(configuration_)
-    ConfigurationPtr configuration_,
-    ContextPtr context_)
-    : WithContext(context_)
-    , object_storage(object_storage_)
-    , configuration(configuration_)
 {
 }
 

src/Storages/ObjectStorage/DataLakes/HudiMetadata.h

@@ -13,14 +13,11 @@ namespace DB
 class HudiMetadata final : public IDataLakeMetadata, private WithContext
 {
 public:
-    using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr;
+    using ConfigurationObserverPtr = StorageObjectStorage::ConfigurationObserverPtr;
 
     static constexpr auto name = "Hudi";
 
-    HudiMetadata(
+    HudiMetadata(ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, ContextPtr context_);
-        ObjectStoragePtr object_storage_,
-        ConfigurationPtr configuration_,
-        ContextPtr context_);
 
     Strings getDataFiles() const override;
 
@@ -38,17 +35,14 @@ public:
             && data_files == hudi_metadata->data_files;
     }
 
-    static DataLakeMetadataPtr create(
+    static DataLakeMetadataPtr create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context)
-        ObjectStoragePtr object_storage,
-        ConfigurationPtr configuration,
-        ContextPtr local_context)
     {
         return std::make_unique<HudiMetadata>(object_storage, configuration, local_context);
     }
 
 private:
     const ObjectStoragePtr object_storage;
-    const ConfigurationPtr configuration;
+    const ConfigurationObserverPtr configuration;
     mutable Strings data_files;
     std::unordered_map<String, String> column_name_to_physical_name;
     DataLakePartitionColumns partition_columns;

src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h

@@ -1,169 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#if USE_AVRO
-
-#include <Storages/IStorage.h>
-#include <Storages/StorageFactory.h>
-#include <Storages/ObjectStorage/StorageObjectStorage.h>
-#include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
-#include <Storages/ObjectStorage/DataLakes/IcebergMetadata.h>
-#include <Storages/ObjectStorage/DataLakes/HudiMetadata.h>
-#include <Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h>
-#include <Common/logger_useful.h>
-
-
-namespace DB
-{
-
-/// Storage for read-only integration with Apache Iceberg tables in Amazon S3 (see https://iceberg.apache.org/)
-/// Right now it's implemented on top of StorageS3 and right now it doesn't support
-/// many Iceberg features like schema evolution, partitioning, positional and equality deletes.
-template <typename DataLakeMetadata>
-class IStorageDataLake final : public StorageObjectStorage
-{
-public:
-    using Storage = StorageObjectStorage;
-    using ConfigurationPtr = Storage::ConfigurationPtr;
-
-    static StoragePtr create(
-        ConfigurationPtr base_configuration,
-        ContextPtr context,
-        const StorageID & table_id_,
-        const ColumnsDescription & columns_,
-        const ConstraintsDescription & constraints_,
-        const String & comment_,
-        std::optional<FormatSettings> format_settings_,
-        LoadingStrictnessLevel mode)
-    {
-        auto object_storage = base_configuration->createObjectStorage(context, /* is_readonly */true);
-        DataLakeMetadataPtr metadata;
-        NamesAndTypesList schema_from_metadata;
-        const bool use_schema_from_metadata = columns_.empty();
-
-        if (base_configuration->format == "auto")
-            base_configuration->format = "Parquet";
-
-        ConfigurationPtr configuration = base_configuration->clone();
-
-        try
-        {
-            metadata = DataLakeMetadata::create(object_storage, base_configuration, context);
-            configuration->setPaths(metadata->getDataFiles());
-            if (use_schema_from_metadata)
-                schema_from_metadata = metadata->getTableSchema();
-        }
-        catch (...)
-        {
-            if (mode <= LoadingStrictnessLevel::CREATE)
-                throw;
-
-            metadata.reset();
-            configuration->setPaths({});
-            tryLogCurrentException(__PRETTY_FUNCTION__);
-        }
-
-        return std::make_shared<IStorageDataLake<DataLakeMetadata>>(
-            base_configuration, std::move(metadata), configuration, object_storage,
-            context, table_id_,
-            use_schema_from_metadata ? ColumnsDescription(schema_from_metadata) : columns_,
-            constraints_, comment_, format_settings_);
-    }
-
-    String getName() const override { return DataLakeMetadata::name; }
-
-    static ColumnsDescription getTableStructureFromData(
-        ObjectStoragePtr object_storage_,
-        ConfigurationPtr base_configuration,
-        const std::optional<FormatSettings> & format_settings_,
-        ContextPtr local_context)
-    {
-        auto metadata = DataLakeMetadata::create(object_storage_, base_configuration, local_context);
-
-        auto schema_from_metadata = metadata->getTableSchema();
-        if (!schema_from_metadata.empty())
-        {
-            return ColumnsDescription(std::move(schema_from_metadata));
-        }
-
-        ConfigurationPtr configuration = base_configuration->clone();
-        configuration->setPaths(metadata->getDataFiles());
-        std::string sample_path;
-        return Storage::resolveSchemaFromData(object_storage_, configuration, format_settings_, sample_path, local_context);
-    }
-
-    void updateConfiguration(ContextPtr local_context) override
-    {
-        Storage::updateConfiguration(local_context);
-
-        auto new_metadata = DataLakeMetadata::create(Storage::object_storage, base_configuration, local_context);
-        if (current_metadata && *current_metadata == *new_metadata)
-            return;
-
-        current_metadata = std::move(new_metadata);
-        auto updated_configuration = base_configuration->clone();
-        updated_configuration->setPaths(current_metadata->getDataFiles());
-        updated_configuration->setPartitionColumns(current_metadata->getPartitionColumns());
-
-        Storage::configuration = updated_configuration;
-    }
-
-    template <typename... Args>
-    IStorageDataLake(
-        ConfigurationPtr base_configuration_,
-        DataLakeMetadataPtr metadata_,
-        Args &&... args)
-        : Storage(std::forward<Args>(args)...)
-        , base_configuration(base_configuration_)
-        , current_metadata(std::move(metadata_))
-    {
-        if (base_configuration->format == "auto")
-        {
-            base_configuration->format = Storage::configuration->format;
-        }
-
-        if (current_metadata)
-        {
-            const auto & columns = current_metadata->getPartitionColumns();
-            base_configuration->setPartitionColumns(columns);
-            Storage::configuration->setPartitionColumns(columns);
-        }
-    }
-
-private:
-    ConfigurationPtr base_configuration;
-    DataLakeMetadataPtr current_metadata;
-
-    ReadFromFormatInfo prepareReadingFromFormat(
-        const Strings & requested_columns,
-        const StorageSnapshotPtr & storage_snapshot,
-        bool supports_subset_of_columns,
-        ContextPtr local_context) override
-    {
-        auto info = DB::prepareReadingFromFormat(requested_columns, storage_snapshot, local_context, supports_subset_of_columns);
-        if (!current_metadata)
-        {
-            Storage::updateConfiguration(local_context);
-            current_metadata = DataLakeMetadata::create(Storage::object_storage, base_configuration, local_context);
-        }
-        auto column_mapping = current_metadata->getColumnNameToPhysicalNameMapping();
-        if (!column_mapping.empty())
-        {
-            for (const auto & [column_name, physical_name] : column_mapping)
-            {
-                auto & column = info.format_header.getByName(column_name);
-                column.name = physical_name;
-            }
-        }
-        return info;
-    }
-};
-
-using StorageIceberg = IStorageDataLake<IcebergMetadata>;
-using StorageDeltaLake = IStorageDataLake<DeltaLakeMetadata>;
-using StorageHudi = IStorageDataLake<HudiMetadata>;
-
-}
-
-#endif

src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp

@@ -51,7 +51,7 @@ extern const int UNSUPPORTED_METHOD;
 
 IcebergMetadata::IcebergMetadata(
     ObjectStoragePtr object_storage_,
-    ConfigurationPtr configuration_,
+    ConfigurationObserverPtr configuration_,
     DB::ContextPtr context_,
     Int32 metadata_version_,
     Int32 format_version_,
@@ -382,12 +382,12 @@ std::pair<Int32, String> getMetadataFileAndVersion(
 
 }
 
-DataLakeMetadataPtr IcebergMetadata::create(
+DataLakeMetadataPtr
-    ObjectStoragePtr object_storage,
+IcebergMetadata::create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context)
-    ConfigurationPtr configuration,
-    ContextPtr local_context)
 {
-    const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration);
+    auto configuration_ptr = configuration.lock();
+
+    const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration_ptr);
 
     auto log = getLogger("IcebergMetadata");
     LOG_DEBUG(log, "Parse metadata {}", metadata_file_path);
@@ -416,12 +416,13 @@ DataLakeMetadataPtr IcebergMetadata::create(
         if (snapshot->getValue<Int64>("snapshot-id") == current_snapshot_id)
         {
             const auto path = snapshot->getValue<String>("manifest-list");
-            manifest_list_file = std::filesystem::path(configuration->getPath()) / "metadata" / std::filesystem::path(path).filename();
+            manifest_list_file = std::filesystem::path(configuration_ptr->getPath()) / "metadata" / std::filesystem::path(path).filename();
             break;
         }
     }
 
-    return std::make_unique<IcebergMetadata>(object_storage, configuration, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema);
+    return std::make_unique<IcebergMetadata>(
+        object_storage, configuration_ptr, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema);
 }
 
 /**
@@ -451,6 +452,7 @@ DataLakeMetadataPtr IcebergMetadata::create(
  */
 Strings IcebergMetadata::getDataFiles() const
 {
+    auto configuration_ptr = configuration.lock();
     if (!data_files.empty())
         return data_files;
 
@@ -483,7 +485,7 @@ Strings IcebergMetadata::getDataFiles() const
     {
         const auto file_path = col_str->getDataAt(i).toView();
         const auto filename = std::filesystem::path(file_path).filename();
-        manifest_files.emplace_back(std::filesystem::path(configuration->getPath()) / "metadata" / filename);
+        manifest_files.emplace_back(std::filesystem::path(configuration_ptr->getPath()) / "metadata" / filename);
     }
 
     NameSet files;
@@ -618,9 +620,9 @@ Strings IcebergMetadata::getDataFiles() const
 
             const auto status = status_int_column->getInt(i);
             const auto data_path = std::string(file_path_string_column->getDataAt(i).toView());
-            const auto pos = data_path.find(configuration->getPath());
+            const auto pos = data_path.find(configuration_ptr->getPath());
             if (pos == std::string::npos)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected to find {} in data path: {}", configuration->getPath(), data_path);
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected to find {} in data path: {}", configuration_ptr->getPath(), data_path);
 
             const auto file_path = data_path.substr(pos);
 

src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "config.h"
+
 #if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format.
 
 #include <Interpreters/Context_fwd.h>
@@ -61,13 +63,13 @@ namespace DB
 class IcebergMetadata : public IDataLakeMetadata, private WithContext
 {
 public:
-    using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr;
+    using ConfigurationObserverPtr = StorageObjectStorage::ConfigurationObserverPtr;
 
     static constexpr auto name = "Iceberg";
 
     IcebergMetadata(
         ObjectStoragePtr object_storage_,
-        ConfigurationPtr configuration_,
+        ConfigurationObserverPtr configuration_,
         ContextPtr context_,
         Int32 metadata_version_,
         Int32 format_version_,
@@ -92,16 +94,13 @@ public:
         return iceberg_metadata && getVersion() == iceberg_metadata->getVersion();
     }
 
-    static DataLakeMetadataPtr create(
+    static DataLakeMetadataPtr create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context);
-        ObjectStoragePtr object_storage,
-        ConfigurationPtr configuration,
-        ContextPtr local_context);
 
 private:
     size_t getVersion() const { return metadata_version; }
 
     const ObjectStoragePtr object_storage;
-    const ConfigurationPtr configuration;
+    const ConfigurationObserverPtr configuration;
     Int32 metadata_version;
     Int32 format_version;
     String manifest_list_file;

src/Storages/ObjectStorage/DataLakes/registerDataLakeStorages.cpp

@@ -1,154 +0,0 @@
-#include "config.h"
-
-#if USE_AWS_S3
-
-#    include <Storages/ObjectStorage/Azure/Configuration.h>
-#    include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
-#    include <Storages/ObjectStorage/DataLakes/IStorageDataLake.h>
-#    include <Storages/ObjectStorage/DataLakes/IcebergMetadata.h>
-#    include <Storages/ObjectStorage/Local/Configuration.h>
-#    include <Storages/ObjectStorage/S3/Configuration.h>
-
-#if USE_HDFS
-#    include <Storages/ObjectStorage/HDFS/Configuration.h>
-#endif
-
-namespace DB
-{
-
-#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format.
-
-void registerStorageIceberg(StorageFactory & factory)
-{
-    factory.registerStorage(
-        "Iceberg",
-        [&](const StorageFactory::Arguments & args)
-        {
-            auto configuration = std::make_shared<StorageS3Configuration>();
-            StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false);
-
-            return StorageIceberg::create(
-                configuration, args.getContext(), args.table_id, args.columns, args.constraints, args.comment, std::nullopt, args.mode);
-        },
-        {
-            .supports_settings = false,
-            .supports_schema_inference = true,
-            .source_access_type = AccessType::S3,
-        });
-
-    factory.registerStorage(
-        "IcebergS3",
-        [&](const StorageFactory::Arguments & args)
-        {
-            auto configuration = std::make_shared<StorageS3Configuration>();
-            StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false);
-
-            return StorageIceberg::create(
-                configuration, args.getContext(), args.table_id, args.columns, args.constraints, args.comment, std::nullopt, args.mode);
-        },
-        {
-            .supports_settings = false,
-            .supports_schema_inference = true,
-            .source_access_type = AccessType::S3,
-        });
-
-    factory.registerStorage(
-        "IcebergAzure",
-        [&](const StorageFactory::Arguments & args)
-        {
-            auto configuration = std::make_shared<StorageAzureConfiguration>();
-            StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), true);
-
-            return StorageIceberg::create(
-                configuration, args.getContext(), args.table_id, args.columns, args.constraints, args.comment, std::nullopt, args.mode);
-        },
-        {
-            .supports_settings = false,
-            .supports_schema_inference = true,
-            .source_access_type = AccessType::AZURE,
-        });
-
-    factory.registerStorage(
-        "IcebergLocal",
-        [&](const StorageFactory::Arguments & args)
-        {
-            auto configuration = std::make_shared<StorageLocalConfiguration>();
-            StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false);
-
-            return StorageIceberg::create(
-                configuration, args.getContext(), args.table_id, args.columns,
-                args.constraints, args.comment, std::nullopt, args.mode);
-        },
-        {
-            .supports_settings = false,
-            .supports_schema_inference = true,
-            .source_access_type = AccessType::FILE,
-        });
-
-#if USE_HDFS
-    factory.registerStorage(
-        "IcebergHDFS",
-        [&](const StorageFactory::Arguments & args)
-        {
-            auto configuration = std::make_shared<StorageHDFSConfiguration>();
-            StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false);
-
-            return StorageIceberg::create(
-                configuration, args.getContext(), args.table_id, args.columns,
-                args.constraints, args.comment, std::nullopt, args.mode);
-        },
-        {
-            .supports_settings = false,
-            .supports_schema_inference = true,
-            .source_access_type = AccessType::HDFS,
-        });
-#endif
-}
-
-#endif
-
-#if USE_PARQUET
-void registerStorageDeltaLake(StorageFactory & factory)
-{
-    factory.registerStorage(
-        "DeltaLake",
-        [&](const StorageFactory::Arguments & args)
-        {
-            auto configuration = std::make_shared<StorageS3Configuration>();
-            StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false);
-
-            return StorageDeltaLake::create(
-                configuration, args.getContext(), args.table_id, args.columns,
-                args.constraints, args.comment, std::nullopt, args.mode);
-        },
-        {
-            .supports_settings = false,
-            .supports_schema_inference = true,
-            .source_access_type = AccessType::S3,
-        });
-}
-#endif
-
-void registerStorageHudi(StorageFactory & factory)
-{
-    factory.registerStorage(
-        "Hudi",
-        [&](const StorageFactory::Arguments & args)
-        {
-            auto configuration = std::make_shared<StorageS3Configuration>();
-            StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false);
-
-            return StorageHudi::create(
-                configuration, args.getContext(), args.table_id, args.columns,
-                args.constraints, args.comment, std::nullopt, args.mode);
-        },
-        {
-            .supports_settings = false,
-            .supports_schema_inference = true,
-            .source_access_type = AccessType::S3,
-        });
-}
-
-}
-
-#endif

src/Storages/ObjectStorage/StorageObjectStorage.cpp

@@ -14,14 +14,16 @@
 #include <Processors/Executors/PullingPipelineExecutor.h>
 #include <Processors/Transforms/ExtractColumnsTransform.h>
 
-#include <Storages/StorageFactory.h>
 #include <Storages/Cache/SchemaCache.h>
-#include <Storages/VirtualColumnUtils.h>
-#include <Storages/ObjectStorage/Utils.h>
 #include <Storages/NamedCollectionsHelpers.h>
+#include <Storages/ObjectStorage/ReadBufferIterator.h>
 #include <Storages/ObjectStorage/StorageObjectStorageSink.h>
 #include <Storages/ObjectStorage/StorageObjectStorageSource.h>
-#include <Storages/ObjectStorage/ReadBufferIterator.h>
+#include <Storages/ObjectStorage/Utils.h>
+#include <Storages/StorageFactory.h>
+#include <Storages/VirtualColumnUtils.h>
+#include "Databases/LoadingStrictnessLevel.h"
+#include "Storages/ColumnsDescription.h"
 
 
 namespace DB
@@ -76,6 +78,7 @@ StorageObjectStorage::StorageObjectStorage(
     const ConstraintsDescription & constraints_,
     const String & comment,
     std::optional<FormatSettings> format_settings_,
+    LoadingStrictnessLevel mode,
     bool distributed_processing_,
     ASTPtr partition_by_)
     : IStorage(table_id_)
@@ -86,9 +89,25 @@ StorageObjectStorage::StorageObjectStorage(
     , distributed_processing(distributed_processing_)
     , log(getLogger(fmt::format("Storage{}({})", configuration->getEngineName(), table_id_.getFullTableName())))
 {
-    ColumnsDescription columns{columns_};
+    try
+    {
+        configuration->update(object_storage, context);
+    }
+    catch (...)
+    {
+        // If we don't have format or schema yet, we can't ignore failed configuration update, because relevant configuration is crucial for format and schema inference
+        if (mode <= LoadingStrictnessLevel::CREATE || columns_.empty() || (configuration->format == "auto"))
+        {
+            throw;
+        }
+        else
+        {
+            tryLogCurrentException(log);
+        }
+    }
 
     std::string sample_path;
+    ColumnsDescription columns{columns_};
     resolveSchemaAndFormat(columns, configuration->format, object_storage, configuration, format_settings, sample_path, context);
     configuration->check(context);
 
@@ -124,12 +143,11 @@ bool StorageObjectStorage::supportsSubsetOfColumns(const ContextPtr & context) c
     return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(configuration->format, context, format_settings);
 }
 
-void StorageObjectStorage::updateConfiguration(ContextPtr context)
+void StorageObjectStorage::Configuration::update(ObjectStoragePtr object_storage_ptr, ContextPtr context)
 {
-    IObjectStorage::ApplyNewSettingsOptions options{ .allow_client_change = !configuration->isStaticConfiguration() };
+    IObjectStorage::ApplyNewSettingsOptions options{.allow_client_change = !isStaticConfiguration()};
-    object_storage->applyNewSettings(context->getConfigRef(), configuration->getTypeName() + ".", context, options);
+    object_storage_ptr->applyNewSettings(context->getConfigRef(), getTypeName() + ".", context, options);
 }
-
 namespace
 {
 class ReadFromObjectStorageStep : public SourceStepWithFilter
@@ -243,7 +261,8 @@ private:
 };
 }
 
-ReadFromFormatInfo StorageObjectStorage::prepareReadingFromFormat(
+ReadFromFormatInfo StorageObjectStorage::Configuration::prepareReadingFromFormat(
+    ObjectStoragePtr,
     const Strings & requested_columns,
     const StorageSnapshotPtr & storage_snapshot,
     bool supports_subset_of_columns,
@@ -252,6 +271,11 @@ ReadFromFormatInfo StorageObjectStorage::prepareReadingFromFormat(
     return DB::prepareReadingFromFormat(requested_columns, storage_snapshot, local_context, supports_subset_of_columns);
 }
 
+std::optional<ColumnsDescription> StorageObjectStorage::Configuration::tryGetTableStructureFromMetadata() const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method tryGetTableStructureFromMetadata is not implemented for basic configuration");
+}
+
 void StorageObjectStorage::read(
     QueryPlan & query_plan,
     const Names & column_names,
@@ -262,7 +286,7 @@ void StorageObjectStorage::read(
     size_t max_block_size,
     size_t num_streams)
 {
-    updateConfiguration(local_context);
+    configuration->update(object_storage, local_context);
     if (partition_by && configuration->withPartitionWildcard())
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED,
@@ -270,8 +294,8 @@ void StorageObjectStorage::read(
                         getName());
     }
 
-    const auto read_from_format_info = prepareReadingFromFormat(
+    const auto read_from_format_info = configuration->prepareReadingFromFormat(
-        column_names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context);
+        object_storage, column_names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context);
     const bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty())
         && local_context->getSettingsRef()[Setting::optimize_count_from_files];
 
@@ -300,7 +324,7 @@ SinkToStoragePtr StorageObjectStorage::write(
     ContextPtr local_context,
     bool /* async_insert */)
 {
-    updateConfiguration(local_context);
+    configuration->update(object_storage, local_context);
     const auto sample_block = metadata_snapshot->getSampleBlock();
     const auto & settings = configuration->getQuerySettings(local_context);
 
@@ -409,6 +433,16 @@ ColumnsDescription StorageObjectStorage::resolveSchemaFromData(
     std::string & sample_path,
     const ContextPtr & context)
 {
+    if (configuration->isDataLakeConfiguration())
+    {
+        configuration->update(object_storage, context);
+        auto table_structure = configuration->tryGetTableStructureFromMetadata();
+        if (table_structure)
+        {
+            return table_structure.value();
+        }
+    }
+
     ObjectInfos read_keys;
     auto iterator = createReadBufferIterator(object_storage, configuration, format_settings, read_keys, context);
     auto schema = readSchemaFromFormat(configuration->format, format_settings, *iterator, context);
@@ -489,10 +523,17 @@ void StorageObjectStorage::Configuration::initialize(
 
     if (configuration.format == "auto")
     {
-        configuration.format = FormatFactory::instance().tryGetFormatFromFileName(
+        if (configuration.isDataLakeConfiguration())
-            configuration.isArchive()
+        {
-            ? configuration.getPathInArchive()
+            configuration.format = "Parquet";
-            : configuration.getPath()).value_or("auto");
+        }
+        else
+        {
+            configuration.format
+                = FormatFactory::instance()
+                      .tryGetFormatFromFileName(configuration.isArchive() ? configuration.getPathInArchive() : configuration.getPath())
+                      .value_or("auto");
+        }
     }
     else
         FormatFactory::instance().checkFormatName(configuration.format);