Merge remote-tracking branch 'ClickHouse/master' into add-serial-function

.github/PULL_REQUEST_TEMPLATE.md

@@ -46,42 +46,35 @@ At a minimum, the following information should be added (but add more as needed)
 
 **NOTE:** If your merge the PR with modified CI you **MUST KNOW** what you are doing
 **NOTE:** Checked options will be applied if set before CI RunConfig/PrepareRunConfig step
-
-#### Run these jobs only (required builds will be added automatically):
-- [ ] <!---ci_include_integration--> Integration Tests
-- [ ] <!---ci_include_stateless--> Stateless tests
-- [ ] <!---ci_include_stateful--> Stateful tests
-- [ ] <!---ci_include_unit--> Unit tests
-- [ ] <!---ci_include_performance--> Performance tests
-- [ ] <!---ci_include_aarch64--> All with aarch64
-- [ ] <!---ci_include_asan--> All with ASAN
-- [ ] <!---ci_include_tsan--> All with TSAN
-- [ ] <!---ci_include_analyzer--> All with Analyzer
-- [ ] <!---ci_include_azure --> All with Azure
-- [ ] <!---ci_include_KEYWORD--> Add your option here
-
-#### Deny these jobs:
-- [ ] <!---ci_exclude_fast--> Fast test
-- [ ] <!---ci_exclude_integration--> Integration Tests
-- [ ] <!---ci_exclude_stateless--> Stateless tests
-- [ ] <!---ci_exclude_stateful--> Stateful tests
-- [ ] <!---ci_exclude_performance--> Performance tests
-- [ ] <!---ci_exclude_asan--> All with ASAN
-- [ ] <!---ci_exclude_tsan--> All with TSAN
-- [ ] <!---ci_exclude_msan--> All with MSAN
-- [ ] <!---ci_exclude_ubsan--> All with UBSAN
-- [ ] <!---ci_exclude_coverage--> All with Coverage
-- [ ] <!---ci_exclude_aarch64--> All with Aarch64
-
-#### Extra options:
+- [ ] <!---ci_include_integration--> Allow: Integration Tests
+- [ ] <!---ci_include_stateless--> Allow: Stateless tests
+- [ ] <!---ci_include_stateful--> Allow: Stateful tests
+- [ ] <!---ci_include_unit--> Allow: Unit tests
+- [ ] <!---ci_include_performance--> Allow: Performance tests
+- [ ] <!---ci_include_aarch64--> Allow: All with aarch64
+- [ ] <!---ci_include_asan--> Allow: All with ASAN
+- [ ] <!---ci_include_tsan--> Allow: All with TSAN
+- [ ] <!---ci_include_analyzer--> Allow: All with Analyzer
+- [ ] <!---ci_include_azure --> Allow: All with Azure
+- [ ] <!---ci_include_KEYWORD--> Allow: Add your option here
+---
+- [ ] <!---ci_exclude_fast--> Exclude: Fast test
+- [ ] <!---ci_exclude_integration--> Exclude: Integration Tests
+- [ ] <!---ci_exclude_stateless--> Exclude: Stateless tests
+- [ ] <!---ci_exclude_stateful--> Exclude: Stateful tests
+- [ ] <!---ci_exclude_performance--> Exclude: Performance tests
+- [ ] <!---ci_exclude_asan--> Exclude: All with ASAN
+- [ ] <!---ci_exclude_tsan--> Exclude: All with TSAN
+- [ ] <!---ci_exclude_msan--> Exclude: All with MSAN
+- [ ] <!---ci_exclude_ubsan--> Exclude: All with UBSAN
+- [ ] <!---ci_exclude_coverage--> Exclude: All with Coverage
+- [ ] <!---ci_exclude_aarch64--> Exclude: All with Aarch64
+---
 - [ ] <!---do_not_test--> do not test (only style check)
 - [ ] <!---no_merge_commit--> disable merge-commit (no merge from master before tests)
 - [ ] <!---no_ci_cache--> disable CI cache (job reuse)
-
-#### Only specified batches in multi-batch jobs:
-- [ ] <!---batch_0--> 1
-- [ ] <!---batch_1--> 2
-- [ ] <!---batch_2--> 3
-- [ ] <!---batch_3--> 4
-
+- [ ] <!---batch_0--> allow: batch 1 for multi-batch jobs
+- [ ] <!---batch_1--> allow: batch 2
+- [ ] <!---batch_2--> allow: batch 3
+- [ ] <!---batch_3_4_5--> allow: batch 4, 5 and 6
 </details>

.github/workflows/master.yml

@@ -27,15 +27,16 @@ jobs:
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 sync_pr.py --merge || :
-      - name: Python unit tests
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          echo "Testing the main ci directory"
-          python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
+# Runs in MQ:
+#      - name: Python unit tests
+#        run: |
+#          cd "$GITHUB_WORKSPACE/tests/ci"
+#          echo "Testing the main ci directory"
+#          python3 -m unittest discover -s . -p 'test_*.py'
+#          for dir in *_lambda/; do
+#            echo "Testing $dir"
+#            python3 -m unittest discover -s "$dir" -p 'test_*.py'
+#          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
@@ -162,7 +163,7 @@ jobs:
           python3 mark_release_ready.py
 
   FinishCheck:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !cancelled() }}
     needs: [RunConfig, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:

.github/workflows/pull_request.yml

@@ -33,9 +33,12 @@ jobs:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get a version
           filter: tree:0
-      - name: Cancel Sync PR workflow
+      - name: Cancel previous Sync PR workflow
         run: |
           python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
+      - name: Set pending Sync status
+        run: |
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --set-pending-status
       - name: Labels check
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
@@ -177,7 +180,7 @@ jobs:
   ################################# Stage Final #################################
   #
   FinishCheck:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !cancelled() }}
     needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:

contrib/aws

@@ -1 +1 @@
-Subproject commit 2e12d7c6dafa81311ee3d73ac6a178550ffa75be
+Subproject commit eb96e740453ae27afa1f367ba19f99bdcb38484d

docs/en/engines/table-engines/integrations/s3queue.md

@@ -202,8 +202,7 @@ Example:
   CREATE TABLE s3queue_engine_table (name String, value UInt32)
     ENGINE=S3Queue('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/*', 'CSV', 'gzip')
     SETTINGS
-        mode = 'unordered',
-        keeper_path = '/clickhouse/s3queue/';
+        mode = 'unordered';
 
   CREATE TABLE stats (name String, value UInt32)
     ENGINE = MergeTree() ORDER BY name;

docs/en/operations/settings/settings.md

@@ -3665,6 +3665,26 @@ Possible values:
 
 Default value: `0`.
 
+## s3_ignore_file_doesnt_exist {#s3_ignore_file_doesnt_exist}
+
+Ignore absence of file if it does not exist when reading certain keys.
+
+Possible values:
+- 1 — `SELECT` returns empty result.
+- 0 — `SELECT` throws an exception.
+
+Default value: `0`.
+
+## s3_validate_request_settings {#s3_validate_request_settings}
+
+Enables s3 request settings validation.
+
+Possible values:
+- 1 — validate settings.
+- 0 — do not validate settings.
+
+Default value: `1`.
+
 ## hdfs_truncate_on_insert {#hdfs_truncate_on_insert}
 
 Enables or disables truncation before an insert in hdfs engine tables. If disabled, an exception will be thrown on an attempt to insert if a file in HDFS already exists.
@@ -3697,6 +3717,56 @@ Possible values:
 
 Default value: `0`.
 
+## hdfs_throw_on_zero_files_match {#hdfs_throw_on_zero_files_match}
+
+Throw an error if matched zero files according to glob expansion rules.
+
+Possible values:
+- 1 — `SELECT` throws an exception.
+- 0 — `SELECT` returns empty result.
+
+Default value: `0`.
+
+## hdfs_ignore_file_doesnt_exist {#hdfs_ignore_file_doesnt_exist}
+
+Ignore absence of file if it does not exist when reading certain keys.
+
+Possible values:
+- 1 — `SELECT` returns empty result.
+- 0 — `SELECT` throws an exception.
+
+Default value: `0`.
+
+## azure_throw_on_zero_files_match {#azure_throw_on_zero_files_match}
+
+Throw an error if matched zero files according to glob expansion rules.
+
+Possible values:
+- 1 — `SELECT` throws an exception.
+- 0 — `SELECT` returns empty result.
+
+Default value: `0`.
+
+## azure_ignore_file_doesnt_exist {#azure_ignore_file_doesnt_exist}
+
+Ignore absence of file if it does not exist when reading certain keys.
+
+Possible values:
+- 1 — `SELECT` returns empty result.
+- 0 — `SELECT` throws an exception.
+
+Default value: `0`.
+
+## azure_skip_empty_files {#azure_skip_empty_files}
+
+Enables or disables skipping empty files in S3 engine.
+
+Possible values:
+- 0 — `SELECT` throws an exception if empty file is not compatible with requested format.
+- 1 — `SELECT` returns empty result for empty file.
+
+Default value: `0`.
+
 ## engine_url_skip_empty_files {#engine_url_skip_empty_files}
 
 Enables or disables skipping empty files in [URL](../../engines/table-engines/special/url.md) engine tables.
@@ -5468,3 +5538,15 @@ Defines how MySQL types are converted to corresponding ClickHouse types. A comma
 - `datetime64`: convert `DATETIME` and `TIMESTAMP` types to `DateTime64` instead of `DateTime` when precision is not `0`.
 - `date2Date32`: convert `DATE` to `Date32` instead of `Date`. Takes precedence over `date2String`.
 - `date2String`: convert `DATE` to `String` instead of `Date`. Overridden by `datetime64`.
+
+## cross_join_min_rows_to_compress
+
+Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.
+
+Default value: `10000000`.
+
+## cross_join_min_bytes_to_compress
+
+Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.
+
+Default value: `1GiB`.

src/Analyzer/ValidationUtils.cpp

@@ -331,6 +331,9 @@ void validateAggregates(const QueryTreeNodePtr & query_node, AggregatesValidatio
         if (query_node_typed.hasOrderBy())
             validate_group_by_columns_visitor.visit(query_node_typed.getOrderByNode());
 
+        if (query_node_typed.hasInterpolate())
+            validate_group_by_columns_visitor.visit(query_node_typed.getInterpolate());
+
         validate_group_by_columns_visitor.visit(query_node_typed.getProjectionNode());
     }
 

src/Backups/BackupIO_AzureBlobStorage.cpp

@@ -6,7 +6,6 @@
 #include <Interpreters/Context.h>
 #include <IO/SharedThreadPools.h>
 #include <IO/HTTPHeaderEntries.h>
-#include <Storages/StorageAzureBlobCluster.h>
 #include <Disks/IO/ReadBufferFromAzureBlobStorage.h>
 #include <Disks/IO/WriteBufferFromAzureBlobStorage.h>
 #include <IO/AzureBlobStorage/copyAzureBlobStorageFile.h>
@@ -30,7 +29,7 @@ namespace ErrorCodes
 }
 
 BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage(
-    StorageAzureBlob::Configuration configuration_,
+    const StorageAzureConfiguration & configuration_,
     bool allow_azure_native_copy,
     const ReadSettings & read_settings_,
     const WriteSettings & write_settings_,
@@ -39,15 +38,14 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage(
     , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.getConnectionURL().toString(), false, false}
     , configuration(configuration_)
 {
-    auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false);
+    auto client_ptr = configuration.createClient(/* is_readonly */false, /* attempt_to_create_container */true);
     client_ptr->SetClickhouseOptions(Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=true});
 
-    object_storage = std::make_unique<AzureObjectStorage>(
-        "BackupReaderAzureBlobStorage",
-        std::move(client_ptr),
-        StorageAzureBlob::createSettings(context_),
-        configuration.container,
-        configuration.getConnectionURL().toString());
+    object_storage = std::make_unique<AzureObjectStorage>("BackupReaderAzureBlobStorage",
+                                                          std::move(client_ptr),
+                                                          configuration.createSettings(context_),
+                                                          configuration_.container,
+                                                          configuration.getConnectionURL().toString());
 
     client = object_storage->getAzureBlobStorageClient();
     auto settings_copy = *object_storage->getSettings();
@@ -121,7 +119,7 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup,
 
 
 BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage(
-    StorageAzureBlob::Configuration configuration_,
+    const StorageAzureConfiguration & configuration_,
     bool allow_azure_native_copy,
     const ReadSettings & read_settings_,
     const WriteSettings & write_settings_,
@@ -131,13 +129,13 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage(
     , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.getConnectionURL().toString(), false, false}
     , configuration(configuration_)
 {
-    auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false, attempt_to_create_container);
+    auto client_ptr = configuration.createClient(/* is_readonly */false, attempt_to_create_container);
     client_ptr->SetClickhouseOptions(Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=true});
 
     object_storage = std::make_unique<AzureObjectStorage>("BackupWriterAzureBlobStorage",
                                                           std::move(client_ptr),
-                                                          StorageAzureBlob::createSettings(context_),
-                                                          configuration_.container,
+                                                          configuration.createSettings(context_),
+                                                          configuration.container,
                                                           configuration_.getConnectionURL().toString());
     client = object_storage->getAzureBlobStorageClient();
     auto settings_copy = *object_storage->getSettings();
@@ -145,8 +143,13 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage(
     settings = std::make_unique<const AzureObjectStorageSettings>(settings_copy);
 }
 
-void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
-                                      bool copy_encrypted, UInt64 start_pos, UInt64 length)
+void BackupWriterAzureBlobStorage::copyFileFromDisk(
+    const String & path_in_backup,
+    DiskPtr src_disk,
+    const String & src_path,
+    bool copy_encrypted,
+    UInt64 start_pos,
+    UInt64 length)
 {
     /// Use the native copy as a more optimal way to copy a file from AzureBlobStorage to AzureBlobStorage if it's possible.
     auto source_data_source_description = src_disk->getDataSourceDescription();
@@ -196,10 +199,16 @@ void BackupWriterAzureBlobStorage::copyFile(const String & destination, const St
        threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupWRAzure"));
 }
 
-void BackupWriterAzureBlobStorage::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length)
+void BackupWriterAzureBlobStorage::copyDataToFile(
+    const String & path_in_backup,
+    const CreateReadBufferFunction & create_read_buffer,
+    UInt64 start_pos,
+    UInt64 length)
 {
-    copyDataToAzureBlobStorageFile(create_read_buffer, start_pos, length, client, configuration.container, fs::path(configuration.blob_path) / path_in_backup, settings,
-                     threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupWRAzure"));
+    copyDataToAzureBlobStorageFile(
+        create_read_buffer, start_pos, length, client, configuration.container,
+        fs::path(configuration.blob_path) / path_in_backup, settings,
+        threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupWRAzure"));
 }
 
 BackupWriterAzureBlobStorage::~BackupWriterAzureBlobStorage() = default;
@@ -217,7 +226,7 @@ UInt64 BackupWriterAzureBlobStorage::getFileSize(const String & file_name)
     object_storage->listObjects(key,children,/*max_keys*/0);
     if (children.empty())
         throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Object must exist");
-    return children[0].metadata.size_bytes;
+    return children[0]->metadata->size_bytes;
 }
 
 std::unique_ptr<ReadBuffer> BackupWriterAzureBlobStorage::readFile(const String & file_name, size_t /*expected_file_size*/)

src/Backups/BackupIO_AzureBlobStorage.h

@@ -5,8 +5,8 @@
 #if USE_AZURE_BLOB_STORAGE
 #include <Backups/BackupIO_Default.h>
 #include <Disks/DiskType.h>
-#include <Storages/StorageAzureBlobCluster.h>
 #include <Interpreters/Context_fwd.h>
+#include <Storages/ObjectStorage/Azure/Configuration.h>
 
 
 namespace DB
@@ -17,24 +17,30 @@ class BackupReaderAzureBlobStorage : public BackupReaderDefault
 {
 public:
     BackupReaderAzureBlobStorage(
-        StorageAzureBlob::Configuration configuration_,
+        const StorageAzureConfiguration & configuration_,
         bool allow_azure_native_copy,
         const ReadSettings & read_settings_,
         const WriteSettings & write_settings_,
         const ContextPtr & context_);
+
     ~BackupReaderAzureBlobStorage() override;
 
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
     std::unique_ptr<SeekableReadBuffer> readFile(const String & file_name) override;
 
-    void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
-                        DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override;
+    void copyFileToDisk(
+        const String & path_in_backup,
+        size_t file_size,
+        bool encrypted_in_backup,
+        DiskPtr destination_disk,
+        const String & destination_path,
+        WriteMode write_mode) override;
 
 private:
     const DataSourceDescription data_source_description;
     std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> client;
-    StorageAzureBlob::Configuration configuration;
+    StorageAzureConfiguration configuration;
     std::unique_ptr<AzureObjectStorage> object_storage;
     std::shared_ptr<const AzureObjectStorageSettings> settings;
 };
@@ -43,21 +49,32 @@ class BackupWriterAzureBlobStorage : public BackupWriterDefault
 {
 public:
     BackupWriterAzureBlobStorage(
-        StorageAzureBlob::Configuration configuration_,
+        const StorageAzureConfiguration & configuration_,
         bool allow_azure_native_copy,
         const ReadSettings & read_settings_,
         const WriteSettings & write_settings_,
         const ContextPtr & context_,
         bool attempt_to_create_container);
+
     ~BackupWriterAzureBlobStorage() override;
 
     bool fileExists(const String & file_name) override;
     UInt64 getFileSize(const String & file_name) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
 
-    void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) override;
-    void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
-                          bool copy_encrypted, UInt64 start_pos, UInt64 length) override;
+    void copyDataToFile(
+        const String & path_in_backup,
+        const CreateReadBufferFunction & create_read_buffer,
+        UInt64 start_pos,
+        UInt64 length) override;
+
+    void copyFileFromDisk(
+        const String & path_in_backup,
+        DiskPtr src_disk,
+        const String & src_path,
+        bool copy_encrypted,
+        UInt64 start_pos,
+        UInt64 length) override;
 
     void copyFile(const String & destination, const String & source, size_t size) override;
 
@@ -67,9 +84,10 @@ public:
 private:
     std::unique_ptr<ReadBuffer> readFile(const String & file_name, size_t expected_file_size) override;
     void removeFilesBatch(const Strings & file_names);
+
     const DataSourceDescription data_source_description;
     std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> client;
-    StorageAzureBlob::Configuration configuration;
+    StorageAzureConfiguration configuration;
     std::unique_ptr<AzureObjectStorage> object_storage;
     std::shared_ptr<const AzureObjectStorageSettings> settings;
 };

src/Backups/BackupIO_S3.cpp

@@ -131,10 +131,10 @@ BackupReaderS3::BackupReaderS3(
     : BackupReaderDefault(read_settings_, write_settings_, getLogger("BackupReaderS3"))
     , s3_uri(s3_uri_)
     , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::S3, MetadataStorageType::None, s3_uri.endpoint, false, false}
-    , s3_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString(), context_->getUserName(), /*ignore_user=*/is_internal_backup))
+    , s3_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString(), context_->getUserName(), /*ignore_user=*/is_internal_backup).value_or(S3Settings{}))
 {
     auto & request_settings = s3_settings.request_settings;
-    request_settings.updateFromSettings(context_->getSettingsRef());
+    request_settings.updateFromSettingsIfChanged(context_->getSettingsRef());
     request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint
     request_settings.allow_native_copy = allow_s3_native_copy;
     client = makeS3Client(s3_uri_, access_key_id_, secret_access_key_, s3_settings, context_);
@@ -222,10 +222,10 @@ BackupWriterS3::BackupWriterS3(
     : BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterS3"))
     , s3_uri(s3_uri_)
     , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::S3, MetadataStorageType::None, s3_uri.endpoint, false, false}
-    , s3_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString(), context_->getUserName(), /*ignore_user=*/is_internal_backup))
+    , s3_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString(), context_->getUserName(), /*ignore_user=*/is_internal_backup).value_or(S3Settings{}))
 {
     auto & request_settings = s3_settings.request_settings;
-    request_settings.updateFromSettings(context_->getSettingsRef());
+    request_settings.updateFromSettingsIfChanged(context_->getSettingsRef());
     request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint
     request_settings.allow_native_copy = allow_s3_native_copy;
     request_settings.setStorageClassName(storage_class_name);

src/Backups/registerBackupEngineAzureBlobStorage.cpp

@@ -5,11 +5,11 @@
 
 #if USE_AZURE_BLOB_STORAGE
 #include <Backups/BackupIO_AzureBlobStorage.h>
-#include <Storages/StorageAzureBlob.h>
 #include <Backups/BackupImpl.h>
 #include <IO/Archives/hasRegisteredArchiveFileExtension.h>
 #include <Interpreters/Context.h>
 #include <Poco/Util/AbstractConfiguration.h>
+#include <Storages/ObjectStorage/Azure/Configuration.h>
 #include <filesystem>
 #endif
 
@@ -49,7 +49,7 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
         const String & id_arg = params.backup_info.id_arg;
         const auto & args = params.backup_info.args;
 
-        StorageAzureBlob::Configuration configuration;
+        StorageAzureConfiguration configuration;
 
         if (!id_arg.empty())
         {
@@ -81,10 +81,11 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
             }
 
             if (args.size() > 1)
-                throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Backup AzureBlobStorage requires 1 or 2 arguments: named_collection, [filename]");
+                throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                                "Backup AzureBlobStorage requires 1 or 2 arguments: named_collection, [filename]");
 
             if (args.size() == 1)
-                configuration.blob_path = args[0].safeGet<String>();
+                configuration.setPath(args[0].safeGet<String>());
 
         }
         else
@@ -116,12 +117,16 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
         }
 
         BackupImpl::ArchiveParams archive_params;
-        if (hasRegisteredArchiveFileExtension(configuration.blob_path))
+        if (hasRegisteredArchiveFileExtension(configuration.getPath()))
         {
             if (params.is_internal_backup)
                 throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Using archives with backups on clusters is disabled");
 
-            archive_params.archive_name = removeFileNameFromURL(configuration.blob_path);
+            auto path = configuration.getPath();
+            auto filename = removeFileNameFromURL(path);
+            configuration.setPath(path);
+
+            archive_params.archive_name = filename;
             archive_params.compression_method = params.compression_method;
             archive_params.compression_level = params.compression_level;
             archive_params.password = params.password;

src/CMakeLists.txt

@@ -115,8 +115,11 @@ if (TARGET ch_contrib::nats_io)
     add_headers_and_sources(dbms Storages/NATS)
 endif()
 
-add_headers_and_sources(dbms Storages/DataLakes)
-add_headers_and_sources(dbms Storages/DataLakes/Iceberg)
+add_headers_and_sources(dbms Storages/ObjectStorage)
+add_headers_and_sources(dbms Storages/ObjectStorage/Azure)
+add_headers_and_sources(dbms Storages/ObjectStorage/S3)
+add_headers_and_sources(dbms Storages/ObjectStorage/HDFS)
+add_headers_and_sources(dbms Storages/ObjectStorage/DataLakes)
 add_headers_and_sources(dbms Common/NamedCollections)
 
 if (TARGET ch_contrib::amqp_cpp)
@@ -144,7 +147,6 @@ if (TARGET ch_contrib::azure_sdk)
 endif()
 
 if (TARGET ch_contrib::hdfs)
-    add_headers_and_sources(dbms Storages/HDFS)
     add_headers_and_sources(dbms Disks/ObjectStorages/HDFS)
 endif()
 

src/Common/CurrentMetrics.cpp

@@ -168,6 +168,9 @@
     M(ObjectStorageS3Threads, "Number of threads in the S3ObjectStorage thread pool.") \
     M(ObjectStorageS3ThreadsActive, "Number of threads in the S3ObjectStorage thread pool running a task.") \
     M(ObjectStorageS3ThreadsScheduled, "Number of queued or active jobs in the S3ObjectStorage thread pool.") \
+    M(StorageObjectStorageThreads, "Number of threads in the remote table engines thread pools.") \
+    M(StorageObjectStorageThreadsActive, "Number of threads in the remote table engines thread pool running a task.") \
+    M(StorageObjectStorageThreadsScheduled, "Number of queued or active jobs in remote table engines thread pool.") \
     M(ObjectStorageAzureThreads, "Number of threads in the AzureObjectStorage thread pool.") \
     M(ObjectStorageAzureThreadsActive, "Number of threads in the AzureObjectStorage thread pool running a task.") \
     M(ObjectStorageAzureThreadsScheduled, "Number of queued or active jobs in the AzureObjectStorage thread pool.") \

src/Common/ZooKeeper/ZooKeeperImpl.cpp

@@ -1259,11 +1259,13 @@ void ZooKeeper::initFeatureFlags()
 
 void ZooKeeper::executeGenericRequest(
     const ZooKeeperRequestPtr & request,
-    ResponseCallback callback)
+    ResponseCallback callback,
+    WatchCallbackPtr watch)
 {
     RequestInfo request_info;
     request_info.request = request;
     request_info.callback = callback;
+    request_info.watch = watch;
 
     pushRequest(std::move(request_info));
 }

src/Common/ZooKeeper/ZooKeeperImpl.h

@@ -139,7 +139,8 @@ public:
 
     void executeGenericRequest(
         const ZooKeeperRequestPtr & request,
-        ResponseCallback callback);
+        ResponseCallback callback,
+        WatchCallbackPtr watch = nullptr);
 
     /// See the documentation about semantics of these methods in IKeeper class.
 

src/Coordination/Standalone/Context.cpp

@@ -5,6 +5,7 @@
 #include <Common/ThreadPool.h>
 #include <Common/callOnce.h>
 #include <Disks/IO/IOUringReader.h>
+#include <Storages/StorageS3Settings.h>
 #include <Disks/IO/getIOUringReader.h>
 
 #include <Core/ServerSettings.h>
@@ -145,9 +146,10 @@ struct ContextSharedPart : boost::noncopyable
     mutable ThrottlerPtr local_read_throttler;              /// A server-wide throttler for local IO reads
     mutable ThrottlerPtr local_write_throttler;             /// A server-wide throttler for local IO writes
 
+    std::optional<StorageS3Settings> storage_s3_settings TSA_GUARDED_BY(mutex);   /// Settings of S3 storage
+
     mutable std::mutex keeper_dispatcher_mutex;
     mutable std::shared_ptr<KeeperDispatcher> keeper_dispatcher TSA_GUARDED_BY(keeper_dispatcher_mutex);
-
 };
 
 ContextData::ContextData() = default;
@@ -453,6 +455,19 @@ std::shared_ptr<zkutil::ZooKeeper> Context::getZooKeeper() const
     throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Cannot connect to ZooKeeper from Keeper");
 }
 
+const StorageS3Settings & Context::getStorageS3Settings() const
+{
+    std::lock_guard lock(shared->mutex);
+
+    if (!shared->storage_s3_settings)
+    {
+        const auto & config = shared->config ? *shared->config : Poco::Util::Application::instance().config();
+        shared->storage_s3_settings.emplace().loadFromConfig("s3", config, getSettingsRef());
+    }
+
+    return *shared->storage_s3_settings;
+}
+
 const ServerSettings & Context::getServerSettings() const
 {
     return shared->server_settings;

src/Coordination/Standalone/Context.h

@@ -37,6 +37,7 @@ class FilesystemCacheLog;
 class FilesystemReadPrefetchesLog;
 class BlobStorageLog;
 class IOUringReader;
+class StorageS3Settings;
 
 /// A small class which owns ContextShared.
 /// We don't use something like unique_ptr directly to allow ContextShared type to be incomplete.
@@ -162,6 +163,10 @@ public:
 
     zkutil::ZooKeeperPtr getZooKeeper() const;
 
+    const StorageS3Settings & getStorageS3Settings() const;
+
+    const String & getUserName() const { static std::string user; return user; }
+
     const ServerSettings & getServerSettings() const;
 
     bool hasTraceCollector() const;

src/Core/Settings.h

@@ -116,6 +116,12 @@ class IColumn;
     M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \
     M(Bool, azure_allow_parallel_part_upload, true, "Use multiple threads for azure multipart upload.", 0) \
     M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \
+    M(Bool, hdfs_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \
+    M(Bool, azure_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \
+    M(Bool, s3_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in S3 table engine", 0) \
+    M(Bool, hdfs_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in HDFS table engine", 0) \
+    M(Bool, azure_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in AzureBlobStorage table engine", 0) \
+    M(Bool, s3_validate_request_settings, true, "Validate S3 request settings", 0) \
     M(Bool, s3_disable_checksum, false, "Do not calculate a checksum when sending a file to S3. This speeds up writes by avoiding excessive processing passes on a file. It is mostly safe as the data of MergeTree tables is checksummed by ClickHouse anyway, and when S3 is accessed with HTTPS, the TLS layer already provides integrity while transferring through the network. While additional checksums on S3 give defense in depth.", 0) \
     M(UInt64, s3_retry_attempts, 100, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \
     M(UInt64, s3_request_timeout_ms, 30000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \
@@ -128,6 +134,7 @@ class IColumn;
     M(Bool, hdfs_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables", 0) \
     M(Bool, hdfs_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in hdfs engine tables", 0) \
     M(Bool, hdfs_skip_empty_files, false, "Allow to skip empty files in hdfs table engine", 0) \
+    M(Bool, azure_skip_empty_files, false, "Allow to skip empty files in azure table engine", 0) \
     M(UInt64, hsts_max_age, 0, "Expired time for hsts. 0 means disable HSTS.", 0) \
     M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \
     M(Bool, use_uncompressed_cache, false, "Whether to use the cache of uncompressed blocks.", 0) \

src/Core/SettingsChangesHistory.h

@@ -85,12 +85,20 @@ namespace SettingsChangesHistory
 /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972)
 static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> settings_changes_history =
 {
+    {"24.6", {{"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
+              {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
+              {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
+              {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
+              {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
+              {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
+              {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
+              }},
     {"24.5", {{"allow_deprecated_functions", true, false, "Allow usage of deprecated functions"},
               {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
               {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
               {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
-              {"cross_join_min_rows_to_compress", 0, 10000000, "A new setting."},
-              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "A new setting."},
+              {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
+              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
               {"http_max_chunk_size", 0, 0, "Internal limitation"},
               {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
               {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},

src/Databases/DatabaseHDFS.cpp

@@ -11,7 +11,7 @@
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/ParserCreateQuery.h>
-#include <Storages/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
 #include <Storages/IStorage.h>
 #include <TableFunctions/TableFunctionFactory.h>
 #include <Common/re2.h>

src/Disks/IO/WriteBufferFromAzureBlobStorage.h

@@ -13,7 +13,7 @@
 #include <azure/core/io/body_stream.hpp>
 #include <Common/ThreadPoolTaskTracker.h>
 #include <Common/BufferAllocationPolicy.h>
-#include <Storages/StorageAzureBlob.h>
+#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
 
 namespace Poco
 {

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp

@@ -79,14 +79,14 @@ private:
 
         for (const auto & blob : blobs_list)
         {
-            batch.emplace_back(
+            batch.emplace_back(std::make_shared<RelativePathWithMetadata>(
                 blob.Name,
                 ObjectMetadata{
                     static_cast<uint64_t>(blob.BlobSize),
                     Poco::Timestamp::fromEpochTime(
                         std::chrono::duration_cast<std::chrono::seconds>(
                             static_cast<std::chrono::system_clock::time_point>(blob.Details.LastModified).time_since_epoch()).count()),
-                    {}});
+                    {}}));
         }
 
         if (!blob_list_response.NextPageToken.HasValue() || blob_list_response.NextPageToken.Value().empty())
@@ -148,15 +148,15 @@ bool AzureObjectStorage::exists(const StoredObject & object) const
     return false;
 }
 
-ObjectStorageIteratorPtr AzureObjectStorage::iterate(const std::string & path_prefix) const
+ObjectStorageIteratorPtr AzureObjectStorage::iterate(const std::string & path_prefix, size_t max_keys) const
 {
     auto settings_ptr = settings.get();
     auto client_ptr = client.get();
 
-    return std::make_shared<AzureIteratorAsync>(path_prefix, client_ptr, settings_ptr->list_object_keys_size);
+    return std::make_shared<AzureIteratorAsync>(path_prefix, client_ptr, max_keys);
 }
 
-void AzureObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const
+void AzureObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const
 {
     auto client_ptr = client.get();
 
@@ -179,19 +179,19 @@ void AzureObjectStorage::listObjects(const std::string & path, RelativePathsWith
 
         for (const auto & blob : blobs_list)
         {
-            children.emplace_back(
+            children.emplace_back(std::make_shared<RelativePathWithMetadata>(
                 blob.Name,
                 ObjectMetadata{
                     static_cast<uint64_t>(blob.BlobSize),
                     Poco::Timestamp::fromEpochTime(
                         std::chrono::duration_cast<std::chrono::seconds>(
                             static_cast<std::chrono::system_clock::time_point>(blob.Details.LastModified).time_since_epoch()).count()),
-                    {}});
+                    {}}));
         }
 
         if (max_keys)
         {
-            int keys_left = max_keys - static_cast<int>(children.size());
+            size_t keys_left = max_keys - children.size();
             if (keys_left <= 0)
                 break;
             options.PageSizeHint = keys_left;
@@ -346,10 +346,11 @@ void AzureObjectStorage::removeObjectsIfExist(const StoredObjects & objects)
 {
     auto client_ptr = client.get();
     for (const auto & object : objects)
+    {
         removeObjectImpl(object, client_ptr, true);
+    }
 }
 
-
 ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path) const
 {
     auto client_ptr = client.get();
@@ -366,9 +367,9 @@ ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path) c
     {
         result.attributes.emplace();
         for (const auto & [key, value] : properties.Metadata)
-            (*result.attributes)[key] = value;
+            result.attributes[key] = value;
     }
-    result.last_modified.emplace(static_cast<std::chrono::system_clock::time_point>(properties.LastModified).time_since_epoch().count());
+    result.last_modified = static_cast<std::chrono::system_clock::time_point>(properties.LastModified).time_since_epoch().count();
     return result;
 }
 
@@ -397,7 +398,9 @@ void AzureObjectStorage::copyObject( /// NOLINT
     dest_blob_client.CopyFromUri(source_blob_client.GetUrl(), copy_options);
 }
 
-void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
+void AzureObjectStorage::applyNewSettings(
+    const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
+    ContextPtr context, const ApplyNewSettingsOptions &)
 {
     auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context);
     settings.set(std::move(new_settings));

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h

@@ -85,9 +85,9 @@ public:
         const String & object_namespace_,
         const String & description_);
 
-    void listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const override;
+    void listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const override;
 
-    ObjectStorageIteratorPtr iterate(const std::string & path_prefix) const override;
+    ObjectStorageIteratorPtr iterate(const std::string & path_prefix, size_t max_keys) const override;
 
     std::string getName() const override { return "AzureObjectStorage"; }
 
@@ -144,7 +144,8 @@ public:
     void applyNewSettings(
         const Poco::Util::AbstractConfiguration & config,
         const std::string & config_prefix,
-        ContextPtr context) override;
+        ContextPtr context,
+        const ApplyNewSettingsOptions & options) override;
 
     String getObjectsNamespace() const override { return object_namespace ; }
 

src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp

@@ -176,7 +176,7 @@ std::unique_ptr<IObjectStorage> CachedObjectStorage::cloneObjectStorage(
     return object_storage->cloneObjectStorage(new_namespace, config, config_prefix, context);
 }
 
-void CachedObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const
+void CachedObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const
 {
     object_storage->listObjects(path, children, max_keys);
 }
@@ -192,9 +192,10 @@ void CachedObjectStorage::shutdown()
 }
 
 void CachedObjectStorage::applyNewSettings(
-    const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
+    const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
+    ContextPtr context, const ApplyNewSettingsOptions & options)
 {
-    object_storage->applyNewSettings(config, config_prefix, context);
+    object_storage->applyNewSettings(config, config_prefix, context, options);
 }
 
 String CachedObjectStorage::getObjectsNamespace() const

src/Disks/ObjectStorages/Cached/CachedObjectStorage.h

@@ -80,7 +80,7 @@ public:
         const std::string & config_prefix,
         ContextPtr context) override;
 
-    void listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const override;
+    void listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const override;
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 
@@ -91,7 +91,8 @@ public:
     void applyNewSettings(
         const Poco::Util::AbstractConfiguration & config,
         const std::string & config_prefix,
-        ContextPtr context) override;
+        ContextPtr context,
+        const ApplyNewSettingsOptions & options) override;
 
     String getObjectsNamespace() const override;
 

src/Disks/ObjectStorages/DiskObjectStorage.cpp

@@ -544,7 +544,7 @@ void DiskObjectStorage::applyNewSettings(
 {
     /// FIXME we cannot use config_prefix that was passed through arguments because the disk may be wrapped with cache and we need another name
     const auto config_prefix = "storage_configuration.disks." + name;
-    object_storage->applyNewSettings(config, config_prefix, context_);
+    object_storage->applyNewSettings(config, config_prefix, context_, IObjectStorage::ApplyNewSettingsOptions{ .allow_client_change = true });
 
     {
         std::unique_lock lock(resource_mutex);

src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp

@@ -364,18 +364,18 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage *
         for (const auto & object : objects)
         {
 
-            LOG_INFO(disk->log, "Calling restore for key for disk {}", object.relative_path);
+            LOG_INFO(disk->log, "Calling restore for key for disk {}", object->relative_path);
 
             /// Skip file operations objects. They will be processed separately.
-            if (object.relative_path.find("/operations/") != String::npos)
+            if (object->relative_path.find("/operations/") != String::npos)
                 continue;
 
-            const auto [revision, _] = extractRevisionAndOperationFromKey(object.relative_path);
+            const auto [revision, _] = extractRevisionAndOperationFromKey(object->relative_path);
             /// Filter early if it's possible to get revision from key.
             if (revision > restore_information.revision)
                 continue;
 
-            keys_names.push_back(object.relative_path);
+            keys_names.push_back(object->relative_path);
         }
 
         if (!keys_names.empty())
@@ -405,26 +405,20 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::processRestoreFiles(
 {
     for (const auto & key : keys)
     {
-        auto meta = source_object_storage->getObjectMetadata(key);
-        auto object_attributes = meta.attributes;
+        auto metadata = source_object_storage->getObjectMetadata(key);
+        auto object_attributes = metadata.attributes;
 
         String path;
-        if (object_attributes.has_value())
+        /// Restore file if object has 'path' in metadata.
+        auto path_entry = object_attributes.find("path");
+        if (path_entry == object_attributes.end())
         {
-            /// Restore file if object has 'path' in metadata.
-            auto path_entry = object_attributes->find("path");
-            if (path_entry == object_attributes->end())
-            {
-                /// Such keys can remain after migration, we can skip them.
-                LOG_WARNING(disk->log, "Skip key {} because it doesn't have 'path' in metadata", key);
-                continue;
-            }
-
-            path = path_entry->second;
-        }
-        else
+            /// Such keys can remain after migration, we can skip them.
+            LOG_WARNING(disk->log, "Skip key {} because it doesn't have 'path' in metadata", key);
             continue;
+        }
 
+        path = path_entry->second;
         disk->createDirectories(directoryPath(path));
         auto object_key = ObjectStorageKey::createAsRelative(disk->object_key_prefix, shrinkKey(source_path, key));
 
@@ -436,7 +430,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::processRestoreFiles(
             source_object_storage->copyObjectToAnotherObjectStorage(object_from, object_to, read_settings, write_settings, *disk->object_storage);
 
         auto tx = disk->metadata_storage->createTransaction();
-        tx->addBlobToMetadata(path, object_key, meta.size_bytes);
+        tx->addBlobToMetadata(path, object_key, metadata.size_bytes);
         tx->commit();
 
         LOG_TRACE(disk->log, "Restored file {}", path);
@@ -475,10 +469,10 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject
 
         for (const auto & object : objects)
         {
-            const auto [revision, operation] = extractRevisionAndOperationFromKey(object.relative_path);
+            const auto [revision, operation] = extractRevisionAndOperationFromKey(object->relative_path);
             if (revision == UNKNOWN_REVISION)
             {
-                LOG_WARNING(disk->log, "Skip key {} with unknown revision", object.relative_path);
+                LOG_WARNING(disk->log, "Skip key {} with unknown revision", object->relative_path);
                 continue;
             }
 
@@ -491,7 +485,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject
             if (send_metadata)
                 revision_counter = revision - 1;
 
-            auto object_attributes = *(source_object_storage->getObjectMetadata(object.relative_path).attributes);
+            auto object_attributes = source_object_storage->getObjectMetadata(object->relative_path).attributes;
             if (operation == rename)
             {
                 auto from_path = object_attributes["from_path"];

src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp

@@ -1,12 +1,13 @@
 #include <Disks/ObjectStorages/HDFS/HDFSObjectStorage.h>
 
 #include <IO/copyData.h>
-#include <Storages/HDFS/WriteBufferFromHDFS.h>
-#include <Storages/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/WriteBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
 
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 #include <Common/getRandomASCIIString.h>
+#include <Common/logger_useful.h>
 
 
 #if USE_HDFS
@@ -18,28 +19,57 @@ namespace ErrorCodes
 {
     extern const int UNSUPPORTED_METHOD;
     extern const int HDFS_ERROR;
+    extern const int ACCESS_DENIED;
+    extern const int LOGICAL_ERROR;
 }
 
-void HDFSObjectStorage::shutdown()
+void HDFSObjectStorage::initializeHDFSFS() const
 {
+    if (initialized)
+        return;
+
+    std::lock_guard lock(init_mutex);
+    if (initialized)
+        return;
+
+    hdfs_builder = createHDFSBuilder(url, config);
+    hdfs_fs = createHDFSFS(hdfs_builder.get());
+    initialized = true;
 }
 
-void HDFSObjectStorage::startup()
+std::string HDFSObjectStorage::extractObjectKeyFromURL(const StoredObject & object) const
 {
+    /// This is very unfortunate, but for disk HDFS we made a mistake
+    /// and now its behaviour is inconsistent with S3 and Azure disks.
+    /// The mistake is that for HDFS we write into metadata files whole URL + data directory + key,
+    /// while for S3 and Azure we write there only data_directory + key.
+    /// This leads us into ambiguity that for StorageHDFS we have just key in object.remote_path,
+    /// but for DiskHDFS we have there URL as well.
+    auto path = object.remote_path;
+    if (path.starts_with(url))
+        path = path.substr(url.size());
+    if (path.starts_with("/"))
+        path.substr(1);
+    return path;
 }
 
 ObjectStorageKey HDFSObjectStorage::generateObjectKeyForPath(const std::string & /* path */) const
 {
+    initializeHDFSFS();
     /// what ever data_source_description.description value is, consider that key as relative key
-    return ObjectStorageKey::createAsRelative(hdfs_root_path, getRandomASCIIString(32));
+    chassert(data_directory.starts_with("/"));
+    return ObjectStorageKey::createAsRelative(
+        fs::path(url_without_path) / data_directory.substr(1), getRandomASCIIString(32));
 }
 
 bool HDFSObjectStorage::exists(const StoredObject & object) const
 {
-    const auto & path = object.remote_path;
-    const size_t begin_of_path = path.find('/', path.find("//") + 2);
-    const String remote_fs_object_path = path.substr(begin_of_path);
-    return (0 == hdfsExists(hdfs_fs.get(), remote_fs_object_path.c_str()));
+    initializeHDFSFS();
+    std::string path = object.remote_path;
+    if (path.starts_with(url_without_path))
+        path = path.substr(url_without_path.size());
+
+    return (0 == hdfsExists(hdfs_fs.get(), path.c_str()));
 }
 
 std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObject( /// NOLINT
@@ -48,7 +78,10 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObject( /// NOLIN
     std::optional<size_t>,
     std::optional<size_t>) const
 {
-    return std::make_unique<ReadBufferFromHDFS>(object.remote_path, object.remote_path, config, patchSettings(read_settings));
+    initializeHDFSFS();
+    auto path = extractObjectKeyFromURL(object);
+    return std::make_unique<ReadBufferFromHDFS>(
+        fs::path(url_without_path) / "", fs::path(data_directory) / path, config, patchSettings(read_settings));
 }
 
 std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLINT
@@ -57,18 +90,15 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLI
     std::optional<size_t>,
     std::optional<size_t>) const
 {
+    initializeHDFSFS();
     auto disk_read_settings = patchSettings(read_settings);
     auto read_buffer_creator =
         [this, disk_read_settings]
         (bool /* restricted_seek */, const StoredObject & object_) -> std::unique_ptr<ReadBufferFromFileBase>
     {
-        const auto & path = object_.remote_path;
-        size_t begin_of_path = path.find('/', path.find("//") + 2);
-        auto hdfs_path = path.substr(begin_of_path);
-        auto hdfs_uri = path.substr(0, begin_of_path);
-
+        auto path = extractObjectKeyFromURL(object_);
         return std::make_unique<ReadBufferFromHDFS>(
-            hdfs_uri, hdfs_path, config, disk_read_settings, /* read_until_position */0, /* use_external_buffer */true);
+            fs::path(url_without_path) / "", fs::path(data_directory) / path, config, disk_read_settings, /* read_until_position */0, /* use_external_buffer */true);
     };
 
     return std::make_unique<ReadBufferFromRemoteFSGather>(
@@ -82,14 +112,21 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
     size_t buf_size,
     const WriteSettings & write_settings)
 {
+    initializeHDFSFS();
     if (attributes.has_value())
         throw Exception(
             ErrorCodes::UNSUPPORTED_METHOD,
             "HDFS API doesn't support custom attributes/metadata for stored objects");
 
+    std::string path = object.remote_path;
+    if (path.starts_with("/"))
+        path = path.substr(1);
+    if (!path.starts_with(url))
+        path = fs::path(url) / path;
+
     /// Single O_WRONLY in libhdfs adds O_TRUNC
     return std::make_unique<WriteBufferFromHDFS>(
-        object.remote_path, config, settings->replication, patchSettings(write_settings), buf_size,
+        path, config, settings->replication, patchSettings(write_settings), buf_size,
         mode == WriteMode::Rewrite ? O_WRONLY : O_WRONLY | O_APPEND);
 }
 
@@ -97,11 +134,13 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
 /// Remove file. Throws exception if file doesn't exists or it's a directory.
 void HDFSObjectStorage::removeObject(const StoredObject & object)
 {
-    const auto & path = object.remote_path;
-    const size_t begin_of_path = path.find('/', path.find("//") + 2);
+    initializeHDFSFS();
+    auto path = object.remote_path;
+    if (path.starts_with(url_without_path))
+        path = path.substr(url_without_path.size());
 
     /// Add path from root to file name
-    int res = hdfsDelete(hdfs_fs.get(), path.substr(begin_of_path).c_str(), 0);
+    int res = hdfsDelete(hdfs_fs.get(), path.c_str(), 0);
     if (res == -1)
         throw Exception(ErrorCodes::HDFS_ERROR, "HDFSDelete failed with path: {}", path);
 
@@ -109,27 +148,85 @@ void HDFSObjectStorage::removeObject(const StoredObject & object)
 
 void HDFSObjectStorage::removeObjects(const StoredObjects & objects)
 {
+    initializeHDFSFS();
     for (const auto & object : objects)
         removeObject(object);
 }
 
 void HDFSObjectStorage::removeObjectIfExists(const StoredObject & object)
 {
+    initializeHDFSFS();
     if (exists(object))
         removeObject(object);
 }
 
 void HDFSObjectStorage::removeObjectsIfExist(const StoredObjects & objects)
 {
+    initializeHDFSFS();
     for (const auto & object : objects)
         removeObjectIfExists(object);
 }
 
-ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string &) const
+ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string & path) const
 {
-    throw Exception(
-        ErrorCodes::UNSUPPORTED_METHOD,
-        "HDFS API doesn't support custom attributes/metadata for stored objects");
+    initializeHDFSFS();
+    auto * file_info = hdfsGetPathInfo(hdfs_fs.get(), path.data());
+    if (!file_info)
+        throw Exception(ErrorCodes::HDFS_ERROR,
+                        "Cannot get file info for: {}. Error: {}", path, hdfsGetLastError());
+
+    ObjectMetadata metadata;
+    metadata.size_bytes = static_cast<size_t>(file_info->mSize);
+    metadata.last_modified = Poco::Timestamp::fromEpochTime(file_info->mLastMod);
+
+    hdfsFreeFileInfo(file_info, 1);
+    return metadata;
+}
+
+void HDFSObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const
+{
+    initializeHDFSFS();
+    LOG_TEST(log, "Trying to list files for {}", path);
+
+    HDFSFileInfo ls;
+    ls.file_info = hdfsListDirectory(hdfs_fs.get(), path.data(), &ls.length);
+
+    if (ls.file_info == nullptr && errno != ENOENT) // NOLINT
+    {
+        // ignore file not found exception, keep throw other exception,
+        // libhdfs3 doesn't have function to get exception type, so use errno.
+        throw Exception(ErrorCodes::ACCESS_DENIED, "Cannot list directory {}: {}",
+                        path, String(hdfsGetLastError()));
+    }
+
+    if (!ls.file_info && ls.length > 0)
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "file_info shouldn't be null");
+    }
+
+    LOG_TEST(log, "Listed {} files for {}", ls.length, path);
+
+    for (int i = 0; i < ls.length; ++i)
+    {
+        const String file_path = fs::path(ls.file_info[i].mName).lexically_normal();
+        const bool is_directory = ls.file_info[i].mKind == 'D';
+        if (is_directory)
+        {
+            listObjects(fs::path(file_path) / "", children, max_keys);
+        }
+        else
+        {
+            children.emplace_back(std::make_shared<RelativePathWithMetadata>(
+                String(file_path),
+                ObjectMetadata{
+                    static_cast<uint64_t>(ls.file_info[i].mSize),
+                    Poco::Timestamp::fromEpochTime(ls.file_info[i].mLastMod),
+                    {}}));
+        }
+
+        if (max_keys && children.size() >= max_keys)
+            break;
+    }
 }
 
 void HDFSObjectStorage::copyObject( /// NOLINT
@@ -139,6 +236,7 @@ void HDFSObjectStorage::copyObject( /// NOLINT
     const WriteSettings & write_settings,
     std::optional<ObjectAttributes> object_to_attributes)
 {
+    initializeHDFSFS();
     if (object_to_attributes.has_value())
         throw Exception(
             ErrorCodes::UNSUPPORTED_METHOD,
@@ -151,7 +249,10 @@ void HDFSObjectStorage::copyObject( /// NOLINT
 }
 
 
-std::unique_ptr<IObjectStorage> HDFSObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)
+std::unique_ptr<IObjectStorage> HDFSObjectStorage::cloneObjectStorage(
+    const std::string &,
+    const Poco::Util::AbstractConfiguration &,
+    const std::string &, ContextPtr)
 {
     throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "HDFS object storage doesn't support cloning");
 }

src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h

@@ -6,7 +6,7 @@
 
 #include <Disks/IDisk.h>
 #include <Disks/ObjectStorages/IObjectStorage.h>
-#include <Storages/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
 #include <Core/UUID.h>
 #include <memory>
 #include <Poco/Util/AbstractConfiguration.h>
@@ -16,21 +16,13 @@ namespace DB
 
 struct HDFSObjectStorageSettings
 {
-
-    HDFSObjectStorageSettings() = default;
-
-    size_t min_bytes_for_seek;
-    int objects_chunk_size_to_delete;
-    int replication;
-
-    HDFSObjectStorageSettings(
-            int min_bytes_for_seek_,
-            int objects_chunk_size_to_delete_,
-            int replication_)
+    HDFSObjectStorageSettings(int min_bytes_for_seek_, int replication_)
         : min_bytes_for_seek(min_bytes_for_seek_)
-        , objects_chunk_size_to_delete(objects_chunk_size_to_delete_)
         , replication(replication_)
     {}
+
+    size_t min_bytes_for_seek;
+    int replication;
 };
 
 
@@ -43,20 +35,29 @@ public:
     HDFSObjectStorage(
         const String & hdfs_root_path_,
         SettingsPtr settings_,
-        const Poco::Util::AbstractConfiguration & config_)
+        const Poco::Util::AbstractConfiguration & config_,
+        bool lazy_initialize)
         : config(config_)
-        , hdfs_builder(createHDFSBuilder(hdfs_root_path_, config))
-        , hdfs_fs(createHDFSFS(hdfs_builder.get()))
         , settings(std::move(settings_))
-        , hdfs_root_path(hdfs_root_path_)
+        , log(getLogger("HDFSObjectStorage(" + hdfs_root_path_ + ")"))
     {
+        const size_t begin_of_path = hdfs_root_path_.find('/', hdfs_root_path_.find("//") + 2);
+        url = hdfs_root_path_;
+        url_without_path = url.substr(0, begin_of_path);
+        if (begin_of_path < url.size())
+            data_directory = url.substr(begin_of_path);
+        else
+            data_directory = "/";
+
+        if (!lazy_initialize)
+            initializeHDFSFS();
     }
 
     std::string getName() const override { return "HDFSObjectStorage"; }
 
-    std::string getCommonKeyPrefix() const override { return hdfs_root_path; }
+    std::string getCommonKeyPrefix() const override { return url; }
 
-    std::string getDescription() const override { return hdfs_root_path; }
+    std::string getDescription() const override { return url; }
 
     ObjectStorageType getType() const override { return ObjectStorageType::HDFS; }
 
@@ -100,9 +101,7 @@ public:
         const WriteSettings & write_settings,
         std::optional<ObjectAttributes> object_to_attributes = {}) override;
 
-    void shutdown() override;
-
-    void startup() override;
+    void listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const override;
 
     String getObjectsNamespace() const override { return ""; }
 
@@ -116,13 +115,28 @@ public:
 
     bool isRemote() const override { return true; }
 
+    void startup() override { }
+
+    void shutdown() override { }
+
 private:
+    void initializeHDFSFS() const;
+    std::string extractObjectKeyFromURL(const StoredObject & object) const;
+
     const Poco::Util::AbstractConfiguration & config;
 
-    HDFSBuilderWrapper hdfs_builder;
-    HDFSFSPtr hdfs_fs;
+    mutable HDFSBuilderWrapper hdfs_builder;
+    mutable HDFSFSPtr hdfs_fs;
+
+    mutable std::mutex init_mutex;
+    mutable std::atomic_bool initialized{false};
+
     SettingsPtr settings;
-    const std::string hdfs_root_path;
+    std::string url;
+    std::string url_without_path;
+    std::string data_directory;
+
+    LoggerPtr log;
 };
 
 }

src/Disks/ObjectStorages/IObjectStorage.cpp

@@ -25,16 +25,16 @@ bool IObjectStorage::existsOrHasAnyChild(const std::string & path) const
     return !files.empty();
 }
 
-void IObjectStorage::listObjects(const std::string &, RelativePathsWithMetadata &, int) const
+void IObjectStorage::listObjects(const std::string &, RelativePathsWithMetadata &, size_t) const
 {
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "listObjects() is not supported");
 }
 
 
-ObjectStorageIteratorPtr IObjectStorage::iterate(const std::string & path_prefix) const
+ObjectStorageIteratorPtr IObjectStorage::iterate(const std::string & path_prefix, size_t max_keys) const
 {
     RelativePathsWithMetadata files;
-    listObjects(path_prefix, files, 0);
+    listObjects(path_prefix, files, max_keys);
 
     return std::make_shared<ObjectStorageIteratorFromList>(std::move(files));
 }

src/Disks/ObjectStorages/IObjectStorage.h

@@ -37,6 +37,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int NOT_IMPLEMENTED;
+    extern const int LOGICAL_ERROR;
 }
 
 class ReadBufferFromFileBase;
@@ -47,21 +48,28 @@ using ObjectAttributes = std::map<std::string, std::string>;
 struct ObjectMetadata
 {
     uint64_t size_bytes = 0;
-    std::optional<Poco::Timestamp> last_modified;
-    std::optional<ObjectAttributes> attributes;
+    Poco::Timestamp last_modified;
+    ObjectAttributes attributes;
 };
 
 struct RelativePathWithMetadata
 {
     String relative_path;
-    ObjectMetadata metadata;
+    std::optional<ObjectMetadata> metadata;
 
     RelativePathWithMetadata() = default;
 
-    RelativePathWithMetadata(String relative_path_, ObjectMetadata metadata_)
+    explicit RelativePathWithMetadata(String relative_path_, std::optional<ObjectMetadata> metadata_ = std::nullopt)
         : relative_path(std::move(relative_path_))
         , metadata(std::move(metadata_))
     {}
+
+    virtual ~RelativePathWithMetadata() = default;
+
+    virtual std::string getFileName() const { return std::filesystem::path(relative_path).filename(); }
+    virtual std::string getPath() const { return relative_path; }
+    virtual bool isArchive() const { return false; }
+    virtual std::string getPathToArchive() const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not an archive"); }
 };
 
 struct ObjectKeyWithMetadata
@@ -77,7 +85,8 @@ struct ObjectKeyWithMetadata
     {}
 };
 
-using RelativePathsWithMetadata = std::vector<RelativePathWithMetadata>;
+using RelativePathWithMetadataPtr = std::shared_ptr<RelativePathWithMetadata>;
+using RelativePathsWithMetadata = std::vector<RelativePathWithMetadataPtr>;
 using ObjectKeysWithMetadata = std::vector<ObjectKeyWithMetadata>;
 
 class IObjectStorageIterator;
@@ -111,9 +120,9 @@ public:
     /// /, /a, /a/b, /a/b/c, /a/b/c/d while exists will return true only for /a/b/c/d
     virtual bool existsOrHasAnyChild(const std::string & path) const;
 
-    virtual void listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const;
+    virtual void listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const;
 
-    virtual ObjectStorageIteratorPtr iterate(const std::string & path_prefix) const;
+    virtual ObjectStorageIteratorPtr iterate(const std::string & path_prefix, size_t max_keys) const;
 
     /// Get object metadata if supported. It should be possible to receive
     /// at least size of object
@@ -190,11 +199,15 @@ public:
     virtual void startup() = 0;
 
     /// Apply new settings, in most cases reiniatilize client and some other staff
+    struct ApplyNewSettingsOptions
+    {
+        bool allow_client_change = true;
+    };
     virtual void applyNewSettings(
-        const Poco::Util::AbstractConfiguration &,
+        const Poco::Util::AbstractConfiguration & /* config */,
         const std::string & /*config_prefix*/,
-        ContextPtr)
-    {}
+        ContextPtr /* context */,
+        const ApplyNewSettingsOptions & /* options */) {}
 
     /// Sometimes object storages have something similar to chroot or namespace, for example
     /// buckets in S3. If object storage doesn't have any namepaces return empty string.

src/Disks/ObjectStorages/IObjectStorage_fwd.h

@@ -10,4 +10,7 @@ using ObjectStoragePtr = std::shared_ptr<IObjectStorage>;
 class IMetadataStorage;
 using MetadataStoragePtr = std::shared_ptr<IMetadataStorage>;
 
+class IObjectStorageIterator;
+using ObjectStorageIteratorPtr = std::shared_ptr<IObjectStorageIterator>;
+
 }

src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp

@@ -172,7 +172,7 @@ ObjectMetadata LocalObjectStorage::getObjectMetadata(const std::string & path) c
     return object_metadata;
 }
 
-void LocalObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int /* max_keys */) const
+void LocalObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t/* max_keys */) const
 {
     for (const auto & entry : fs::directory_iterator(path))
     {
@@ -182,8 +182,7 @@ void LocalObjectStorage::listObjects(const std::string & path, RelativePathsWith
             continue;
         }
 
-        auto metadata = getObjectMetadata(entry.path());
-        children.emplace_back(entry.path(), std::move(metadata));
+        children.emplace_back(std::make_shared<RelativePathWithMetadata>(entry.path(), getObjectMetadata(entry.path())));
     }
 }
 
@@ -223,11 +222,6 @@ std::unique_ptr<IObjectStorage> LocalObjectStorage::cloneObjectStorage(
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "cloneObjectStorage() is not implemented for LocalObjectStorage");
 }
 
-void LocalObjectStorage::applyNewSettings(
-    const Poco::Util::AbstractConfiguration & /* config */, const std::string & /* config_prefix */, ContextPtr /* context */)
-{
-}
-
 ObjectStorageKey LocalObjectStorage::generateObjectKeyForPath(const std::string & /* path */) const
 {
     constexpr size_t key_name_total_size = 32;

src/Disks/ObjectStorages/Local/LocalObjectStorage.h

@@ -58,7 +58,7 @@ public:
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 
-    void listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const override;
+    void listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const override;
 
     bool existsOrHasAnyChild(const std::string & path) const override;
 
@@ -73,11 +73,6 @@ public:
 
     void startup() override;
 
-    void applyNewSettings(
-        const Poco::Util::AbstractConfiguration & config,
-        const std::string & config_prefix,
-        ContextPtr context) override;
-
     String getObjectsNamespace() const override { return ""; }
 
     std::unique_ptr<IObjectStorage> cloneObjectStorage(

src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp

@@ -105,7 +105,7 @@ std::vector<std::string> MetadataStorageFromPlainObjectStorage::getDirectChildre
     std::unordered_set<std::string> duplicates_filter;
     for (const auto & elem : remote_paths)
     {
-        const auto & path = elem.relative_path;
+        const auto & path = elem->relative_path;
         chassert(path.find(storage_key) == 0);
         const auto child_pos = storage_key.size();
         /// string::npos is ok.

src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp

@@ -26,11 +26,11 @@ MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::stri
     object_storage->listObjects(root, files, 0);
     for (const auto & file : files)
     {
-        auto remote_path = std::filesystem::path(file.relative_path);
+        auto remote_path = std::filesystem::path(file->relative_path);
         if (remote_path.filename() != PREFIX_PATH_FILE_NAME)
             continue;
 
-        StoredObject object{file.relative_path};
+        StoredObject object{file->relative_path};
 
         auto read_buf = object_storage->readObject(object);
         String local_path;
@@ -88,7 +88,7 @@ std::vector<std::string> getDirectChildrenOnRewritableDisk(
     auto skip_list = std::set<std::string>{PREFIX_PATH_FILE_NAME};
     for (const auto & elem : remote_paths)
     {
-        const auto & path = elem.relative_path;
+        const auto & path = elem->relative_path;
         chassert(path.find(storage_key) == 0);
         const auto child_pos = storage_key.size();
 

src/Disks/ObjectStorages/ObjectStorageFactory.cpp

@@ -9,7 +9,7 @@
 #endif
 #if USE_HDFS && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
 #include <Disks/ObjectStorages/HDFS/HDFSObjectStorage.h>
-#include <Storages/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
 #endif
 #if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
 #include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
@@ -183,7 +183,7 @@ void registerS3ObjectStorage(ObjectStorageFactory & factory)
         auto uri = getS3URI(config, config_prefix, context);
         auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix);
         auto settings = getSettings(config, config_prefix, context);
-        auto client = getClient(config, config_prefix, context, *settings);
+        auto client = getClient(config, config_prefix, context, *settings, true);
         auto key_generator = getKeyGenerator(uri, config, config_prefix);
 
         auto object_storage = createObjectStorage<S3ObjectStorage>(
@@ -219,7 +219,7 @@ void registerS3PlainObjectStorage(ObjectStorageFactory & factory)
         auto uri = getS3URI(config, config_prefix, context);
         auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix);
         auto settings = getSettings(config, config_prefix, context);
-        auto client = getClient(config, config_prefix, context, *settings);
+        auto client = getClient(config, config_prefix, context, *settings, true);
         auto key_generator = getKeyGenerator(uri, config, config_prefix);
 
         auto object_storage = std::make_shared<PlainObjectStorage<S3ObjectStorage>>(
@@ -253,7 +253,7 @@ void registerS3PlainRewritableObjectStorage(ObjectStorageFactory & factory)
             auto uri = getS3URI(config, config_prefix, context);
             auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix);
             auto settings = getSettings(config, config_prefix, context);
-            auto client = getClient(config, config_prefix, context, *settings);
+            auto client = getClient(config, config_prefix, context, *settings, true);
             auto key_generator = getKeyGenerator(uri, config, config_prefix);
 
             auto object_storage = std::make_shared<PlainRewritableObjectStorage<S3ObjectStorage>>(
@@ -287,10 +287,9 @@ void registerHDFSObjectStorage(ObjectStorageFactory & factory)
 
             std::unique_ptr<HDFSObjectStorageSettings> settings = std::make_unique<HDFSObjectStorageSettings>(
                 config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024),
-                config.getInt(config_prefix + ".objects_chunk_size_to_delete", 1000),
                 context->getSettingsRef().hdfs_replication);
 
-            return createObjectStorage<HDFSObjectStorage>(ObjectStorageType::HDFS, config, config_prefix, uri, std::move(settings), config);
+            return createObjectStorage<HDFSObjectStorage>(ObjectStorageType::HDFS, config, config_prefix, uri, std::move(settings), config, /* lazy_initialize */false);
         });
 }
 #endif

src/Disks/ObjectStorages/ObjectStorageIterator.cpp

@@ -9,7 +9,7 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-RelativePathWithMetadata ObjectStorageIteratorFromList::current()
+RelativePathWithMetadataPtr ObjectStorageIteratorFromList::current()
 {
     if (!isValid())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to access invalid iterator");

src/Disks/ObjectStorages/ObjectStorageIterator.h

@@ -12,9 +12,9 @@ public:
     virtual void next() = 0;
     virtual void nextBatch() = 0;
     virtual bool isValid() = 0;
-    virtual RelativePathWithMetadata current() = 0;
+    virtual RelativePathWithMetadataPtr current() = 0;
     virtual RelativePathsWithMetadata currentBatch() = 0;
-    virtual std::optional<RelativePathsWithMetadata> getCurrrentBatchAndScheduleNext() = 0;
+    virtual std::optional<RelativePathsWithMetadata> getCurrentBatchAndScheduleNext() = 0;
     virtual size_t getAccumulatedSize() const = 0;
 
     virtual ~IObjectStorageIterator() = default;
@@ -27,9 +27,7 @@ class ObjectStorageIteratorFromList : public IObjectStorageIterator
 public:
     explicit ObjectStorageIteratorFromList(RelativePathsWithMetadata && batch_)
         : batch(std::move(batch_))
-        , batch_iterator(batch.begin())
-    {
-    }
+        , batch_iterator(batch.begin()) {}
 
     void next() override
     {
@@ -37,32 +35,26 @@ public:
             ++batch_iterator;
     }
 
-    void nextBatch() override
+    void nextBatch() override { batch_iterator = batch.end(); }
+
+    bool isValid() override { return batch_iterator != batch.end(); }
+
+    RelativePathWithMetadataPtr current() override;
+
+    RelativePathsWithMetadata currentBatch() override { return batch; }
+
+    std::optional<RelativePathsWithMetadata> getCurrentBatchAndScheduleNext() override
     {
-        batch_iterator = batch.end();
+        if (batch.empty())
+            return {};
+
+        auto current_batch = std::move(batch);
+        batch = {};
+        return current_batch;
     }
 
-    bool isValid() override
-    {
-        return batch_iterator != batch.end();
-    }
+    size_t getAccumulatedSize() const override { return batch.size(); }
 
-    RelativePathWithMetadata current() override;
-
-    RelativePathsWithMetadata currentBatch() override
-    {
-        return batch;
-    }
-
-    std::optional<RelativePathsWithMetadata> getCurrrentBatchAndScheduleNext() override
-    {
-        return std::nullopt;
-    }
-
-    size_t getAccumulatedSize() const override
-    {
-        return batch.size();
-    }
 private:
     RelativePathsWithMetadata batch;
     RelativePathsWithMetadata::iterator batch_iterator;

src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp

@@ -11,10 +11,37 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
+IObjectStorageIteratorAsync::IObjectStorageIteratorAsync(
+    CurrentMetrics::Metric threads_metric,
+    CurrentMetrics::Metric threads_active_metric,
+    CurrentMetrics::Metric threads_scheduled_metric,
+    const std::string & thread_name)
+    : list_objects_pool(threads_metric, threads_active_metric, threads_scheduled_metric, 1)
+    , list_objects_scheduler(threadPoolCallbackRunnerUnsafe<BatchAndHasNext>(list_objects_pool, thread_name))
+{
+}
+
+IObjectStorageIteratorAsync::~IObjectStorageIteratorAsync()
+{
+    if (!deactivated)
+        deactivate();
+}
+
+void IObjectStorageIteratorAsync::deactivate()
+{
+    list_objects_pool.wait();
+    deactivated = true;
+}
+
 void IObjectStorageIteratorAsync::nextBatch()
 {
     std::lock_guard lock(mutex);
-    if (!is_finished)
+    if (is_finished)
+    {
+        current_batch.clear();
+        current_batch_iterator = current_batch.begin();
+    }
+    else
     {
         if (!is_initialized)
         {
@@ -22,19 +49,27 @@ void IObjectStorageIteratorAsync::nextBatch()
             is_initialized = true;
         }
 
-         BatchAndHasNext next_batch = outcome_future.get();
-         current_batch = std::move(next_batch.batch);
-         accumulated_size.fetch_add(current_batch.size(), std::memory_order_relaxed);
-         current_batch_iterator = current_batch.begin();
-         if (next_batch.has_next)
-             outcome_future = scheduleBatch();
-         else
-             is_finished = true;
-    }
-    else
-    {
-        current_batch.clear();
+        chassert(outcome_future.valid());
+        BatchAndHasNext result;
+        try
+        {
+            result = outcome_future.get();
+        }
+        catch (...)
+        {
+            is_finished = true;
+            throw;
+        }
+
+        current_batch = std::move(result.batch);
         current_batch_iterator = current_batch.begin();
+
+        accumulated_size.fetch_add(current_batch.size(), std::memory_order_relaxed);
+
+        if (result.has_next)
+            outcome_future = scheduleBatch();
+        else
+            is_finished = true;
     }
 }
 
@@ -42,24 +77,10 @@ void IObjectStorageIteratorAsync::next()
 {
     std::lock_guard lock(mutex);
 
-    if (current_batch_iterator != current_batch.end())
-    {
+    if (current_batch_iterator == current_batch.end())
+        nextBatch();
+    else
         ++current_batch_iterator;
-    }
-    else if (!is_finished)
-    {
-        if (outcome_future.valid())
-        {
-            BatchAndHasNext next_batch = outcome_future.get();
-            current_batch = std::move(next_batch.batch);
-            accumulated_size.fetch_add(current_batch.size(), std::memory_order_relaxed);
-            current_batch_iterator = current_batch.begin();
-            if (next_batch.has_next)
-                outcome_future = scheduleBatch();
-            else
-                is_finished = true;
-        }
-    }
 }
 
 std::future<IObjectStorageIteratorAsync::BatchAndHasNext> IObjectStorageIteratorAsync::scheduleBatch()
@@ -72,7 +93,6 @@ std::future<IObjectStorageIteratorAsync::BatchAndHasNext> IObjectStorageIterator
     }, Priority{});
 }
 
-
 bool IObjectStorageIteratorAsync::isValid()
 {
     if (!is_initialized)
@@ -82,7 +102,7 @@ bool IObjectStorageIteratorAsync::isValid()
     return current_batch_iterator != current_batch.end();
 }
 
-RelativePathWithMetadata IObjectStorageIteratorAsync::current()
+RelativePathWithMetadataPtr IObjectStorageIteratorAsync::current()
 {
     if (!isValid())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to access invalid iterator");
@@ -101,20 +121,20 @@ RelativePathsWithMetadata IObjectStorageIteratorAsync::currentBatch()
     return current_batch;
 }
 
-std::optional<RelativePathsWithMetadata> IObjectStorageIteratorAsync::getCurrrentBatchAndScheduleNext()
+std::optional<RelativePathsWithMetadata> IObjectStorageIteratorAsync::getCurrentBatchAndScheduleNext()
 {
     std::lock_guard lock(mutex);
     if (!is_initialized)
         nextBatch();
 
-    if (current_batch_iterator != current_batch.end())
+    if (current_batch_iterator == current_batch.end())
     {
-        auto temp_current_batch = current_batch;
-        nextBatch();
-        return temp_current_batch;
+        return std::nullopt;
     }
 
-    return std::nullopt;
+    auto temp_current_batch = std::move(current_batch);
+    nextBatch();
+    return temp_current_batch;
 }
 
 size_t IObjectStorageIteratorAsync::getAccumulatedSize() const

src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h

@@ -17,24 +17,22 @@ public:
         CurrentMetrics::Metric threads_metric,
         CurrentMetrics::Metric threads_active_metric,
         CurrentMetrics::Metric threads_scheduled_metric,
-        const std::string & thread_name)
-        : list_objects_pool(threads_metric, threads_active_metric, threads_scheduled_metric, 1)
-        , list_objects_scheduler(threadPoolCallbackRunnerUnsafe<BatchAndHasNext>(list_objects_pool, thread_name))
-    {
-    }
+        const std::string & thread_name);
+
+    ~IObjectStorageIteratorAsync() override;
+
+    bool isValid() override;
+
+    RelativePathWithMetadataPtr current() override;
+    RelativePathsWithMetadata currentBatch() override;
 
     void next() override;
     void nextBatch() override;
-    bool isValid() override;
-    RelativePathWithMetadata current() override;
-    RelativePathsWithMetadata currentBatch() override;
-    size_t getAccumulatedSize() const override;
-    std::optional<RelativePathsWithMetadata> getCurrrentBatchAndScheduleNext() override;
 
-    ~IObjectStorageIteratorAsync() override
-    {
-        list_objects_pool.wait();
-    }
+    size_t getAccumulatedSize() const override;
+    std::optional<RelativePathsWithMetadata> getCurrentBatchAndScheduleNext() override;
+
+    void deactivate();
 
 protected:
 
@@ -50,6 +48,7 @@ protected:
 
     bool is_initialized{false};
     bool is_finished{false};
+    bool deactivated{false};
 
     mutable std::recursive_mutex mutex;
     ThreadPool list_objects_pool;

src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp

@@ -61,7 +61,10 @@ void throwIfError(const Aws::Utils::Outcome<Result, Error> & response)
     if (!response.IsSuccess())
     {
         const auto & err = response.GetError();
-        throw S3Exception(fmt::format("{} (Code: {})", err.GetMessage(), static_cast<size_t>(err.GetErrorType())), err.GetErrorType());
+        throw S3Exception(
+            fmt::format("{} (Code: {}, s3 exception: {})",
+                        err.GetMessage(), static_cast<size_t>(err.GetErrorType()), err.GetExceptionName()),
+            err.GetErrorType());
     }
 }
 
@@ -111,10 +114,19 @@ public:
             CurrentMetrics::ObjectStorageS3ThreadsScheduled,
             "ListObjectS3")
         , client(client_)
+        , request(std::make_unique<S3::ListObjectsV2Request>())
     {
-        request.SetBucket(bucket_);
-        request.SetPrefix(path_prefix);
-        request.SetMaxKeys(static_cast<int>(max_list_size));
+        request->SetBucket(bucket_);
+        request->SetPrefix(path_prefix);
+        request->SetMaxKeys(static_cast<int>(max_list_size));
+    }
+
+    ~S3IteratorAsync() override
+    {
+        /// Deactivate background threads before resetting the request to avoid data race.
+        deactivate();
+        request.reset();
+        client.reset();
     }
 
 private:
@@ -123,34 +135,32 @@ private:
         ProfileEvents::increment(ProfileEvents::S3ListObjects);
         ProfileEvents::increment(ProfileEvents::DiskS3ListObjects);
 
-        bool result = false;
-        auto outcome = client->ListObjectsV2(request);
+        auto outcome = client->ListObjectsV2(*request);
+
         /// Outcome failure will be handled on the caller side.
         if (outcome.IsSuccess())
         {
+            request->SetContinuationToken(outcome.GetResult().GetNextContinuationToken());
+
             auto objects = outcome.GetResult().GetContents();
-
-            result = !objects.empty();
-
             for (const auto & object : objects)
-                batch.emplace_back(
-                    object.GetKey(),
-                    ObjectMetadata{static_cast<uint64_t>(object.GetSize()), Poco::Timestamp::fromEpochTime(object.GetLastModified().Seconds()), {}}
-                );
+            {
+                ObjectMetadata metadata{static_cast<uint64_t>(object.GetSize()), Poco::Timestamp::fromEpochTime(object.GetLastModified().Seconds()), {}};
+                batch.emplace_back(std::make_shared<RelativePathWithMetadata>(object.GetKey(), std::move(metadata)));
+            }
 
-            if (result)
-                request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken());
-
-            return result;
+            /// It returns false when all objects were returned
+            return outcome.GetResult().GetIsTruncated();
         }
 
-        throw S3Exception(outcome.GetError().GetErrorType(), "Could not list objects in bucket {} with prefix {}, S3 exception: {}, message: {}",
-                quoteString(request.GetBucket()), quoteString(request.GetPrefix()),
-                backQuote(outcome.GetError().GetExceptionName()), quoteString(outcome.GetError().GetMessage()));
+        throw S3Exception(outcome.GetError().GetErrorType(),
+                          "Could not list objects in bucket {} with prefix {}, S3 exception: {}, message: {}",
+                          quoteString(request->GetBucket()), quoteString(request->GetPrefix()),
+                          backQuote(outcome.GetError().GetExceptionName()), quoteString(outcome.GetError().GetMessage()));
     }
 
     std::shared_ptr<const S3::Client> client;
-    S3::ListObjectsV2Request request;
+    std::unique_ptr<S3::ListObjectsV2Request> request;
 };
 
 }
@@ -248,12 +258,16 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
     if (mode != WriteMode::Rewrite)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 doesn't support append to files");
 
-    auto settings_ptr = s3_settings.get();
+    S3Settings::RequestSettings request_settings = s3_settings.get()->request_settings;
+    if (auto query_context = CurrentThread::getQueryContext())
+    {
+        request_settings.updateFromSettingsIfChanged(query_context->getSettingsRef());
+    }
+
     ThreadPoolCallbackRunnerUnsafe<void> scheduler;
     if (write_settings.s3_allow_parallel_part_upload)
         scheduler = threadPoolCallbackRunnerUnsafe<void>(getThreadPoolWriter(), "VFSWrite");
 
-
     auto blob_storage_log = BlobStorageLogWriter::create(disk_name);
     if (blob_storage_log)
         blob_storage_log->local_path = object.local_path;
@@ -263,7 +277,7 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
         uri.bucket,
         object.remote_path,
         buf_size,
-        settings_ptr->request_settings,
+        request_settings,
         std::move(blob_storage_log),
         attributes,
         std::move(scheduler),
@@ -271,13 +285,13 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
 }
 
 
-ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix) const
+ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix, size_t max_keys) const
 {
     auto settings_ptr = s3_settings.get();
-    return std::make_shared<S3IteratorAsync>(uri.bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size);
+    return std::make_shared<S3IteratorAsync>(uri.bucket, path_prefix, client.get(), max_keys);
 }
 
-void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const
+void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const
 {
     auto settings_ptr = s3_settings.get();
 
@@ -285,7 +299,7 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet
     request.SetBucket(uri.bucket);
     request.SetPrefix(path);
     if (max_keys)
-        request.SetMaxKeys(max_keys);
+        request.SetMaxKeys(static_cast<int>(max_keys));
     else
         request.SetMaxKeys(settings_ptr->list_object_keys_size);
 
@@ -305,19 +319,19 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet
             break;
 
         for (const auto & object : objects)
-            children.emplace_back(
+            children.emplace_back(std::make_shared<RelativePathWithMetadata>(
                 object.GetKey(),
                 ObjectMetadata{
                     static_cast<uint64_t>(object.GetSize()),
                     Poco::Timestamp::fromEpochTime(object.GetLastModified().Seconds()),
-                    {}});
+                    {}}));
 
         if (max_keys)
         {
-            int keys_left = max_keys - static_cast<int>(children.size());
+            size_t keys_left = max_keys - children.size();
             if (keys_left <= 0)
                 break;
-            request.SetMaxKeys(keys_left);
+            request.SetMaxKeys(static_cast<int>(keys_left));
         }
 
         request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken());
@@ -425,14 +439,16 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects)
 std::optional<ObjectMetadata> S3ObjectStorage::tryGetObjectMetadata(const std::string & path) const
 {
     auto settings_ptr = s3_settings.get();
-    auto object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* throw_on_error= */ false);
+    auto object_info = S3::getObjectInfo(
+        *client.get(), uri.bucket, path, {}, settings_ptr->request_settings,
+        /* with_metadata= */ true, /* throw_on_error= */ false);
 
     if (object_info.size == 0 && object_info.last_modification_time == 0 && object_info.metadata.empty())
         return {};
 
     ObjectMetadata result;
     result.size_bytes = object_info.size;
-    result.last_modified = object_info.last_modification_time;
+    result.last_modified = Poco::Timestamp::fromEpochTime(object_info.last_modification_time);
     result.attributes = object_info.metadata;
 
     return result;
@@ -441,11 +457,20 @@ std::optional<ObjectMetadata> S3ObjectStorage::tryGetObjectMetadata(const std::s
 ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const
 {
     auto settings_ptr = s3_settings.get();
-    auto object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true);
+    S3::ObjectInfo object_info;
+    try
+    {
+        object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true);
+    }
+    catch (DB::Exception & e)
+    {
+        e.addMessage("while reading " + path);
+        throw;
+    }
 
     ObjectMetadata result;
     result.size_bytes = object_info.size;
-    result.last_modified = object_info.last_modification_time;
+    result.last_modified = Poco::Timestamp::fromEpochTime(object_info.last_modification_time);
     result.attributes = object_info.metadata;
 
     return result;
@@ -544,19 +569,40 @@ void S3ObjectStorage::startup()
     const_cast<S3::Client &>(*client.get()).EnableRequestProcessing();
 }
 
-void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
+void S3ObjectStorage::applyNewSettings(
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix,
+    ContextPtr context,
+    const ApplyNewSettingsOptions & options)
 {
-    auto new_s3_settings = getSettings(config, config_prefix, context);
-    auto new_client = getClient(config, config_prefix, context, *new_s3_settings);
+    auto new_s3_settings = getSettings(config, config_prefix, context, context->getSettingsRef().s3_validate_request_settings);
+    if (!static_headers.empty())
+    {
+        new_s3_settings->auth_settings.headers.insert(
+            new_s3_settings->auth_settings.headers.end(),
+            static_headers.begin(), static_headers.end());
+    }
+
+    if (auto endpoint_settings = context->getStorageS3Settings().getSettings(uri.uri.toString(), context->getUserName()))
+        new_s3_settings->auth_settings.updateFrom(endpoint_settings->auth_settings);
+
+    auto current_s3_settings = s3_settings.get();
+    if (options.allow_client_change && (current_s3_settings->auth_settings.hasUpdates(new_s3_settings->auth_settings) || for_disk_s3))
+    {
+        auto new_client = getClient(config, config_prefix, context, *new_s3_settings, for_disk_s3, &uri);
+        client.set(std::move(new_client));
+    }
     s3_settings.set(std::move(new_s3_settings));
-    client.set(std::move(new_client));
 }
 
 std::unique_ptr<IObjectStorage> S3ObjectStorage::cloneObjectStorage(
-    const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
+    const std::string & new_namespace,
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix,
+    ContextPtr context)
 {
     auto new_s3_settings = getSettings(config, config_prefix, context);
-    auto new_client = getClient(config, config_prefix, context, *new_s3_settings);
+    auto new_client = getClient(config, config_prefix, context, *new_s3_settings, true);
 
     auto new_uri{uri};
     new_uri.bucket = new_namespace;

src/Disks/ObjectStorages/S3/S3ObjectStorage.h

@@ -21,11 +21,13 @@ struct S3ObjectStorageSettings
 
     S3ObjectStorageSettings(
         const S3Settings::RequestSettings & request_settings_,
+        const S3::AuthSettings & auth_settings_,
         uint64_t min_bytes_for_seek_,
         int32_t list_object_keys_size_,
         int32_t objects_chunk_size_to_delete_,
         bool read_only_)
         : request_settings(request_settings_)
+        , auth_settings(auth_settings_)
         , min_bytes_for_seek(min_bytes_for_seek_)
         , list_object_keys_size(list_object_keys_size_)
         , objects_chunk_size_to_delete(objects_chunk_size_to_delete_)
@@ -33,6 +35,7 @@ struct S3ObjectStorageSettings
     {}
 
     S3Settings::RequestSettings request_settings;
+    S3::AuthSettings auth_settings;
 
     uint64_t min_bytes_for_seek;
     int32_t list_object_keys_size;
@@ -50,7 +53,9 @@ private:
         S3::URI uri_,
         const S3Capabilities & s3_capabilities_,
         ObjectStorageKeysGeneratorPtr key_generator_,
-        const String & disk_name_)
+        const String & disk_name_,
+        bool for_disk_s3_ = true,
+        const HTTPHeaderEntries & static_headers_ = {})
         : uri(uri_)
         , disk_name(disk_name_)
         , client(std::move(client_))
@@ -58,6 +63,8 @@ private:
         , s3_capabilities(s3_capabilities_)
         , key_generator(std::move(key_generator_))
         , log(getLogger(logger_name))
+        , for_disk_s3(for_disk_s3_)
+        , static_headers(static_headers_)
     {
     }
 
@@ -98,9 +105,9 @@ public:
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
         const WriteSettings & write_settings = {}) override;
 
-    void listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const override;
+    void listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const override;
 
-    ObjectStorageIteratorPtr iterate(const std::string & path_prefix) const override;
+    ObjectStorageIteratorPtr iterate(const std::string & path_prefix, size_t max_keys) const override;
 
     /// Uses `DeleteObjectRequest`.
     void removeObject(const StoredObject & object) override;
@@ -142,7 +149,8 @@ public:
     void applyNewSettings(
         const Poco::Util::AbstractConfiguration & config,
         const std::string & config_prefix,
-        ContextPtr context) override;
+        ContextPtr context,
+        const ApplyNewSettingsOptions & options) override;
 
     std::string getObjectsNamespace() const override { return uri.bucket; }
 
@@ -179,6 +187,9 @@ private:
     ObjectStorageKeysGeneratorPtr key_generator;
 
     LoggerPtr log;
+
+    const bool for_disk_s3;
+    const HTTPHeaderEntries static_headers;
 };
 
 }

src/Disks/ObjectStorages/S3/diskSettings.cpp

@@ -25,19 +25,29 @@
 
 namespace DB
 {
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
 
 namespace ErrorCodes
 {
 extern const int NO_ELEMENTS_IN_CONFIG;
 }
 
-std::unique_ptr<S3ObjectStorageSettings> getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context)
+std::unique_ptr<S3ObjectStorageSettings> getSettings(
+    const Poco::Util::AbstractConfiguration & config,
+    const String & config_prefix,
+    ContextPtr context,
+    bool validate_settings)
 {
     const Settings & settings = context->getSettingsRef();
-    S3Settings::RequestSettings request_settings(config, config_prefix, settings, "s3_");
+    auto request_settings = S3Settings::RequestSettings(config, config_prefix, settings, "s3_", validate_settings);
+    auto auth_settings = S3::AuthSettings::loadFromConfig(config_prefix, config);
 
     return std::make_unique<S3ObjectStorageSettings>(
         request_settings,
+        auth_settings,
         config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024),
         config.getInt(config_prefix + ".list_object_keys_size", 1000),
         config.getInt(config_prefix + ".objects_chunk_size_to_delete", 1000),
@@ -48,82 +58,99 @@ std::unique_ptr<S3::Client> getClient(
     const Poco::Util::AbstractConfiguration & config,
     const String & config_prefix,
     ContextPtr context,
-    const S3ObjectStorageSettings & settings)
+    const S3ObjectStorageSettings & settings,
+    bool for_disk_s3,
+    const S3::URI * url_)
 {
     const Settings & global_settings = context->getGlobalContext()->getSettingsRef();
     const Settings & local_settings = context->getSettingsRef();
 
-    const String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint"));
-    S3::URI uri(endpoint);
-    if (!uri.key.ends_with('/'))
-        uri.key.push_back('/');
+    const auto & auth_settings = settings.auth_settings;
+    const auto & request_settings = settings.request_settings;
 
-    if (S3::isS3ExpressEndpoint(endpoint) && !config.has(config_prefix + ".region"))
+    S3::URI url;
+    if (for_disk_s3)
+    {
+        String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint"));
+        url = S3::URI(endpoint);
+        if (!url.key.ends_with('/'))
+            url.key.push_back('/');
+    }
+    else
+    {
+        if (!url_)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "URL not passed");
+        url = *url_;
+    }
+    const bool is_s3_express_bucket = S3::isS3ExpressEndpoint(url.endpoint);
+    if (is_s3_express_bucket && !config.has(config_prefix + ".region"))
+    {
         throw Exception(
             ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Region should be explicitly specified for directory buckets ({})", config_prefix);
+    }
 
     S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
-        config.getString(config_prefix + ".region", ""),
+        auth_settings.region,
         context->getRemoteHostFilter(),
         static_cast<int>(global_settings.s3_max_redirects),
         static_cast<int>(global_settings.s3_retry_attempts),
         global_settings.enable_s3_requests_logging,
-        /* for_disk_s3 = */ true,
+        for_disk_s3,
         settings.request_settings.get_request_throttler,
         settings.request_settings.put_request_throttler,
-        uri.uri.getScheme());
+        url.uri.getScheme());
 
-    client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", S3::DEFAULT_CONNECT_TIMEOUT_MS);
-    client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", S3::DEFAULT_REQUEST_TIMEOUT_MS);
-    client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", S3::DEFAULT_MAX_CONNECTIONS);
+    client_configuration.connectTimeoutMs = config.getUInt64(config_prefix + ".connect_timeout_ms", local_settings.s3_connect_timeout_ms.value);
+    client_configuration.requestTimeoutMs = config.getUInt64(config_prefix + ".request_timeout_ms", local_settings.s3_request_timeout_ms.value);
+    client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", static_cast<unsigned>(request_settings.max_connections));
     client_configuration.http_keep_alive_timeout = config.getUInt(config_prefix + ".http_keep_alive_timeout", S3::DEFAULT_KEEP_ALIVE_TIMEOUT);
     client_configuration.http_keep_alive_max_requests = config.getUInt(config_prefix + ".http_keep_alive_max_requests", S3::DEFAULT_KEEP_ALIVE_MAX_REQUESTS);
 
-    client_configuration.endpointOverride = uri.endpoint;
+    client_configuration.endpointOverride = url.endpoint;
     client_configuration.s3_use_adaptive_timeouts = config.getBool(
         config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts);
 
-    /*
-     * Override proxy configuration for backwards compatibility with old configuration format.
-     * */
-    auto proxy_config = DB::ProxyConfigurationResolverProvider::getFromOldSettingsFormat(
-        ProxyConfiguration::protocolFromString(uri.uri.getScheme()),
-        config_prefix,
-        config
-    );
-    if (proxy_config)
+    if (for_disk_s3)
     {
-        client_configuration.per_request_configuration
-            = [proxy_config]() { return proxy_config->resolve(); };
-        client_configuration.error_report
-            = [proxy_config](const auto & request_config) { proxy_config->errorReport(request_config); };
+        /*
+        * Override proxy configuration for backwards compatibility with old configuration format.
+        * */
+        if (auto proxy_config = DB::ProxyConfigurationResolverProvider::getFromOldSettingsFormat(
+                ProxyConfiguration::protocolFromString(url.uri.getScheme()), config_prefix, config))
+        {
+            client_configuration.per_request_configuration
+                = [proxy_config]() { return proxy_config->resolve(); };
+            client_configuration.error_report
+                = [proxy_config](const auto & request_config) { proxy_config->errorReport(request_config); };
+        }
     }
 
-    HTTPHeaderEntries headers = S3::getHTTPHeaders(config_prefix, config);
     S3::ServerSideEncryptionKMSConfig sse_kms_config = S3::getSSEKMSConfig(config_prefix, config);
-
     S3::ClientSettings client_settings{
-        .use_virtual_addressing = uri.is_virtual_hosted_style,
+        .use_virtual_addressing = url.is_virtual_hosted_style,
         .disable_checksum = local_settings.s3_disable_checksum,
         .gcs_issue_compose_request = config.getBool("s3.gcs_issue_compose_request", false),
-        .is_s3express_bucket = S3::isS3ExpressEndpoint(endpoint),
+        .is_s3express_bucket = is_s3_express_bucket,
+    };
+
+    auto credentials_configuration = S3::CredentialsConfiguration
+    {
+        auth_settings.use_environment_credentials.value_or(context->getConfigRef().getBool("s3.use_environment_credentials", true)),
+        auth_settings.use_insecure_imds_request.value_or(context->getConfigRef().getBool("s3.use_insecure_imds_request", false)),
+        auth_settings.expiration_window_seconds.value_or(context->getConfigRef().getUInt64("s3.expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS)),
+        auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)),
     };
 
     return S3::ClientFactory::instance().create(
         client_configuration,
         client_settings,
-        config.getString(config_prefix + ".access_key_id", ""),
-        config.getString(config_prefix + ".secret_access_key", ""),
-        config.getString(config_prefix + ".server_side_encryption_customer_key_base64", ""),
+        auth_settings.access_key_id,
+        auth_settings.secret_access_key,
+        auth_settings.server_side_encryption_customer_key_base64,
         std::move(sse_kms_config),
-        std::move(headers),
-        S3::CredentialsConfiguration
-        {
-            config.getBool(config_prefix + ".use_environment_credentials", config.getBool("s3.use_environment_credentials", true)),
-            config.getBool(config_prefix + ".use_insecure_imds_request", config.getBool("s3.use_insecure_imds_request", false)),
-            config.getUInt64(config_prefix + ".expiration_window_seconds", config.getUInt64("s3.expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS)),
-            config.getBool(config_prefix + ".no_sign_request", config.getBool("s3.no_sign_request", false))
-        });
+        auth_settings.headers,
+        credentials_configuration,
+        auth_settings.session_token);
 }
 
 }

src/Disks/ObjectStorages/S3/diskSettings.h

@@ -14,9 +14,19 @@ namespace DB
 
 struct S3ObjectStorageSettings;
 
-std::unique_ptr<S3ObjectStorageSettings> getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context);
+std::unique_ptr<S3ObjectStorageSettings> getSettings(
+    const Poco::Util::AbstractConfiguration & config,
+    const String & config_prefix,
+    ContextPtr context,
+    bool validate_settings = true);
 
-std::unique_ptr<S3::Client> getClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context, const S3ObjectStorageSettings & settings);
+std::unique_ptr<S3::Client> getClient(
+    const Poco::Util::AbstractConfiguration & config,
+    const String & config_prefix,
+    ContextPtr context,
+    const S3ObjectStorageSettings & settings,
+    bool for_disk_s3,
+    const S3::URI * url_ = nullptr);
 
 }
 

src/Disks/ObjectStorages/Web/WebObjectStorage.cpp

@@ -344,11 +344,6 @@ void WebObjectStorage::startup()
 {
 }
 
-void WebObjectStorage::applyNewSettings(
-    const Poco::Util::AbstractConfiguration & /* config */, const std::string & /* config_prefix */, ContextPtr /* context */)
-{
-}
-
 ObjectMetadata WebObjectStorage::getObjectMetadata(const std::string & /* path */) const
 {
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Metadata is not supported for {}", getName());

src/Disks/ObjectStorages/Web/WebObjectStorage.h

@@ -72,11 +72,6 @@ public:
 
     void startup() override;
 
-    void applyNewSettings(
-        const Poco::Util::AbstractConfiguration & config,
-        const std::string & config_prefix,
-        ContextPtr context) override;
-
     String getObjectsNamespace() const override { return ""; }
 
     std::unique_ptr<IObjectStorage> cloneObjectStorage(

src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h

@@ -4,8 +4,7 @@
 
 #if USE_AZURE_BLOB_STORAGE
 
-#include <Storages/StorageAzureBlobCluster.h>
-#include <Storages/StorageAzureBlob.h>
+#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
 #include <Common/threadPoolCallbackRunner.h>
 #include <base/types.h>
 #include <functional>

src/IO/S3/URI.h

@@ -29,6 +29,7 @@ struct URI
     std::string key;
     std::string version_id;
     std::string storage_name;
+    /// Path (or path pattern) in archive if uri is an archive.
     std::optional<std::string> archive_pattern;
     std::string uri_str;
 

src/IO/S3/getObjectInfo.cpp

@@ -53,7 +53,7 @@ namespace
         const auto & result = outcome.GetResult();
         ObjectInfo object_info;
         object_info.size = static_cast<size_t>(result.GetContentLength());
-        object_info.last_modification_time = result.GetLastModified().Millis() / 1000;
+        object_info.last_modification_time = result.GetLastModified().Seconds();
 
         if (with_metadata)
             object_info.metadata = result.GetMetadata();

src/IO/S3Common.cpp

@@ -174,8 +174,11 @@ void AuthSettings::updateFrom(const AuthSettings & from)
     if (!from.session_token.empty())
         session_token = from.session_token;
 
-    headers = from.headers;
-    region = from.region;
+    if (!from.headers.empty())
+        headers = from.headers;
+    if (!from.region.empty())
+        region = from.region;
+
     server_side_encryption_customer_key_base64 = from.server_side_encryption_customer_key_base64;
     server_side_encryption_kms_config = from.server_side_encryption_kms_config;
 

src/IO/examples/read_buffer_from_hdfs.cpp

@@ -2,7 +2,7 @@
 #include <string>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/copyData.h>
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
 #include <base/types.h>
 #include <Common/Config/ConfigProcessor.h>
 

src/IO/tests/gtest_writebuffer_s3.cpp

@@ -547,7 +547,7 @@ public:
     std::unique_ptr<WriteBufferFromS3> getWriteBuffer(String file_name = "file")
     {
         S3Settings::RequestSettings request_settings;
-        request_settings.updateFromSettings(settings);
+        request_settings.updateFromSettingsIfChanged(settings);
 
         client->resetCounters();
 

src/Interpreters/ClusterProxy/executeQuery.cpp

@@ -403,6 +403,10 @@ void executeQueryWithParallelReplicas(
     ContextPtr context,
     std::shared_ptr<const StorageLimitsList> storage_limits)
 {
+    auto logger = getLogger("executeQueryWithParallelReplicas");
+    LOG_DEBUG(logger, "Executing read from {}, header {}, query ({}), stage {} with parallel replicas",
+        storage_id.getNameForLogs(), header.dumpStructure(), query_ast->formatForLogging(), processed_stage);
+
     const auto & settings = context->getSettingsRef();
 
     /// check cluster for parallel replicas

src/Interpreters/InterpreterCreateQuery.cpp

@@ -1500,7 +1500,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
 
     validateVirtualColumns(*res);
 
-    if (!res->supportsDynamicSubcolumnsDeprecated() && hasDynamicSubcolumns(res->getInMemoryMetadataPtr()->getColumns()))
+    if (!res->supportsDynamicSubcolumnsDeprecated() && hasDynamicSubcolumns(res->getInMemoryMetadataPtr()->getColumns()) && mode <= LoadingStrictnessLevel::CREATE)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN,
             "Cannot create table with column of type Object, "

src/Interpreters/InterpreterDeleteQuery.cpp

@@ -25,6 +25,7 @@ namespace ErrorCodes
     extern const int TABLE_IS_READ_ONLY;
     extern const int SUPPORT_IS_DISABLED;
     extern const int BAD_ARGUMENTS;
+    extern const int NOT_IMPLEMENTED;
 }
 
 
@@ -107,7 +108,19 @@ BlockIO InterpreterDeleteQuery::execute()
     }
     else
     {
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "DELETE query is not supported for table {}", table->getStorageID().getFullTableName());
+        /// Currently just better exception for the case of a table with projection,
+        /// can act differently according to the setting.
+        if (table->hasProjection())
+        {
+            throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+                "DELETE query is not supported for table {} as it has projections. "
+                "User should drop all the projections manually before running the query",
+                table->getStorageID().getFullTableName());
+        }
+
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            "DELETE query is not supported for table {}",
+            table->getStorageID().getFullTableName());
     }
 }
 

src/Interpreters/InterpreterSystemQuery.cpp

@@ -51,11 +51,12 @@
 #include <Storages/Freeze.h>
 #include <Storages/StorageFactory.h>
 #include <Storages/StorageFile.h>
-#include <Storages/StorageS3.h>
 #include <Storages/StorageURL.h>
-#include <Storages/StorageAzureBlob.h>
+#include <Storages/ObjectStorage/StorageObjectStorage.h>
+#include <Storages/ObjectStorage/S3/Configuration.h>
+#include <Storages/ObjectStorage/HDFS/Configuration.h>
+#include <Storages/ObjectStorage/Azure/Configuration.h>
 #include <Storages/MaterializedView/RefreshTask.h>
-#include <Storages/HDFS/StorageHDFS.h>
 #include <Storages/System/StorageSystemFilesystemCache.h>
 #include <Parsers/ASTSystemQuery.h>
 #include <Parsers/ASTCreateQuery.h>
@@ -500,17 +501,17 @@ BlockIO InterpreterSystemQuery::execute()
                 StorageFile::getSchemaCache(getContext()).clear();
 #if USE_AWS_S3
             if (caches_to_drop.contains("S3"))
-                StorageS3::getSchemaCache(getContext()).clear();
+                StorageObjectStorage::getSchemaCache(getContext(), StorageS3Configuration::type_name).clear();
 #endif
 #if USE_HDFS
             if (caches_to_drop.contains("HDFS"))
-                StorageHDFS::getSchemaCache(getContext()).clear();
+                StorageObjectStorage::getSchemaCache(getContext(), StorageHDFSConfiguration::type_name).clear();
 #endif
             if (caches_to_drop.contains("URL"))
                 StorageURL::getSchemaCache(getContext()).clear();
 #if USE_AZURE_BLOB_STORAGE
             if (caches_to_drop.contains("AZURE"))
-                StorageAzureBlob::getSchemaCache(getContext()).clear();
+                StorageObjectStorage::getSchemaCache(getContext(), StorageAzureConfiguration::type_name).clear();
 #endif
             break;
         }

src/Planner/PlannerExpressionAnalysis.cpp

@@ -444,7 +444,6 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node,
         for (auto & interpolate_node : interpolate_list_node.getNodes())
         {
             auto & interpolate_node_typed = interpolate_node->as<InterpolateNode &>();
-            interpolate_actions_visitor.visit(interpolate_actions_dag, interpolate_node_typed.getExpression());
             interpolate_actions_visitor.visit(interpolate_actions_dag, interpolate_node_typed.getInterpolateExpression());
         }
 

src/Processors/QueryPlan/ReadFromRemote.cpp

@@ -386,6 +386,8 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep(
     chassert(cluster->getShardCount() == 1);
 
     std::vector<String> description;
+    description.push_back(fmt::format("query: {}", formattedAST(query_ast)));
+
     for (const auto & pool : cluster->getShardsInfo().front().per_replica_pools)
         description.push_back(fmt::format("Replica: {}", pool->getHost()));
 

src/Server/TCPHandler.cpp

@@ -34,6 +34,7 @@
 #include <Server/TCPServer.h>
 #include <Storages/StorageReplicatedMergeTree.h>
 #include <Storages/MergeTree/MergeTreeDataPartUUID.h>
+#include <Storages/ObjectStorage/StorageObjectStorageCluster.h>
 #include <Core/ExternalTable.h>
 #include <Core/ServerSettings.h>
 #include <Access/AccessControl.h>

src/Storages/DataLakes/DeltaLakeMetadataParser.h

@@ -1,22 +0,0 @@
-#pragma once
-
-#include <Interpreters/Context_fwd.h>
-#include <Core/Types.h>
-
-namespace DB
-{
-
-template <typename Configuration, typename MetadataReadHelper>
-struct DeltaLakeMetadataParser
-{
-public:
-    DeltaLakeMetadataParser<Configuration, MetadataReadHelper>();
-
-    Strings getFiles(const Configuration & configuration, ContextPtr context);
-
-private:
-    struct Impl;
-    std::shared_ptr<Impl> impl;
-};
-
-}

src/Storages/DataLakes/HudiMetadataParser.cpp

@@ -1,116 +0,0 @@
-#include <Storages/DataLakes/HudiMetadataParser.h>
-#include <Common/logger_useful.h>
-#include <ranges>
-#include <base/find_symbols.h>
-#include <Poco/String.h>
-#include "config.h"
-#include <filesystem>
-#include <IO/ReadHelpers.h>
-
-#if USE_AWS_S3
-#include <Storages/DataLakes/S3MetadataReader.h>
-#include <Storages/StorageS3.h>
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-    extern const int LOGICAL_ERROR;
-}
-
-template <typename Configuration, typename MetadataReadHelper>
-struct HudiMetadataParser<Configuration, MetadataReadHelper>::Impl
-{
-    /**
-     * Useful links:
-     * - https://hudi.apache.org/tech-specs/
-     * - https://hudi.apache.org/docs/file_layouts/
-     */
-
-    /**
-      * Hudi tables store metadata files and data files.
-      * Metadata files are stored in .hoodie/metadata directory. Though unlike DeltaLake and Iceberg,
-      * metadata is not required in order to understand which files we need to read, moreover,
-      * for Hudi metadata does not always exist.
-      *
-      * There can be two types of data files
-      * 1. base files (columnar file formats like Apache Parquet/Orc)
-      * 2. log files
-      * Currently we support reading only `base files`.
-      * Data file name format:
-      * [File Id]_[File Write Token]_[Transaction timestamp].[File Extension]
-      *
-      * To find needed parts we need to find out latest part file for every file group for every partition.
-      * Explanation why:
-      *    Hudi reads in and overwrites the entire table/partition with each update.
-      *    Hudi controls the number of file groups under a single partition according to the
-      *    hoodie.parquet.max.file.size option. Once a single Parquet file is too large, Hudi creates a second file group.
-      *    Each file group is identified by File Id.
-      */
-    Strings processMetadataFiles(const Configuration & configuration)
-    {
-        auto log = getLogger("HudiMetadataParser");
-
-        const auto keys = MetadataReadHelper::listFiles(configuration, "", Poco::toLower(configuration.format));
-
-        using Partition = std::string;
-        using FileID = std::string;
-        struct FileInfo
-        {
-            String key;
-            UInt64 timestamp = 0;
-        };
-        std::unordered_map<Partition, std::unordered_map<FileID, FileInfo>> data_files;
-
-        for (const auto & key : keys)
-        {
-            auto key_file = std::filesystem::path(key);
-            Strings file_parts;
-            const String stem = key_file.stem();
-            splitInto<'_'>(file_parts, stem);
-            if (file_parts.size() != 3)
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected format for file: {}", key);
-
-            const auto partition = key_file.parent_path().stem();
-            const auto & file_id = file_parts[0];
-            const auto timestamp = parse<UInt64>(file_parts[2]);
-
-            auto & file_info = data_files[partition][file_id];
-            if (file_info.timestamp == 0 || file_info.timestamp < timestamp)
-            {
-                file_info.key = std::move(key);
-                file_info.timestamp = timestamp;
-            }
-        }
-
-        Strings result;
-        for (auto & [partition, partition_data] : data_files)
-        {
-            LOG_TRACE(log, "Adding {} data files from partition {}", partition, partition_data.size());
-            for (auto & [file_id, file_data] : partition_data)
-                result.push_back(std::move(file_data.key));
-        }
-        return result;
-    }
-};
-
-
-template <typename Configuration, typename MetadataReadHelper>
-HudiMetadataParser<Configuration, MetadataReadHelper>::HudiMetadataParser() : impl(std::make_unique<Impl>())
-{
-}
-
-template <typename Configuration, typename MetadataReadHelper>
-Strings HudiMetadataParser<Configuration, MetadataReadHelper>::getFiles(const Configuration & configuration, ContextPtr)
-{
-    return impl->processMetadataFiles(configuration);
-}
-
-template HudiMetadataParser<StorageS3::Configuration, S3DataLakeMetadataReadHelper>::HudiMetadataParser();
-template Strings HudiMetadataParser<StorageS3::Configuration, S3DataLakeMetadataReadHelper>::getFiles(
-    const StorageS3::Configuration & configuration, ContextPtr);
-
-}
-
-#endif

src/Storages/DataLakes/HudiMetadataParser.h

@@ -1,22 +0,0 @@
-#pragma once
-
-#include <Interpreters/Context_fwd.h>
-#include <Core/Types.h>
-
-namespace DB
-{
-
-template <typename Configuration, typename MetadataReadHelper>
-struct HudiMetadataParser
-{
-public:
-    HudiMetadataParser<Configuration, MetadataReadHelper>();
-
-    Strings getFiles(const Configuration & configuration, ContextPtr context);
-
-private:
-    struct Impl;
-    std::shared_ptr<Impl> impl;
-};
-
-}

src/Storages/DataLakes/IStorageDataLake.h

@@ -1,136 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#if USE_AWS_S3
-
-#include <Storages/IStorage.h>
-#include <Common/logger_useful.h>
-#include <Databases/LoadingStrictnessLevel.h>
-#include <Storages/StorageFactory.h>
-#include <Formats/FormatFactory.h>
-#include <filesystem>
-
-
-namespace DB
-{
-
-template <typename Storage, typename Name, typename MetadataParser>
-class IStorageDataLake : public Storage
-{
-public:
-    static constexpr auto name = Name::name;
-    using Configuration = typename Storage::Configuration;
-
-    template <class ...Args>
-    explicit IStorageDataLake(const Configuration & configuration_, ContextPtr context_, LoadingStrictnessLevel mode, Args && ...args)
-        : Storage(getConfigurationForDataRead(configuration_, context_, {}, mode), context_, std::forward<Args>(args)...)
-        , base_configuration(configuration_)
-        , log(getLogger(getName())) {} // NOLINT(clang-analyzer-optin.cplusplus.VirtualCall)
-
-    template <class ...Args>
-    static StoragePtr create(const Configuration & configuration_, ContextPtr context_, LoadingStrictnessLevel mode, Args && ...args)
-    {
-        return std::make_shared<IStorageDataLake<Storage, Name, MetadataParser>>(configuration_, context_, mode, std::forward<Args>(args)...);
-    }
-
-    String getName() const override { return name; }
-
-    static ColumnsDescription getTableStructureFromData(
-        Configuration & base_configuration,
-        const std::optional<FormatSettings> & format_settings,
-        const ContextPtr & local_context)
-    {
-        auto configuration = getConfigurationForDataRead(base_configuration, local_context);
-        return Storage::getTableStructureFromData(configuration, format_settings, local_context);
-    }
-
-    static Configuration getConfiguration(ASTs & engine_args, const ContextPtr & local_context)
-    {
-        return Storage::getConfiguration(engine_args, local_context, /* get_format_from_file */false);
-    }
-
-    Configuration updateConfigurationAndGetCopy(const ContextPtr & local_context) override
-    {
-        std::lock_guard lock(configuration_update_mutex);
-        updateConfigurationImpl(local_context);
-        return Storage::getConfigurationCopy();
-    }
-
-    void updateConfiguration(const ContextPtr & local_context) override
-    {
-        std::lock_guard lock(configuration_update_mutex);
-        updateConfigurationImpl(local_context);
-    }
-
-private:
-    static Configuration getConfigurationForDataRead(
-        const Configuration & base_configuration, const ContextPtr & local_context, const Strings & keys = {},
-        LoadingStrictnessLevel mode = LoadingStrictnessLevel::CREATE)
-    {
-        auto configuration{base_configuration};
-        configuration.update(local_context);
-        configuration.static_configuration = true;
-
-        try
-        {
-            if (keys.empty())
-                configuration.keys = getDataFiles(configuration, local_context);
-            else
-                configuration.keys = keys;
-
-            LOG_TRACE(
-                getLogger("DataLake"),
-                "New configuration path: {}, keys: {}",
-                configuration.getPath(), fmt::join(configuration.keys, ", "));
-
-            configuration.connect(local_context);
-            return configuration;
-        }
-        catch (...)
-        {
-            if (mode <= LoadingStrictnessLevel::CREATE)
-                throw;
-            tryLogCurrentException(__PRETTY_FUNCTION__);
-            return configuration;
-        }
-    }
-
-    static Strings getDataFiles(const Configuration & configuration, const ContextPtr & local_context)
-    {
-        return MetadataParser().getFiles(configuration, local_context);
-    }
-
-    void updateConfigurationImpl(const ContextPtr & local_context)
-    {
-        const bool updated = base_configuration.update(local_context);
-        auto new_keys = getDataFiles(base_configuration, local_context);
-
-        if (!updated && new_keys == Storage::getConfigurationCopy().keys)
-            return;
-
-        Storage::useConfiguration(getConfigurationForDataRead(base_configuration, local_context, new_keys));
-    }
-
-    Configuration base_configuration;
-    std::mutex configuration_update_mutex;
-    LoggerPtr log;
-};
-
-
-template <typename DataLake>
-static StoragePtr createDataLakeStorage(const StorageFactory::Arguments & args)
-{
-    auto configuration = DataLake::getConfiguration(args.engine_args, args.getLocalContext());
-
-    /// Data lakes use parquet format, no need for schema inference.
-    if (configuration.format == "auto")
-        configuration.format = "Parquet";
-
-    return DataLake::create(configuration, args.getContext(), args.mode, args.table_id, args.columns, args.constraints,
-        args.comment, getFormatSettings(args.getContext()));
-}
-
-}
-
-#endif

src/Storages/DataLakes/Iceberg/StorageIceberg.cpp

@@ -1,90 +0,0 @@
-#include <Storages/DataLakes/Iceberg/StorageIceberg.h>
-
-#if USE_AWS_S3 && USE_AVRO
-
-namespace DB
-{
-
-StoragePtr StorageIceberg::create(
-    const DB::StorageIceberg::Configuration & base_configuration,
-    DB::ContextPtr context_,
-    LoadingStrictnessLevel mode,
-    const DB::StorageID & table_id_,
-    const DB::ColumnsDescription & columns_,
-    const DB::ConstraintsDescription & constraints_,
-    const String & comment,
-    std::optional<FormatSettings> format_settings_)
-{
-    auto configuration{base_configuration};
-    configuration.update(context_);
-    std::unique_ptr<IcebergMetadata> metadata;
-    NamesAndTypesList schema_from_metadata;
-    try
-    {
-        metadata = parseIcebergMetadata(configuration, context_);
-        schema_from_metadata = metadata->getTableSchema();
-        configuration.keys = metadata->getDataFiles();
-    }
-    catch (...)
-    {
-        if (mode <= LoadingStrictnessLevel::CREATE)
-            throw;
-        tryLogCurrentException(__PRETTY_FUNCTION__);
-    }
-
-    return std::make_shared<StorageIceberg>(
-        std::move(metadata),
-        configuration,
-        context_,
-        table_id_,
-        columns_.empty() ? ColumnsDescription(schema_from_metadata) : columns_,
-        constraints_,
-        comment,
-        format_settings_);
-}
-
-StorageIceberg::StorageIceberg(
-    std::unique_ptr<IcebergMetadata> metadata_,
-    const Configuration & configuration_,
-    ContextPtr context_,
-    const StorageID & table_id_,
-    const ColumnsDescription & columns_,
-    const ConstraintsDescription & constraints_,
-    const String & comment,
-    std::optional<FormatSettings> format_settings_)
-    : StorageS3(configuration_, context_, table_id_, columns_, constraints_, comment, format_settings_)
-    , current_metadata(std::move(metadata_))
-    , base_configuration(configuration_)
-{
-}
-
-ColumnsDescription StorageIceberg::getTableStructureFromData(
-    Configuration & base_configuration,
-    const std::optional<FormatSettings> &,
-    const ContextPtr & local_context)
-{
-    auto configuration{base_configuration};
-    configuration.update(local_context);
-    auto metadata = parseIcebergMetadata(configuration, local_context);
-    return ColumnsDescription(metadata->getTableSchema());
-}
-
-void StorageIceberg::updateConfigurationImpl(const ContextPtr & local_context)
-{
-    const bool updated = base_configuration.update(local_context);
-    auto new_metadata = parseIcebergMetadata(base_configuration, local_context);
-
-    if (!current_metadata || new_metadata->getVersion() != current_metadata->getVersion())
-        current_metadata = std::move(new_metadata);
-    else if (!updated)
-        return;
-
-    auto updated_configuration{base_configuration};
-    /// If metadata wasn't changed, we won't list data files again.
-    updated_configuration.keys = current_metadata->getDataFiles();
-    StorageS3::useConfiguration(updated_configuration);
-}
-
-}
-
-#endif

src/Storages/DataLakes/Iceberg/StorageIceberg.h

@@ -1,85 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#if USE_AWS_S3 && USE_AVRO
-
-#    include <filesystem>
-#    include <Formats/FormatFactory.h>
-#    include <Storages/DataLakes/Iceberg/IcebergMetadata.h>
-#    include <Storages/IStorage.h>
-#    include <Storages/StorageFactory.h>
-#    include <Storages/StorageS3.h>
-#    include <Common/logger_useful.h>
-
-
-namespace DB
-{
-
-/// Storage for read-only integration with Apache Iceberg tables in Amazon S3 (see https://iceberg.apache.org/)
-/// Right now it's implemented on top of StorageS3 and right now it doesn't support
-/// many Iceberg features like schema evolution, partitioning, positional and equality deletes.
-/// TODO: Implement Iceberg as a separate storage using IObjectStorage
-/// (to support all object storages, not only S3) and add support for missing Iceberg features.
-class StorageIceberg : public StorageS3
-{
-public:
-    static constexpr auto name = "Iceberg";
-
-    using Configuration = StorageS3::Configuration;
-
-    static StoragePtr create(const Configuration & base_configuration,
-        ContextPtr context_,
-        LoadingStrictnessLevel mode,
-        const StorageID & table_id_,
-        const ColumnsDescription & columns_,
-        const ConstraintsDescription & constraints_,
-        const String & comment,
-        std::optional<FormatSettings> format_settings_);
-
-    StorageIceberg(
-        std::unique_ptr<IcebergMetadata> metadata_,
-        const Configuration & configuration_,
-        ContextPtr context_,
-        const StorageID & table_id_,
-        const ColumnsDescription & columns_,
-        const ConstraintsDescription & constraints_,
-        const String & comment,
-        std::optional<FormatSettings> format_settings_);
-
-    String getName() const override { return name; }
-
-    static ColumnsDescription getTableStructureFromData(
-        Configuration & base_configuration,
-        const std::optional<FormatSettings> &,
-        const ContextPtr & local_context);
-
-    static Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context)
-    {
-        return StorageS3::getConfiguration(engine_args, local_context, /* get_format_from_file */false);
-    }
-
-    Configuration updateConfigurationAndGetCopy(const ContextPtr & local_context) override
-    {
-        std::lock_guard lock(configuration_update_mutex);
-        updateConfigurationImpl(local_context);
-        return StorageS3::getConfigurationCopy();
-    }
-
-    void updateConfiguration(const ContextPtr & local_context) override
-    {
-        std::lock_guard lock(configuration_update_mutex);
-        updateConfigurationImpl(local_context);
-    }
-
-private:
-    void updateConfigurationImpl(const ContextPtr & local_context);
-
-    std::unique_ptr<IcebergMetadata> current_metadata;
-    Configuration base_configuration;
-    std::mutex configuration_update_mutex;
-};
-
-}
-
-#endif

src/Storages/DataLakes/S3MetadataReader.cpp

@@ -1,87 +0,0 @@
-#include <config.h>
-
-#if USE_AWS_S3
-
-#include <IO/ReadBufferFromS3.h>
-#include <IO/S3/Requests.h>
-#include <Interpreters/Context.h>
-#include <Storages/DataLakes/S3MetadataReader.h>
-#include <aws/core/auth/AWSCredentials.h>
-#include <aws/s3/S3Client.h>
-#include <aws/s3/model/ListObjectsV2Request.h>
-
-#include <filesystem>
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-    extern const int S3_ERROR;
-}
-
-std::shared_ptr<ReadBuffer>
-S3DataLakeMetadataReadHelper::createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration)
-{
-    S3Settings::RequestSettings request_settings;
-    request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries;
-    return std::make_shared<ReadBufferFromS3>(
-        base_configuration.client,
-        base_configuration.url.bucket,
-        key,
-        base_configuration.url.version_id,
-        request_settings,
-        context->getReadSettings());
-}
-
-bool S3DataLakeMetadataReadHelper::exists(const String & key, const StorageS3::Configuration & configuration)
-{
-    return S3::objectExists(*configuration.client, configuration.url.bucket, key);
-}
-
-std::vector<String> S3DataLakeMetadataReadHelper::listFiles(
-    const StorageS3::Configuration & base_configuration, const String & prefix, const String & suffix)
-{
-    const auto & table_path = base_configuration.url.key;
-    const auto & bucket = base_configuration.url.bucket;
-    const auto & client = base_configuration.client;
-
-    std::vector<String> res;
-    S3::ListObjectsV2Request request;
-    Aws::S3::Model::ListObjectsV2Outcome outcome;
-
-    request.SetBucket(bucket);
-    request.SetPrefix(std::filesystem::path(table_path) / prefix);
-
-    bool is_finished{false};
-    while (!is_finished)
-    {
-        outcome = client->ListObjectsV2(request);
-        if (!outcome.IsSuccess())
-            throw S3Exception(
-                outcome.GetError().GetErrorType(),
-                "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}",
-                quoteString(bucket),
-                quoteString(base_configuration.url.key),
-                backQuote(outcome.GetError().GetExceptionName()),
-                quoteString(outcome.GetError().GetMessage()));
-
-        const auto & result_batch = outcome.GetResult().GetContents();
-        for (const auto & obj : result_batch)
-        {
-            const auto & filename = obj.GetKey();
-            if (filename.ends_with(suffix))
-                res.push_back(filename);
-        }
-
-        request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken());
-        is_finished = !outcome.GetResult().GetIsTruncated();
-    }
-
-    LOG_TRACE(getLogger("S3DataLakeMetadataReadHelper"), "Listed {} files", res.size());
-
-    return res;
-}
-
-}
-#endif

src/Storages/DataLakes/S3MetadataReader.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include <config.h>
-
-#if USE_AWS_S3
-
-#include <Storages/StorageS3.h>
-
-namespace DB
-{
-
-class ReadBuffer;
-
-struct S3DataLakeMetadataReadHelper
-{
-    static std::shared_ptr<ReadBuffer> createReadBuffer(
-        const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration);
-
-    static bool exists(const String & key, const StorageS3::Configuration & configuration);
-
-    static std::vector<String> listFiles(const StorageS3::Configuration & configuration, const std::string & prefix = "", const std::string & suffix = "");
-};
-}
-
-#endif

src/Storages/DataLakes/StorageDeltaLake.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include <Storages/IStorage.h>
-#include <Storages/DataLakes/IStorageDataLake.h>
-#include <Storages/DataLakes/DeltaLakeMetadataParser.h>
-#include "config.h"
-
-#if USE_AWS_S3
-#include <Storages/DataLakes/S3MetadataReader.h>
-#include <Storages/StorageS3.h>
-#endif
-
-namespace DB
-{
-
-struct StorageDeltaLakeName
-{
-    static constexpr auto name = "DeltaLake";
-};
-
-#if USE_AWS_S3 && USE_PARQUET
-using StorageDeltaLakeS3 = IStorageDataLake<StorageS3, StorageDeltaLakeName, DeltaLakeMetadataParser<StorageS3::Configuration, S3DataLakeMetadataReadHelper>>;
-#endif
-
-}

src/Storages/DataLakes/StorageHudi.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include <Storages/IStorage.h>
-#include <Storages/DataLakes/IStorageDataLake.h>
-#include <Storages/DataLakes/HudiMetadataParser.h>
-#include "config.h"
-
-#if USE_AWS_S3
-#include <Storages/DataLakes/S3MetadataReader.h>
-#include <Storages/StorageS3.h>
-#endif
-
-namespace DB
-{
-
-struct StorageHudiName
-{
-    static constexpr auto name = "Hudi";
-};
-
-#if USE_AWS_S3
-using StorageHudiS3 = IStorageDataLake<StorageS3, StorageHudiName, HudiMetadataParser<StorageS3::Configuration, S3DataLakeMetadataReadHelper>>;
-#endif
-
-}

src/Storages/DataLakes/registerDataLakes.cpp

@@ -1,50 +0,0 @@
-#include <Storages/DataLakes/IStorageDataLake.h>
-#include "config.h"
-
-#if USE_AWS_S3
-
-#include <Storages/DataLakes/StorageDeltaLake.h>
-#include <Storages/DataLakes/Iceberg/StorageIceberg.h>
-#include <Storages/DataLakes/StorageHudi.h>
-
-
-namespace DB
-{
-
-#define REGISTER_DATA_LAKE_STORAGE(STORAGE, NAME)       \
-    factory.registerStorage(                            \
-        NAME,                                           \
-        [](const StorageFactory::Arguments & args)      \
-        {                                               \
-            return createDataLakeStorage<STORAGE>(args);\
-        },                                              \
-        {                                               \
-            .supports_settings = false,                 \
-            .supports_schema_inference = true,          \
-            .source_access_type = AccessType::S3,       \
-        });
-
-#if USE_PARQUET
-void registerStorageDeltaLake(StorageFactory & factory)
-{
-    REGISTER_DATA_LAKE_STORAGE(StorageDeltaLakeS3, StorageDeltaLakeName::name)
-}
-#endif
-
-#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format.
-
-void registerStorageIceberg(StorageFactory & factory)
-{
-    REGISTER_DATA_LAKE_STORAGE(StorageIceberg, StorageIceberg::name)
-}
-
-#endif
-
-void registerStorageHudi(StorageFactory & factory)
-{
-    REGISTER_DATA_LAKE_STORAGE(StorageHudiS3, StorageHudiName::name)
-}
-
-}
-
-#endif

src/Storages/HDFS/StorageHDFS.h

@@ -1,190 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#if USE_HDFS
-
-#include <Processors/ISource.h>
-#include <Storages/IStorage.h>
-#include <Storages/Cache/SchemaCache.h>
-#include <Storages/prepareReadingFromFormat.h>
-#include <Storages/SelectQueryInfo.h>
-#include <Poco/URI.h>
-
-namespace DB
-{
-
-class IInputFormat;
-
-/**
- * This class represents table engine for external hdfs files.
- * Read method is supported for now.
- */
-class StorageHDFS final : public IStorage, WithContext
-{
-public:
-    struct PathInfo
-    {
-        time_t last_mod_time;
-        size_t size;
-    };
-
-    struct PathWithInfo
-    {
-        PathWithInfo() = default;
-        PathWithInfo(const String & path_, const std::optional<PathInfo> & info_) : path(path_), info(info_) {}
-        String path;
-        std::optional<PathInfo> info;
-    };
-
-    StorageHDFS(
-        const String & uri_,
-        const StorageID & table_id_,
-        const String & format_name_,
-        const ColumnsDescription & columns_,
-        const ConstraintsDescription & constraints_,
-        const String & comment,
-        const ContextPtr & context_,
-        const String & compression_method_ = "",
-        bool distributed_processing_ = false,
-        ASTPtr partition_by = nullptr);
-
-    String getName() const override { return "HDFS"; }
-
-    void read(
-        QueryPlan & query_plan,
-        const Names & column_names,
-        const StorageSnapshotPtr & storage_snapshot,
-        SelectQueryInfo & query_info,
-        ContextPtr context,
-        QueryProcessingStage::Enum processed_stage,
-        size_t max_block_size,
-        size_t num_streams) override;
-
-    SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, bool async_insert) override;
-
-    void truncate(
-        const ASTPtr & query,
-        const StorageMetadataPtr & metadata_snapshot,
-        ContextPtr local_context,
-        TableExclusiveLockHolder &) override;
-
-    bool supportsPartitionBy() const override { return true; }
-
-    /// Check if the format is column-oriented.
-    /// Is is useful because column oriented formats could effectively skip unknown columns
-    /// So we can create a header of only required columns in read method and ask
-    /// format to read only them. Note: this hack cannot be done with ordinary formats like TSV.
-    bool supportsSubsetOfColumns(const ContextPtr & context_) const;
-
-    bool supportsSubcolumns() const override { return true; }
-
-    bool supportsDynamicSubcolumns() const override { return true; }
-
-    static ColumnsDescription getTableStructureFromData(
-        const String & format,
-        const String & uri,
-        const String & compression_method,
-        const ContextPtr & ctx);
-
-    static std::pair<ColumnsDescription, String> getTableStructureAndFormatFromData(
-        const String & uri,
-        const String & compression_method,
-        const ContextPtr & ctx);
-
-    static SchemaCache & getSchemaCache(const ContextPtr & ctx);
-
-    bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; }
-
-protected:
-    friend class HDFSSource;
-    friend class ReadFromHDFS;
-
-private:
-    static std::pair<ColumnsDescription, String> getTableStructureAndFormatFromDataImpl(
-        std::optional<String> format,
-        const String & uri,
-        const String & compression_method,
-        const ContextPtr & ctx);
-
-    std::vector<String> uris;
-    String format_name;
-    String compression_method;
-    const bool distributed_processing;
-    ASTPtr partition_by;
-    bool is_path_with_globs;
-
-    LoggerPtr log = getLogger("StorageHDFS");
-};
-
-class PullingPipelineExecutor;
-
-class HDFSSource : public ISource, WithContext
-{
-public:
-    class DisclosedGlobIterator
-    {
-        public:
-            DisclosedGlobIterator(const String & uri_, const ActionsDAG::Node * predicate, const NamesAndTypesList & virtual_columns, const ContextPtr & context);
-            StorageHDFS::PathWithInfo next();
-        private:
-            class Impl;
-            /// shared_ptr to have copy constructor
-            std::shared_ptr<Impl> pimpl;
-    };
-
-    class URISIterator
-    {
-        public:
-            URISIterator(const std::vector<String> & uris_, const ActionsDAG::Node * predicate, const NamesAndTypesList & virtual_columns, const ContextPtr & context);
-            StorageHDFS::PathWithInfo next();
-        private:
-            class Impl;
-            /// shared_ptr to have copy constructor
-            std::shared_ptr<Impl> pimpl;
-    };
-
-    using IteratorWrapper = std::function<StorageHDFS::PathWithInfo()>;
-    using StorageHDFSPtr = std::shared_ptr<StorageHDFS>;
-
-    HDFSSource(
-        const ReadFromFormatInfo & info,
-        StorageHDFSPtr storage_,
-        const ContextPtr & context_,
-        UInt64 max_block_size_,
-        std::shared_ptr<IteratorWrapper> file_iterator_,
-        bool need_only_count_);
-
-    ~HDFSSource() override;
-
-    String getName() const override;
-
-    Chunk generate() override;
-
-private:
-    void addNumRowsToCache(const String & path, size_t num_rows);
-    std::optional<size_t> tryGetNumRowsFromCache(const StorageHDFS::PathWithInfo & path_with_info);
-
-    StorageHDFSPtr storage;
-    Block block_for_format;
-    NamesAndTypesList requested_columns;
-    NamesAndTypesList requested_virtual_columns;
-    UInt64 max_block_size;
-    std::shared_ptr<IteratorWrapper> file_iterator;
-    ColumnsDescription columns_description;
-    bool need_only_count;
-    size_t total_rows_in_file = 0;
-
-    std::unique_ptr<ReadBuffer> read_buf;
-    std::shared_ptr<IInputFormat> input_format;
-    std::unique_ptr<QueryPipeline> pipeline;
-    std::unique_ptr<PullingPipelineExecutor> reader;
-    String current_path;
-    std::optional<size_t> current_file_size;
-
-    /// Recreate ReadBuffer and PullingPipelineExecutor for each file.
-    bool initialize();
-};
-}
-
-#endif

src/Storages/HDFS/StorageHDFSCluster.cpp

@@ -1,98 +0,0 @@
-#include "config.h"
-#include "Interpreters/Context_fwd.h"
-
-#if USE_HDFS
-
-#include <Storages/HDFS/StorageHDFSCluster.h>
-
-#include <Core/QueryProcessingStage.h>
-#include <DataTypes/DataTypeString.h>
-#include <Interpreters/getHeaderForProcessingStage.h>
-#include <Interpreters/InterpreterSelectQuery.h>
-#include <QueryPipeline/RemoteQueryExecutor.h>
-
-#include <Processors/Transforms/AddingDefaultsTransform.h>
-
-#include <Processors/Sources/RemoteSource.h>
-#include <Parsers/ASTTablesInSelectQuery.h>
-#include <Parsers/queryToString.h>
-
-#include <Storages/HDFS/HDFSCommon.h>
-#include <Storages/IStorage.h>
-#include <Storages/SelectQueryInfo.h>
-#include <Storages/extractTableFunctionArgumentsFromSelectQuery.h>
-#include <Storages/VirtualColumnUtils.h>
-
-#include <TableFunctions/TableFunctionHDFSCluster.h>
-#include <memory>
-
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-    extern const int LOGICAL_ERROR;
-}
-
-StorageHDFSCluster::StorageHDFSCluster(
-    ContextPtr context_,
-    const String & cluster_name_,
-    const String & uri_,
-    const StorageID & table_id_,
-    const String & format_name_,
-    const ColumnsDescription & columns_,
-    const ConstraintsDescription & constraints_,
-    const String & compression_method)
-    : IStorageCluster(cluster_name_, table_id_, getLogger("StorageHDFSCluster (" + table_id_.table_name + ")"))
-    , uri(uri_)
-    , format_name(format_name_)
-{
-    checkHDFSURL(uri_);
-    context_->getRemoteHostFilter().checkURL(Poco::URI(uri_));
-
-    StorageInMemoryMetadata storage_metadata;
-
-    if (columns_.empty())
-    {
-        ColumnsDescription columns;
-        if (format_name == "auto")
-            std::tie(columns, format_name) = StorageHDFS::getTableStructureAndFormatFromData(uri_, compression_method, context_);
-        else
-            columns = StorageHDFS::getTableStructureFromData(format_name, uri_, compression_method, context_);
-        storage_metadata.setColumns(columns);
-    }
-    else
-    {
-        if (format_name == "auto")
-            format_name = StorageHDFS::getTableStructureAndFormatFromData(uri_, compression_method, context_).second;
-
-        storage_metadata.setColumns(columns_);
-    }
-
-    storage_metadata.setConstraints(constraints_);
-    setInMemoryMetadata(storage_metadata);
-    setVirtuals(VirtualColumnUtils::getVirtualsForFileLikeStorage(storage_metadata.getColumns()));
-}
-
-void StorageHDFSCluster::updateQueryToSendIfNeeded(DB::ASTPtr & query, const DB::StorageSnapshotPtr & storage_snapshot, const DB::ContextPtr & context)
-{
-    ASTExpressionList * expression_list = extractTableFunctionArgumentsFromSelectQuery(query);
-    if (!expression_list)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected SELECT query from table function hdfsCluster, got '{}'", queryToString(query));
-
-    TableFunctionHDFSCluster::updateStructureAndFormatArgumentsIfNeeded(
-        expression_list->children, storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), format_name, context);
-}
-
-
-RemoteQueryExecutor::Extension StorageHDFSCluster::getTaskIteratorExtension(const ActionsDAG::Node * predicate, const ContextPtr & context) const
-{
-    auto iterator = std::make_shared<HDFSSource::DisclosedGlobIterator>(uri, predicate, getVirtualsList(), context);
-    auto callback = std::make_shared<std::function<String()>>([iter = std::move(iterator)]() mutable -> String { return iter->next().path; });
-    return RemoteQueryExecutor::Extension{.task_iterator = std::move(callback)};
-}
-
-}
-
-#endif

src/Storages/HDFS/StorageHDFSCluster.h

@@ -1,53 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#if USE_HDFS
-
-#include <memory>
-#include <optional>
-
-#include <Client/Connection.h>
-#include <Interpreters/Cluster.h>
-#include <Storages/IStorageCluster.h>
-#include <Storages/HDFS/StorageHDFS.h>
-
-namespace DB
-{
-
-class Context;
-
-class StorageHDFSCluster : public IStorageCluster
-{
-public:
-    StorageHDFSCluster(
-        ContextPtr context_,
-        const String & cluster_name_,
-        const String & uri_,
-        const StorageID & table_id_,
-        const String & format_name_,
-        const ColumnsDescription & columns_,
-        const ConstraintsDescription & constraints_,
-        const String & compression_method);
-
-    std::string getName() const override { return "HDFSCluster"; }
-
-    RemoteQueryExecutor::Extension getTaskIteratorExtension(const ActionsDAG::Node * predicate, const ContextPtr & context) const override;
-
-    bool supportsSubcolumns() const override { return true; }
-
-    bool supportsDynamicSubcolumns() const override { return true; }
-
-    bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; }
-
-private:
-    void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override;
-
-    String uri;
-    String format_name;
-};
-
-
-}
-
-#endif

src/Storages/Hive/HiveCommon.h

@@ -12,7 +12,7 @@
 #include <base/types.h>
 #include <Common/CacheBase.h>
 #include <Common/PoolBase.h>
-#include <Storages/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
 #include <Storages/Hive/HiveFile.h>
 
 

src/Storages/Hive/HiveFile.h

@@ -14,7 +14,7 @@
 #include <Core/Block.h>
 #include <Storages/MergeTree/IMergeTreeDataPart.h>
 #include <Storages/Hive/HiveSettings.h>
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
 
 namespace orc
 {

src/Storages/Hive/StorageHive.cpp

@@ -38,8 +38,8 @@
 #include <Processors/QueryPlan/SourceStepWithFilter.h>
 #include <Processors/Sources/NullSource.h>
 #include <Storages/AlterCommands.h>
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
-#include <Storages/HDFS/AsynchronousReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h>
 #include <Storages/Hive/HiveSettings.h>
 #include <Storages/Hive/StorageHiveMetadata.h>
 #include <Storages/MergeTree/KeyCondition.h>

src/Storages/Hive/StorageHive.h

@@ -9,7 +9,7 @@
 
 #include <Interpreters/Context.h>
 #include <Storages/IStorage.h>
-#include <Storages/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
 #include <Storages/Hive/HiveCommon.h>
 #include <Storages/Hive/HiveFile.h>
 

src/Storages/IStorage.cpp

@@ -27,11 +27,14 @@ namespace ErrorCodes
     extern const int CANNOT_RESTORE_TABLE;
 }
 
-IStorage::IStorage(StorageID storage_id_)
+IStorage::IStorage(StorageID storage_id_, std::unique_ptr<StorageInMemoryMetadata> metadata_)
     : storage_id(std::move(storage_id_))
-    , metadata(std::make_unique<StorageInMemoryMetadata>())
     , virtuals(std::make_unique<VirtualColumnsDescription>())
 {
+    if (metadata_)
+        metadata.set(std::move(metadata_));
+    else
+        metadata.set(std::make_unique<StorageInMemoryMetadata>());
 }
 
 bool IStorage::isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const

src/Storages/IStorage.h

@@ -99,7 +99,7 @@ class IStorage : public std::enable_shared_from_this<IStorage>, public TypePromo
 public:
     IStorage() = delete;
     /// Storage metadata can be set separately in setInMemoryMetadata method
-    explicit IStorage(StorageID storage_id_);
+    explicit IStorage(StorageID storage_id_, std::unique_ptr<StorageInMemoryMetadata> metadata_ = nullptr);
 
     IStorage(const IStorage &) = delete;
     IStorage & operator=(const IStorage &) = delete;
@@ -261,6 +261,9 @@ public:
     /// Return true if storage can execute lightweight delete mutations.
     virtual bool supportsLightweightDelete() const { return false; }
 
+    /// Return true if storage has any projection.
+    virtual bool hasProjection() const { return false; }
+
     /// Return true if storage can execute 'DELETE FROM' mutations. This is different from lightweight delete
     /// because those are internally translated into 'ALTER UDPATE' mutations.
     virtual bool supportsDelete() const { return false; }

src/Storages/MergeTree/IMergeTreeDataPart.h

@@ -43,7 +43,6 @@ class IReservation;
 using ReservationPtr = std::unique_ptr<IReservation>;
 
 class IMergeTreeReader;
-class IMergeTreeDataPartWriter;
 class MarkCache;
 class UncompressedCache;
 class MergeTreeTransaction;
@@ -74,7 +73,6 @@ public:
     using VirtualFields = std::unordered_map<String, Field>;
 
     using MergeTreeReaderPtr = std::unique_ptr<IMergeTreeReader>;
-    using MergeTreeWriterPtr = std::unique_ptr<IMergeTreeDataPartWriter>;
 
     using ColumnSizeByName = std::unordered_map<std::string, ColumnSize>;
     using NameToNumber = std::unordered_map<std::string, size_t>;
@@ -106,15 +104,6 @@ public:
         const ValueSizeMap & avg_value_size_hints_,
         const ReadBufferFromFileBase::ProfileCallback & profile_callback_) const = 0;
 
-    virtual MergeTreeWriterPtr getWriter(
-        const NamesAndTypesList & columns_list,
-        const StorageMetadataPtr & metadata_snapshot,
-        const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
-        const Statistics & stats_to_recalc_,
-        const CompressionCodecPtr & default_codec_,
-        const MergeTreeWriterSettings & writer_settings,
-        const MergeTreeIndexGranularity & computed_index_granularity) = 0;
-
     virtual bool isStoredOnDisk() const = 0;
 
     virtual bool isStoredOnRemoteDisk() const = 0;
@@ -172,6 +161,8 @@ public:
 
     const SerializationInfoByName & getSerializationInfos() const { return serialization_infos; }
 
+    const SerializationByName & getSerializations() const { return serializations; }
+
     SerializationPtr getSerialization(const String & column_name) const;
     SerializationPtr tryGetSerialization(const String & column_name) const;
 
@@ -201,6 +192,7 @@ public:
     /// take place, you must take original name of column for this part from
     /// storage and pass it to this method.
     std::optional<size_t> getColumnPosition(const String & column_name) const;
+    const NameToNumber & getColumnPositions() const { return column_name_to_position; }
 
     /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()).
     /// If no checksums are present returns the name of the first physically existing column.
@@ -446,6 +438,8 @@ public:
 
     bool hasProjection(const String & projection_name) const { return projection_parts.contains(projection_name); }
 
+    bool hasProjection() const { return !projection_parts.empty(); }
+
     bool hasBrokenProjection(const String & projection_name) const;
 
     /// Return true, if all projections were loaded successfully and none was marked as broken.

src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp

@@ -3,6 +3,13 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int NO_SUCH_COLUMN_IN_TABLE;
+}
+
+
 Block getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation)
 {
     Block result;
@@ -38,18 +45,27 @@ Block permuteBlockIfNeeded(const Block & block, const IColumn::Permutation * per
 }
 
 IMergeTreeDataPartWriter::IMergeTreeDataPartWriter(
-    const MergeTreeMutableDataPartPtr & data_part_,
+    const String & data_part_name_,
+    const SerializationByName & serializations_,
+    MutableDataPartStoragePtr data_part_storage_,
+    const MergeTreeIndexGranularityInfo & index_granularity_info_,
+    const MergeTreeSettingsPtr & storage_settings_,
     const NamesAndTypesList & columns_list_,
     const StorageMetadataPtr & metadata_snapshot_,
+    const VirtualsDescriptionPtr & virtual_columns_,
     const MergeTreeWriterSettings & settings_,
     const MergeTreeIndexGranularity & index_granularity_)
-    : data_part(data_part_)
-    , storage(data_part_->storage)
+    : data_part_name(data_part_name_)
+    , serializations(serializations_)
+    , index_granularity_info(index_granularity_info_)
+    , storage_settings(storage_settings_)
     , metadata_snapshot(metadata_snapshot_)
+    , virtual_columns(virtual_columns_)
     , columns_list(columns_list_)
     , settings(settings_)
-    , index_granularity(index_granularity_)
     , with_final_mark(settings.can_use_adaptive_granularity)
+    , data_part_storage(data_part_storage_)
+    , index_granularity(index_granularity_)
 {
 }
 
@@ -60,6 +76,102 @@ Columns IMergeTreeDataPartWriter::releaseIndexColumns()
         std::make_move_iterator(index_columns.end()));
 }
 
+SerializationPtr IMergeTreeDataPartWriter::getSerialization(const String & column_name) const
+{
+    auto it = serializations.find(column_name);
+    if (it == serializations.end())
+        throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE,
+            "There is no column or subcolumn {} in part {}", column_name, data_part_name);
+
+    return it->second;
+}
+
+ASTPtr IMergeTreeDataPartWriter::getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec) const
+{
+    auto get_codec_or_default = [&](const auto & column_desc)
+    {
+        return column_desc.codec ? column_desc.codec : default_codec->getFullCodecDesc();
+    };
+
+    const auto & columns = metadata_snapshot->getColumns();
+    if (const auto * column_desc = columns.tryGet(column_name))
+        return get_codec_or_default(*column_desc);
+
+    if (const auto * virtual_desc = virtual_columns->tryGetDescription(column_name))
+        return get_codec_or_default(*virtual_desc);
+
+    return default_codec->getFullCodecDesc();
+}
+
+
 IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default;
 
+
+MergeTreeDataPartWriterPtr createMergeTreeDataPartCompactWriter(
+        const String & data_part_name_,
+        const String & logger_name_,
+        const SerializationByName & serializations_,
+        MutableDataPartStoragePtr data_part_storage_,
+        const MergeTreeIndexGranularityInfo & index_granularity_info_,
+        const MergeTreeSettingsPtr & storage_settings_,
+        const NamesAndTypesList & columns_list,
+        const ColumnPositions & column_positions,
+        const StorageMetadataPtr & metadata_snapshot,
+        const VirtualsDescriptionPtr & virtual_columns,
+        const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
+        const Statistics & stats_to_recalc_,
+        const String & marks_file_extension_,
+        const CompressionCodecPtr & default_codec_,
+        const MergeTreeWriterSettings & writer_settings,
+        const MergeTreeIndexGranularity & computed_index_granularity);
+
+MergeTreeDataPartWriterPtr createMergeTreeDataPartWideWriter(
+        const String & data_part_name_,
+        const String & logger_name_,
+        const SerializationByName & serializations_,
+        MutableDataPartStoragePtr data_part_storage_,
+        const MergeTreeIndexGranularityInfo & index_granularity_info_,
+        const MergeTreeSettingsPtr & storage_settings_,
+        const NamesAndTypesList & columns_list,
+        const StorageMetadataPtr & metadata_snapshot,
+        const VirtualsDescriptionPtr & virtual_columns,
+        const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
+        const Statistics & stats_to_recalc_,
+        const String & marks_file_extension_,
+        const CompressionCodecPtr & default_codec_,
+        const MergeTreeWriterSettings & writer_settings,
+        const MergeTreeIndexGranularity & computed_index_granularity);
+
+
+MergeTreeDataPartWriterPtr createMergeTreeDataPartWriter(
+        MergeTreeDataPartType part_type,
+        const String & data_part_name_,
+        const String & logger_name_,
+        const SerializationByName & serializations_,
+        MutableDataPartStoragePtr data_part_storage_,
+        const MergeTreeIndexGranularityInfo & index_granularity_info_,
+        const MergeTreeSettingsPtr & storage_settings_,
+        const NamesAndTypesList & columns_list,
+        const ColumnPositions & column_positions,
+        const StorageMetadataPtr & metadata_snapshot,
+        const VirtualsDescriptionPtr & virtual_columns,
+        const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
+        const Statistics & stats_to_recalc_,
+        const String & marks_file_extension_,
+        const CompressionCodecPtr & default_codec_,
+        const MergeTreeWriterSettings & writer_settings,
+        const MergeTreeIndexGranularity & computed_index_granularity)
+{
+    if (part_type == MergeTreeDataPartType::Compact)
+        return createMergeTreeDataPartCompactWriter(data_part_name_, logger_name_, serializations_, data_part_storage_,
+            index_granularity_info_, storage_settings_, columns_list, column_positions, metadata_snapshot, virtual_columns, indices_to_recalc, stats_to_recalc_,
+            marks_file_extension_, default_codec_, writer_settings, computed_index_granularity);
+    else if (part_type == MergeTreeDataPartType::Wide)
+        return createMergeTreeDataPartWideWriter(data_part_name_, logger_name_, serializations_, data_part_storage_,
+            index_granularity_info_, storage_settings_, columns_list, metadata_snapshot, virtual_columns, indices_to_recalc, stats_to_recalc_,
+            marks_file_extension_, default_codec_, writer_settings, computed_index_granularity);
+    else
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown part type: {}", part_type.toString());
+}
+
 }

src/Storages/MergeTree/IMergeTreeDataPartWriter.h

@@ -1,12 +1,13 @@
 #pragma once
 
-#include <IO/WriteBufferFromFile.h>
-#include <IO/WriteBufferFromFileBase.h>
-#include <Compression/CompressedWriteBuffer.h>
-#include <IO/HashingWriteBuffer.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-#include <Storages/MergeTree/IMergeTreeDataPart.h>
-#include <Disks/IDisk.h>
+#include <Storages/MergeTree/MergeTreeDataPartType.h>
+#include <Storages/MergeTree/MergeTreeSettings.h>
+#include <Storages/MergeTree/MergeTreeIndexGranularity.h>
+#include <Storages/MergeTree/MergeTreeIndexGranularityInfo.h>
+#include <Storages/MergeTree/MergeTreeIndices.h>
+#include <Storages/MergeTree/IDataPartStorage.h>
+#include <Storages/Statistics/Statistics.h>
+#include <Storages/VirtualColumnsDescription.h>
 
 
 namespace DB
@@ -22,9 +23,14 @@ class IMergeTreeDataPartWriter : private boost::noncopyable
 {
 public:
     IMergeTreeDataPartWriter(
-        const MergeTreeMutableDataPartPtr & data_part_,
+        const String & data_part_name_,
+        const SerializationByName & serializations_,
+        MutableDataPartStoragePtr data_part_storage_,
+        const MergeTreeIndexGranularityInfo & index_granularity_info_,
+        const MergeTreeSettingsPtr & storage_settings_,
         const NamesAndTypesList & columns_list_,
         const StorageMetadataPtr & metadata_snapshot_,
+        const VirtualsDescriptionPtr & virtual_columns_,
         const MergeTreeWriterSettings & settings_,
         const MergeTreeIndexGranularity & index_granularity_ = {});
 
@@ -32,7 +38,7 @@ public:
 
     virtual void write(const Block & block, const IColumn::Permutation * permutation) = 0;
 
-    virtual void fillChecksums(IMergeTreeDataPart::Checksums & checksums, NameSet & checksums_to_remove) = 0;
+    virtual void fillChecksums(MergeTreeDataPartChecksums & checksums, NameSet & checksums_to_remove) = 0;
 
     virtual void finish(bool sync) = 0;
 
@@ -40,16 +46,48 @@ public:
     const MergeTreeIndexGranularity & getIndexGranularity() const { return index_granularity; }
 
 protected:
+    SerializationPtr getSerialization(const String & column_name) const;
 
-    const MergeTreeMutableDataPartPtr data_part;
-    const MergeTreeData & storage;
+    ASTPtr getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec) const;
+
+    IDataPartStorage & getDataPartStorage() { return *data_part_storage; }
+
+    const String data_part_name;
+    /// Serializations for every columns and subcolumns by their names.
+    const SerializationByName serializations;
+    const MergeTreeIndexGranularityInfo index_granularity_info;
+    const MergeTreeSettingsPtr storage_settings;
     const StorageMetadataPtr metadata_snapshot;
+    const VirtualsDescriptionPtr virtual_columns;
     const NamesAndTypesList columns_list;
     const MergeTreeWriterSettings settings;
-    MergeTreeIndexGranularity index_granularity;
     const bool with_final_mark;
 
+    MutableDataPartStoragePtr data_part_storage;
     MutableColumns index_columns;
+    MergeTreeIndexGranularity index_granularity;
 };
 
+using MergeTreeDataPartWriterPtr = std::unique_ptr<IMergeTreeDataPartWriter>;
+using ColumnPositions = std::unordered_map<std::string, size_t>;
+
+MergeTreeDataPartWriterPtr createMergeTreeDataPartWriter(
+        MergeTreeDataPartType part_type,
+        const String & data_part_name_,
+        const String & logger_name_,
+        const SerializationByName & serializations_,
+        MutableDataPartStoragePtr data_part_storage_,
+        const MergeTreeIndexGranularityInfo & index_granularity_info_,
+        const MergeTreeSettingsPtr & storage_settings_,
+        const NamesAndTypesList & columns_list,
+        const ColumnPositions & column_positions,
+        const StorageMetadataPtr & metadata_snapshot,
+        const VirtualsDescriptionPtr & virtual_columns_,
+        const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
+        const Statistics & stats_to_recalc_,
+        const String & marks_file_extension,
+        const CompressionCodecPtr & default_codec_,
+        const MergeTreeWriterSettings & writer_settings,
+        const MergeTreeIndexGranularity & computed_index_granularity);
+
 }

src/Storages/MergeTree/IMergedBlockOutputStream.cpp

@@ -7,20 +7,21 @@ namespace DB
 {
 
 IMergedBlockOutputStream::IMergedBlockOutputStream(
-    const MergeTreeMutableDataPartPtr & data_part,
+    const MergeTreeSettingsPtr & storage_settings_,
+    MutableDataPartStoragePtr data_part_storage_,
     const StorageMetadataPtr & metadata_snapshot_,
     const NamesAndTypesList & columns_list,
     bool reset_columns_)
-    : storage(data_part->storage)
+    : storage_settings(storage_settings_)
     , metadata_snapshot(metadata_snapshot_)
-    , data_part_storage(data_part->getDataPartStoragePtr())
+    , data_part_storage(data_part_storage_)
     , reset_columns(reset_columns_)
 {
     if (reset_columns)
     {
         SerializationInfo::Settings info_settings =
         {
-            .ratio_of_defaults_for_sparse = storage.getSettings()->ratio_of_defaults_for_sparse_serialization,
+            .ratio_of_defaults_for_sparse = storage_settings->ratio_of_defaults_for_sparse_serialization,
             .choose_kind = false,
         };
 
@@ -42,7 +43,7 @@ NameSet IMergedBlockOutputStream::removeEmptyColumnsFromPart(
         return {};
 
     for (const auto & column : empty_columns)
-        LOG_TRACE(storage.log, "Skipping expired/empty column {} for part {}", column, data_part->name);
+        LOG_TRACE(data_part->storage.log, "Skipping expired/empty column {} for part {}", column, data_part->name);
 
     /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes.
     std::map<String, size_t> stream_counts;
@@ -91,7 +92,7 @@ NameSet IMergedBlockOutputStream::removeEmptyColumnsFromPart(
         }
         else /// If we have no file in checksums it doesn't exist on disk
         {
-            LOG_TRACE(storage.log, "Files {} doesn't exist in checksums so it doesn't exist on disk, will not try to remove it", *itr);
+            LOG_TRACE(data_part->storage.log, "Files {} doesn't exist in checksums so it doesn't exist on disk, will not try to remove it", *itr);
             itr = remove_files.erase(itr);
         }
     }

src/Storages/MergeTree/IMergedBlockOutputStream.h

@@ -1,10 +1,12 @@
 #pragma once
 
-#include "Storages/MergeTree/IDataPartStorage.h"
+#include <Storages/MergeTree/IDataPartStorage.h>
+#include <Storages/MergeTree/MergeTreeSettings.h>
 #include <Storages/MergeTree/MergeTreeIndexGranularity.h>
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/MergeTree/IMergeTreeDataPart.h>
 #include <Storages/MergeTree/IMergeTreeDataPartWriter.h>
+#include <Common/Logger.h>
 
 namespace DB
 {
@@ -13,7 +15,8 @@ class IMergedBlockOutputStream
 {
 public:
     IMergedBlockOutputStream(
-        const MergeTreeMutableDataPartPtr & data_part,
+        const MergeTreeSettingsPtr & storage_settings_,
+        MutableDataPartStoragePtr data_part_storage_,
         const StorageMetadataPtr & metadata_snapshot_,
         const NamesAndTypesList & columns_list,
         bool reset_columns_);
@@ -39,11 +42,13 @@ protected:
         SerializationInfoByName & serialization_infos,
         MergeTreeData::DataPart::Checksums & checksums);
 
-    const MergeTreeData & storage;
+    MergeTreeSettingsPtr storage_settings;
+    LoggerPtr log;
+
     StorageMetadataPtr metadata_snapshot;
 
     MutableDataPartStoragePtr data_part_storage;
-    IMergeTreeDataPart::MergeTreeWriterPtr writer;
+    MergeTreeDataPartWriterPtr writer;
 
     bool reset_columns = false;
     SerializationInfoByName new_serialization_infos;

src/Storages/MergeTree/KeyCondition.cpp

@@ -2664,6 +2664,13 @@ BoolMask KeyCondition::checkInHyperrectangle(
         else if (element.function == RPNElement::FUNCTION_IN_RANGE
             || element.function == RPNElement::FUNCTION_NOT_IN_RANGE)
         {
+            if (element.key_column >= hyperrectangle.size())
+            {
+                throw Exception(ErrorCodes::LOGICAL_ERROR,
+                                "Hyperrectangle size is {}, but requested element at posittion {} ({})",
+                                hyperrectangle.size(), element.key_column, element.toString());
+            }
+
             const Range * key_range = &hyperrectangle[element.key_column];
 
             /// The case when the column is wrapped in a chain of possibly monotonic functions.

src/Storages/MergeTree/MergeTask.cpp

@@ -9,7 +9,7 @@
 #include <Common/ActionBlocker.h>
 #include <Processors/Transforms/CheckSortedTransform.h>
 #include <Storages/MergeTree/DataPartStorageOnDiskFull.h>
-
+#include <Compression/CompressedWriteBuffer.h>
 #include <DataTypes/ObjectUtils.h>
 #include <DataTypes/Serializations/SerializationInfo.h>
 #include <IO/IReadableWriteBuffer.h>
@@ -34,6 +34,7 @@
 #include <Processors/Transforms/DistinctTransform.h>
 #include <Processors/QueryPlan/CreatingSetsStep.h>
 #include <Interpreters/PreparedSets.h>
+#include <Interpreters/MergeTreeTransaction.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
 
 namespace DB
@@ -378,7 +379,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare()
         MergeTreeIndexFactory::instance().getMany(global_ctx->metadata_snapshot->getSecondaryIndices()),
         MergeTreeStatisticsFactory::instance().getMany(global_ctx->metadata_snapshot->getColumns()),
         ctx->compression_codec,
-        global_ctx->txn,
+        global_ctx->txn ? global_ctx->txn->tid : Tx::PrehistoricTID,
         /*reset_columns=*/ true,
         ctx->blocks_are_granules_size,
         global_ctx->context->getWriteSettings());

src/Storages/MergeTree/MergeTreeData.cpp

@@ -6135,6 +6135,21 @@ bool MergeTreeData::supportsLightweightDelete() const
     return true;
 }
 
+bool MergeTreeData::hasProjection() const
+{
+    auto lock = lockParts();
+    for (const auto & part : data_parts_by_info)
+    {
+        if (part->getState() == MergeTreeDataPartState::Outdated
+            || part->getState() == MergeTreeDataPartState::Deleting)
+            continue;
+
+        if (part->hasProjection())
+            return true;
+    }
+    return false;
+}
+
 MergeTreeData::ProjectionPartsVector MergeTreeData::getAllProjectionPartsVector(MergeTreeData::DataPartStateVector * out_states) const
 {
     ProjectionPartsVector res;
@@ -8477,7 +8492,7 @@ std::pair<MergeTreeData::MutableDataPartPtr, scope_guard> MergeTreeData::createE
     MergedBlockOutputStream out(new_data_part, metadata_snapshot, columns,
         index_factory.getMany(metadata_snapshot->getSecondaryIndices()),
         Statistics{},
-        compression_codec, txn);
+        compression_codec, txn ? txn->tid : Tx::PrehistoricTID);
 
     bool sync_on_insert = settings->fsync_after_insert;
 

src/Storages/MergeTree/MergeTreeData.h

@@ -439,6 +439,8 @@ public:
 
     bool supportsLightweightDelete() const override;
 
+    bool hasProjection() const override;
+
     bool areAsynchronousInsertsEnabled() const override { return getSettings()->async_insert; }
 
     bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override;

src/Storages/MergeTree/MergeTreeDataPartCompact.cpp

@@ -47,26 +47,36 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader(
         avg_value_size_hints, profile_callback, CLOCK_MONOTONIC_COARSE);
 }
 
-IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter(
+MergeTreeDataPartWriterPtr createMergeTreeDataPartCompactWriter(
+    const String & data_part_name_,
+    const String & logger_name_,
+    const SerializationByName & serializations_,
+    MutableDataPartStoragePtr data_part_storage_,
+    const MergeTreeIndexGranularityInfo & index_granularity_info_,
+    const MergeTreeSettingsPtr & storage_settings_,
     const NamesAndTypesList & columns_list,
+    const ColumnPositions & column_positions,
     const StorageMetadataPtr & metadata_snapshot,
+    const VirtualsDescriptionPtr & virtual_columns,
     const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
     const Statistics & stats_to_recalc_,
+    const String & marks_file_extension_,
     const CompressionCodecPtr & default_codec_,
     const MergeTreeWriterSettings & writer_settings,
     const MergeTreeIndexGranularity & computed_index_granularity)
 {
     NamesAndTypesList ordered_columns_list;
     std::copy_if(columns_list.begin(), columns_list.end(), std::back_inserter(ordered_columns_list),
-        [this](const auto & column) { return getColumnPosition(column.name) != std::nullopt; });
+        [&column_positions](const auto & column) { return column_positions.contains(column.name); });
 
     /// Order of writing is important in compact format
-    ordered_columns_list.sort([this](const auto & lhs, const auto & rhs)
-        { return *getColumnPosition(lhs.name) < *getColumnPosition(rhs.name); });
+    ordered_columns_list.sort([&column_positions](const auto & lhs, const auto & rhs)
+        { return column_positions.at(lhs.name) < column_positions.at(rhs.name); });
 
     return std::make_unique<MergeTreeDataPartWriterCompact>(
-        shared_from_this(), ordered_columns_list, metadata_snapshot,
-        indices_to_recalc, stats_to_recalc_, getMarksFileExtension(),
+        data_part_name_, logger_name_, serializations_, data_part_storage_,
+        index_granularity_info_, storage_settings_, ordered_columns_list, metadata_snapshot, virtual_columns,
+        indices_to_recalc, stats_to_recalc_, marks_file_extension_,
         default_codec_, writer_settings, computed_index_granularity);
 }
 

src/Storages/MergeTree/MergeTreeDataPartCompact.h

@@ -40,15 +40,6 @@ public:
         const ValueSizeMap & avg_value_size_hints,
         const ReadBufferFromFileBase::ProfileCallback & profile_callback) const override;
 
-    MergeTreeWriterPtr getWriter(
-        const NamesAndTypesList & columns_list,
-        const StorageMetadataPtr & metadata_snapshot,
-        const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
-        const Statistics & stats_to_recalc_,
-        const CompressionCodecPtr & default_codec_,
-        const MergeTreeWriterSettings & writer_settings,
-        const MergeTreeIndexGranularity & computed_index_granularity) override;
-
     bool isStoredOnDisk() const override { return true; }
 
     bool isStoredOnRemoteDisk() const override;

src/Storages/MergeTree/MergeTreeDataPartWide.cpp

@@ -53,19 +53,28 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader(
         profile_callback);
 }
 
-IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter(
+MergeTreeDataPartWriterPtr createMergeTreeDataPartWideWriter(
+    const String & data_part_name_,
+    const String & logger_name_,
+    const SerializationByName & serializations_,
+    MutableDataPartStoragePtr data_part_storage_,
+    const MergeTreeIndexGranularityInfo & index_granularity_info_,
+    const MergeTreeSettingsPtr & storage_settings_,
     const NamesAndTypesList & columns_list,
     const StorageMetadataPtr & metadata_snapshot,
+    const VirtualsDescriptionPtr & virtual_columns,
     const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
     const Statistics & stats_to_recalc_,
+    const String & marks_file_extension_,
     const CompressionCodecPtr & default_codec_,
     const MergeTreeWriterSettings & writer_settings,
     const MergeTreeIndexGranularity & computed_index_granularity)
 {
     return std::make_unique<MergeTreeDataPartWriterWide>(
-        shared_from_this(), columns_list,
-        metadata_snapshot, indices_to_recalc, stats_to_recalc_,
-        getMarksFileExtension(),
+        data_part_name_, logger_name_, serializations_, data_part_storage_,
+        index_granularity_info_, storage_settings_, columns_list,
+        metadata_snapshot, virtual_columns, indices_to_recalc, stats_to_recalc_,
+        marks_file_extension_,
         default_codec_, writer_settings, computed_index_granularity);
 }
 

src/Storages/MergeTree/MergeTreeDataPartWide.h

@@ -35,15 +35,6 @@ public:
         const ValueSizeMap & avg_value_size_hints,
         const ReadBufferFromFileBase::ProfileCallback & profile_callback) const override;
 
-    MergeTreeWriterPtr getWriter(
-        const NamesAndTypesList & columns_list,
-        const StorageMetadataPtr & metadata_snapshot,
-        const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
-        const Statistics & stats_to_recalc_,
-        const CompressionCodecPtr & default_codec_,
-        const MergeTreeWriterSettings & writer_settings,
-        const MergeTreeIndexGranularity & computed_index_granularity) override;
-
     bool isStoredOnDisk() const override { return true; }
 
     bool isStoredOnRemoteDisk() const override;

src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp

@@ -10,32 +10,41 @@ namespace ErrorCodes
 }
 
 MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact(
-    const MergeTreeMutableDataPartPtr & data_part_,
+    const String & data_part_name_,
+    const String & logger_name_,
+    const SerializationByName & serializations_,
+    MutableDataPartStoragePtr data_part_storage_,
+    const MergeTreeIndexGranularityInfo & index_granularity_info_,
+    const MergeTreeSettingsPtr & storage_settings_,
     const NamesAndTypesList & columns_list_,
     const StorageMetadataPtr & metadata_snapshot_,
+    const VirtualsDescriptionPtr & virtual_columns_,
     const std::vector<MergeTreeIndexPtr> & indices_to_recalc_,
     const Statistics & stats_to_recalc,
     const String & marks_file_extension_,
     const CompressionCodecPtr & default_codec_,
     const MergeTreeWriterSettings & settings_,
     const MergeTreeIndexGranularity & index_granularity_)
-    : MergeTreeDataPartWriterOnDisk(data_part_, columns_list_, metadata_snapshot_,
+    : MergeTreeDataPartWriterOnDisk(
+        data_part_name_, logger_name_, serializations_,
+        data_part_storage_, index_granularity_info_, storage_settings_,
+        columns_list_, metadata_snapshot_, virtual_columns_,
         indices_to_recalc_, stats_to_recalc, marks_file_extension_,
         default_codec_, settings_, index_granularity_)
-    , plain_file(data_part_->getDataPartStorage().writeFile(
+    , plain_file(getDataPartStorage().writeFile(
             MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION,
             settings.max_compress_block_size,
             settings_.query_write_settings))
     , plain_hashing(*plain_file)
 {
-    marks_file = data_part_->getDataPartStorage().writeFile(
+    marks_file = getDataPartStorage().writeFile(
             MergeTreeDataPartCompact::DATA_FILE_NAME + marks_file_extension_,
             4096,
             settings_.query_write_settings);
 
     marks_file_hashing = std::make_unique<HashingWriteBuffer>(*marks_file);
 
-    if (data_part_->index_granularity_info.mark_type.compressed)
+    if (index_granularity_info.mark_type.compressed)
     {
         marks_compressor = std::make_unique<CompressedWriteBuffer>(
             *marks_file_hashing,
@@ -45,10 +54,9 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact(
         marks_source_hashing = std::make_unique<HashingWriteBuffer>(*marks_compressor);
     }
 
-    auto storage_snapshot = std::make_shared<StorageSnapshot>(data_part->storage, metadata_snapshot);
     for (const auto & column : columns_list)
     {
-        auto compression = storage_snapshot->getCodecDescOrDefault(column.name, default_codec);
+        auto compression = getCodecDescOrDefault(column.name, default_codec);
         addStreams(column, nullptr, compression);
     }
 }
@@ -59,12 +67,11 @@ void MergeTreeDataPartWriterCompact::initDynamicStreamsIfNeeded(const Block & bl
         return;
 
     is_dynamic_streams_initialized = true;
-    auto storage_snapshot = std::make_shared<StorageSnapshot>(data_part->storage, metadata_snapshot);
     for (const auto & column : columns_list)
     {
         if (column.type->hasDynamicSubcolumns())
         {
-            auto compression = storage_snapshot->getCodecDescOrDefault(column.name, default_codec);
+            auto compression = getCodecDescOrDefault(column.name, default_codec);
             addStreams(column, block.getByName(column.name).column, compression);
         }
     }
@@ -98,7 +105,7 @@ void MergeTreeDataPartWriterCompact::addStreams(const NameAndTypePair & name_and
         compressed_streams.emplace(stream_name, stream);
     };
 
-    data_part->getSerialization(name_and_type.name)->enumerateStreams(callback, name_and_type.type, column);
+    getSerialization(name_and_type.name)->enumerateStreams(callback, name_and_type.type, column);
 }
 
 namespace
@@ -251,7 +258,7 @@ void MergeTreeDataPartWriterCompact::writeDataBlock(const Block & block, const G
             writeBinaryLittleEndian(static_cast<UInt64>(0), marks_out);
 
             writeColumnSingleGranule(
-                block.getByName(name_and_type->name), data_part->getSerialization(name_and_type->name),
+                block.getByName(name_and_type->name), getSerialization(name_and_type->name),
                 stream_getter, granule.start_row, granule.rows_to_write);
 
             /// Each type always have at least one substream
@@ -262,7 +269,7 @@ void MergeTreeDataPartWriterCompact::writeDataBlock(const Block & block, const G
     }
 }
 
-void MergeTreeDataPartWriterCompact::fillDataChecksums(IMergeTreeDataPart::Checksums & checksums)
+void MergeTreeDataPartWriterCompact::fillDataChecksums(MergeTreeDataPartChecksums & checksums)
 {
     if (columns_buffer.size() != 0)
     {
@@ -432,7 +439,7 @@ size_t MergeTreeDataPartWriterCompact::ColumnsBuffer::size() const
     return accumulated_columns.at(0)->size();
 }
 
-void MergeTreeDataPartWriterCompact::fillChecksums(IMergeTreeDataPart::Checksums & checksums, NameSet & /*checksums_to_remove*/)
+void MergeTreeDataPartWriterCompact::fillChecksums(MergeTreeDataPartChecksums & checksums, NameSet & /*checksums_to_remove*/)
 {
     // If we don't have anything to write, skip finalization.
     if (!columns_list.empty())

src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h

@@ -11,9 +11,15 @@ class MergeTreeDataPartWriterCompact : public MergeTreeDataPartWriterOnDisk
 {
 public:
     MergeTreeDataPartWriterCompact(
-        const MergeTreeMutableDataPartPtr & data_part,
+        const String & data_part_name_,
+        const String & logger_name_,
+        const SerializationByName & serializations_,
+        MutableDataPartStoragePtr data_part_storage_,
+        const MergeTreeIndexGranularityInfo & index_granularity_info_,
+        const MergeTreeSettingsPtr & storage_settings_,
         const NamesAndTypesList & columns_list,
         const StorageMetadataPtr & metadata_snapshot_,
+        const VirtualsDescriptionPtr & virtual_columns_,
         const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
         const Statistics & stats_to_recalc,
         const String & marks_file_extension,
@@ -23,12 +29,12 @@ public:
 
     void write(const Block & block, const IColumn::Permutation * permutation) override;
 
-    void fillChecksums(IMergeTreeDataPart::Checksums & checksums, NameSet & checksums_to_remove) override;
+    void fillChecksums(MergeTreeDataPartChecksums & checksums, NameSet & checksums_to_remove) override;
     void finish(bool sync) override;
 
 private:
     /// Finish serialization of the data. Flush rows in buffer to disk, compute checksums.
-    void fillDataChecksums(IMergeTreeDataPart::Checksums & checksums);
+    void fillDataChecksums(MergeTreeDataPartChecksums & checksums);
     void finishDataSerialization(bool sync);
 
     void fillIndexGranularity(size_t index_granularity_for_block, size_t rows_in_block) override;

src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp

@@ -140,16 +140,24 @@ void MergeTreeDataPartWriterOnDisk::Stream<only_plain_file>::addToChecksums(Merg
 
 
 MergeTreeDataPartWriterOnDisk::MergeTreeDataPartWriterOnDisk(
-    const MergeTreeMutableDataPartPtr & data_part_,
+    const String & data_part_name_,
+    const String & logger_name_,
+    const SerializationByName & serializations_,
+    MutableDataPartStoragePtr data_part_storage_,
+    const MergeTreeIndexGranularityInfo & index_granularity_info_,
+    const MergeTreeSettingsPtr & storage_settings_,
     const NamesAndTypesList & columns_list_,
     const StorageMetadataPtr & metadata_snapshot_,
+    const VirtualsDescriptionPtr & virtual_columns_,
     const MergeTreeIndices & indices_to_recalc_,
     const Statistics & stats_to_recalc_,
     const String & marks_file_extension_,
     const CompressionCodecPtr & default_codec_,
     const MergeTreeWriterSettings & settings_,
     const MergeTreeIndexGranularity & index_granularity_)
-    : IMergeTreeDataPartWriter(data_part_, columns_list_, metadata_snapshot_, settings_, index_granularity_)
+    : IMergeTreeDataPartWriter(
+        data_part_name_, serializations_, data_part_storage_, index_granularity_info_,
+        storage_settings_, columns_list_, metadata_snapshot_, virtual_columns_, settings_, index_granularity_)
     , skip_indices(indices_to_recalc_)
     , stats(stats_to_recalc_)
     , marks_file_extension(marks_file_extension_)
@@ -157,14 +165,14 @@ MergeTreeDataPartWriterOnDisk::MergeTreeDataPartWriterOnDisk(
     , compute_granularity(index_granularity.empty())
     , compress_primary_key(settings.compress_primary_key)
     , execution_stats(skip_indices.size(), stats.size())
-    , log(getLogger(storage.getLogName() + " (DataPartWriter)"))
+    , log(getLogger(logger_name_ + " (DataPartWriter)"))
 {
     if (settings.blocks_are_granules_size && !index_granularity.empty())
         throw Exception(ErrorCodes::LOGICAL_ERROR,
                         "Can't take information about index granularity from blocks, when non empty index_granularity array specified");
 
-    if (!data_part->getDataPartStorage().exists())
-        data_part->getDataPartStorage().createDirectories();
+    if (!getDataPartStorage().exists())
+        getDataPartStorage().createDirectories();
 
     if (settings.rewrite_primary_key)
         initPrimaryIndex();
@@ -223,7 +231,6 @@ static size_t computeIndexGranularityImpl(
 
 size_t MergeTreeDataPartWriterOnDisk::computeIndexGranularity(const Block & block) const
 {
-    const auto storage_settings = storage.getSettings();
     return computeIndexGranularityImpl(
             block,
             storage_settings->index_granularity_bytes,
@@ -237,7 +244,7 @@ void MergeTreeDataPartWriterOnDisk::initPrimaryIndex()
     if (metadata_snapshot->hasPrimaryKey())
     {
         String index_name = "primary" + getIndexExtension(compress_primary_key);
-        index_file_stream = data_part->getDataPartStorage().writeFile(index_name, DBMS_DEFAULT_BUFFER_SIZE, settings.query_write_settings);
+        index_file_stream = getDataPartStorage().writeFile(index_name, DBMS_DEFAULT_BUFFER_SIZE, settings.query_write_settings);
         index_file_hashing_stream = std::make_unique<HashingWriteBuffer>(*index_file_stream);
 
         if (compress_primary_key)
@@ -256,7 +263,7 @@ void MergeTreeDataPartWriterOnDisk::initStatistics()
         String stats_name = stat_ptr->getFileName();
         stats_streams.emplace_back(std::make_unique<MergeTreeDataPartWriterOnDisk::Stream<true>>(
                                        stats_name,
-                                       data_part->getDataPartStoragePtr(),
+                                       data_part_storage,
                                        stats_name, STAT_FILE_SUFFIX,
                                        default_codec, settings.max_compress_block_size,
                                        settings.query_write_settings));
@@ -275,7 +282,7 @@ void MergeTreeDataPartWriterOnDisk::initSkipIndices()
         skip_indices_streams.emplace_back(
                 std::make_unique<MergeTreeDataPartWriterOnDisk::Stream<false>>(
                         stream_name,
-                        data_part->getDataPartStoragePtr(),
+                        data_part_storage,
                         stream_name, skip_index->getSerializedFileExtension(),
                         stream_name, marks_file_extension,
                         default_codec, settings.max_compress_block_size,
@@ -285,7 +292,7 @@ void MergeTreeDataPartWriterOnDisk::initSkipIndices()
         GinIndexStorePtr store = nullptr;
         if (typeid_cast<const MergeTreeIndexFullText *>(&*skip_index) != nullptr)
         {
-            store = std::make_shared<GinIndexStore>(stream_name, data_part->getDataPartStoragePtr(), data_part->getDataPartStoragePtr(), storage.getSettings()->max_digestion_size_per_segment);
+            store = std::make_shared<GinIndexStore>(stream_name, data_part_storage, data_part_storage, storage_settings->max_digestion_size_per_segment);
             gin_index_stores[stream_name] = store;
         }
         skip_indices_aggregators.push_back(skip_index->createIndexAggregatorForPart(store, settings));
@@ -498,7 +505,7 @@ void MergeTreeDataPartWriterOnDisk::finishStatisticsSerialization(bool sync)
     }
 
     for (size_t i = 0; i < stats.size(); ++i)
-        LOG_DEBUG(log, "Spent {} ms calculating statistics {} for the part {}", execution_stats.statistics_build_us[i] / 1000, stats[i]->columnName(), data_part->name);
+        LOG_DEBUG(log, "Spent {} ms calculating statistics {} for the part {}", execution_stats.statistics_build_us[i] / 1000, stats[i]->columnName(), data_part_name);
 }
 
 void MergeTreeDataPartWriterOnDisk::fillStatisticsChecksums(MergeTreeData::DataPart::Checksums & checksums)
@@ -524,7 +531,7 @@ void MergeTreeDataPartWriterOnDisk::finishSkipIndicesSerialization(bool sync)
         store.second->finalize();
 
     for (size_t i = 0; i < skip_indices.size(); ++i)
-        LOG_DEBUG(log, "Spent {} ms calculating index {} for the part {}", execution_stats.skip_indices_build_us[i] / 1000, skip_indices[i]->index.name, data_part->name);
+        LOG_DEBUG(log, "Spent {} ms calculating index {} for the part {}", execution_stats.skip_indices_build_us[i] / 1000, skip_indices[i]->index.name, data_part_name);
 
     gin_index_stores.clear();
     skip_indices_streams.clear();

src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h

@@ -5,9 +5,6 @@
 #include <IO/WriteBufferFromFileBase.h>
 #include <Compression/CompressedWriteBuffer.h>
 #include <IO/HashingWriteBuffer.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-#include <Storages/MergeTree/IMergeTreeDataPart.h>
-#include <Disks/IDisk.h>
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/parseQuery.h>
 #include <Storages/Statistics/Statistics.h>
@@ -97,16 +94,22 @@ public:
 
         void sync() const;
 
-        void addToChecksums(IMergeTreeDataPart::Checksums & checksums);
+        void addToChecksums(MergeTreeDataPartChecksums & checksums);
     };
 
     using StreamPtr = std::unique_ptr<Stream<false>>;
     using StatisticStreamPtr = std::unique_ptr<Stream<true>>;
 
     MergeTreeDataPartWriterOnDisk(
-        const MergeTreeMutableDataPartPtr & data_part_,
+        const String & data_part_name_,
+        const String & logger_name_,
+        const SerializationByName & serializations_,
+        MutableDataPartStoragePtr data_part_storage_,
+        const MergeTreeIndexGranularityInfo & index_granularity_info_,
+        const MergeTreeSettingsPtr & storage_settings_,
         const NamesAndTypesList & columns_list,
         const StorageMetadataPtr & metadata_snapshot_,
+        const VirtualsDescriptionPtr & virtual_columns_,
         const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
         const Statistics & stats_to_recalc_,
         const String & marks_file_extension,
@@ -133,13 +136,13 @@ protected:
     void calculateAndSerializeStatistics(const Block & stats_block);
 
     /// Finishes primary index serialization: write final primary index row (if required) and compute checksums
-    void fillPrimaryIndexChecksums(MergeTreeData::DataPart::Checksums & checksums);
+    void fillPrimaryIndexChecksums(MergeTreeDataPartChecksums & checksums);
     void finishPrimaryIndexSerialization(bool sync);
     /// Finishes skip indices serialization: write all accumulated data to disk and compute checksums
-    void fillSkipIndicesChecksums(MergeTreeData::DataPart::Checksums & checksums);
+    void fillSkipIndicesChecksums(MergeTreeDataPartChecksums & checksums);
     void finishSkipIndicesSerialization(bool sync);
 
-    void fillStatisticsChecksums(MergeTreeData::DataPart::Checksums & checksums);
+    void fillStatisticsChecksums(MergeTreeDataPartChecksums & checksums);
     void finishStatisticsSerialization(bool sync);
 
     /// Get global number of the current which we are writing (or going to start to write)

src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp

@@ -76,23 +76,31 @@ Granules getGranulesToWrite(const MergeTreeIndexGranularity & index_granularity,
 }
 
 MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide(
-    const MergeTreeMutableDataPartPtr & data_part_,
+    const String & data_part_name_,
+    const String & logger_name_,
+    const SerializationByName & serializations_,
+    MutableDataPartStoragePtr data_part_storage_,
+    const MergeTreeIndexGranularityInfo & index_granularity_info_,
+    const MergeTreeSettingsPtr & storage_settings_,
     const NamesAndTypesList & columns_list_,
     const StorageMetadataPtr & metadata_snapshot_,
+    const VirtualsDescriptionPtr & virtual_columns_,
     const std::vector<MergeTreeIndexPtr> & indices_to_recalc_,
     const Statistics & stats_to_recalc_,
     const String & marks_file_extension_,
     const CompressionCodecPtr & default_codec_,
     const MergeTreeWriterSettings & settings_,
     const MergeTreeIndexGranularity & index_granularity_)
-    : MergeTreeDataPartWriterOnDisk(data_part_, columns_list_, metadata_snapshot_,
-           indices_to_recalc_, stats_to_recalc_, marks_file_extension_,
-           default_codec_, settings_, index_granularity_)
+    : MergeTreeDataPartWriterOnDisk(
+            data_part_name_, logger_name_, serializations_,
+            data_part_storage_, index_granularity_info_, storage_settings_,
+            columns_list_, metadata_snapshot_, virtual_columns_,
+            indices_to_recalc_, stats_to_recalc_, marks_file_extension_,
+            default_codec_, settings_, index_granularity_)
 {
-    auto storage_snapshot = std::make_shared<StorageSnapshot>(data_part->storage, metadata_snapshot);
     for (const auto & column : columns_list)
     {
-        auto compression = storage_snapshot->getCodecDescOrDefault(column.name, default_codec);
+        auto compression = getCodecDescOrDefault(column.name, default_codec);
         addStreams(column, nullptr, compression);
     }
 }
@@ -104,12 +112,11 @@ void MergeTreeDataPartWriterWide::initDynamicStreamsIfNeeded(const DB::Block & b
 
     is_dynamic_streams_initialized = true;
     block_sample = block.cloneEmpty();
-    auto storage_snapshot = std::make_shared<StorageSnapshot>(data_part->storage, metadata_snapshot);
     for (const auto & column : columns_list)
     {
         if (column.type->hasDynamicSubcolumns())
         {
-            auto compression = storage_snapshot->getCodecDescOrDefault(column.name, default_codec);
+            auto compression = getCodecDescOrDefault(column.name, default_codec);
             addStreams(column, block_sample.getByName(column.name).column, compression);
         }
     }
@@ -124,7 +131,6 @@ void MergeTreeDataPartWriterWide::addStreams(
     {
         assert(!substream_path.empty());
 
-        auto storage_settings = storage.getSettings();
         auto full_stream_name = ISerialization::getFileNameForStream(name_and_type, substream_path);
 
         String stream_name;
@@ -168,7 +174,7 @@ void MergeTreeDataPartWriterWide::addStreams(
 
         column_streams[stream_name] = std::make_unique<Stream<false>>(
             stream_name,
-            data_part->getDataPartStoragePtr(),
+            data_part_storage,
             stream_name, DATA_FILE_EXTENSION,
             stream_name, marks_file_extension,
             compression_codec,
@@ -182,7 +188,7 @@ void MergeTreeDataPartWriterWide::addStreams(
     };
 
     ISerialization::SubstreamPath path;
-    data_part->getSerialization(name_and_type.name)->enumerateStreams(callback, name_and_type.type, column);
+    getSerialization(name_and_type.name)->enumerateStreams(callback, name_and_type.type, column);
 }
 
 const String & MergeTreeDataPartWriterWide::getStreamName(
@@ -286,7 +292,7 @@ void MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Perm
     {
         auto & column = block_to_write.getByName(it->name);
 
-        if (data_part->getSerialization(it->name)->getKind() != ISerialization::Kind::SPARSE)
+        if (getSerialization(it->name)->getKind() != ISerialization::Kind::SPARSE)
             column.column = recursiveRemoveSparse(column.column);
 
         if (permutation)
@@ -358,7 +364,7 @@ StreamsWithMarks MergeTreeDataPartWriterWide::getCurrentMarksForColumn(
             min_compress_block_size = value->safeGet<UInt64>();
     if (!min_compress_block_size)
         min_compress_block_size = settings.min_compress_block_size;
-    data_part->getSerialization(name_and_type.name)->enumerateStreams([&] (const ISerialization::SubstreamPath & substream_path)
+    getSerialization(name_and_type.name)->enumerateStreams([&] (const ISerialization::SubstreamPath & substream_path)
     {
         bool is_offsets = !substream_path.empty() && substream_path.back().type == ISerialization::Substream::ArraySizes;
         auto stream_name = getStreamName(name_and_type, substream_path);
@@ -392,7 +398,7 @@ void MergeTreeDataPartWriterWide::writeSingleGranule(
     ISerialization::SerializeBinaryBulkSettings & serialize_settings,
     const Granule & granule)
 {
-    const auto & serialization = data_part->getSerialization(name_and_type.name);
+    const auto & serialization = getSerialization(name_and_type.name);
     serialization->serializeBinaryBulkWithMultipleStreams(column, granule.start_row, granule.rows_to_write, serialize_settings, serialization_state);
 
     /// So that instead of the marks pointing to the end of the compressed block, there were marks pointing to the beginning of the next one.
@@ -422,7 +428,7 @@ void MergeTreeDataPartWriterWide::writeColumn(
 
     const auto & [name, type] = name_and_type;
     auto [it, inserted] = serialization_states.emplace(name, nullptr);
-    auto serialization = data_part->getSerialization(name_and_type.name);
+    auto serialization = getSerialization(name_and_type.name);
 
     if (inserted)
     {
@@ -431,11 +437,10 @@ void MergeTreeDataPartWriterWide::writeColumn(
         serialization->serializeBinaryBulkStatePrefix(column, serialize_settings, it->second);
     }
 
-    const auto & global_settings = storage.getContext()->getSettingsRef();
     ISerialization::SerializeBinaryBulkSettings serialize_settings;
     serialize_settings.getter = createStreamGetter(name_and_type, offset_columns);
-    serialize_settings.low_cardinality_max_dictionary_size = global_settings.low_cardinality_max_dictionary_size;
-    serialize_settings.low_cardinality_use_single_dictionary_for_part = global_settings.low_cardinality_use_single_dictionary_for_part != 0;
+    serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size;
+    serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part;
 
     for (const auto & granule : granules)
     {
@@ -484,7 +489,7 @@ void MergeTreeDataPartWriterWide::writeColumn(
 void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePair & name_type)
 {
     const auto & [name, type] = name_type;
-    const auto & serialization = data_part->getSerialization(name_type.name);
+    const auto & serialization = getSerialization(name_type.name);
 
     if (!type->isValueRepresentedByNumber() || type->haveSubtypes() || serialization->getKind() != ISerialization::Kind::DEFAULT)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot validate column of non fixed type {}", type->getName());
@@ -494,21 +499,21 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai
     String bin_path = escaped_name + DATA_FILE_EXTENSION;
 
     /// Some columns may be removed because of ttl. Skip them.
-    if (!data_part->getDataPartStorage().exists(mrk_path))
+    if (!getDataPartStorage().exists(mrk_path))
         return;
 
-    auto mrk_file_in = data_part->getDataPartStorage().readFile(mrk_path, {}, std::nullopt, std::nullopt);
+    auto mrk_file_in = getDataPartStorage().readFile(mrk_path, {}, std::nullopt, std::nullopt);
     std::unique_ptr<ReadBuffer> mrk_in;
-    if (data_part->index_granularity_info.mark_type.compressed)
+    if (index_granularity_info.mark_type.compressed)
         mrk_in = std::make_unique<CompressedReadBufferFromFile>(std::move(mrk_file_in));
     else
         mrk_in = std::move(mrk_file_in);
 
-    DB::CompressedReadBufferFromFile bin_in(data_part->getDataPartStorage().readFile(bin_path, {}, std::nullopt, std::nullopt));
+    DB::CompressedReadBufferFromFile bin_in(getDataPartStorage().readFile(bin_path, {}, std::nullopt, std::nullopt));
     bool must_be_last = false;
     UInt64 offset_in_compressed_file = 0;
     UInt64 offset_in_decompressed_block = 0;
-    UInt64 index_granularity_rows = data_part->index_granularity_info.fixed_index_granularity;
+    UInt64 index_granularity_rows = index_granularity_info.fixed_index_granularity;
 
     size_t mark_num;
 
@@ -524,7 +529,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai
         if (settings.can_use_adaptive_granularity)
             readBinaryLittleEndian(index_granularity_rows, *mrk_in);
         else
-            index_granularity_rows = data_part->index_granularity_info.fixed_index_granularity;
+            index_granularity_rows = index_granularity_info.fixed_index_granularity;
 
         if (must_be_last)
         {
@@ -557,7 +562,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai
                             ErrorCodes::LOGICAL_ERROR,
                             "Incorrect mark rows for part {} for mark #{}"
                             " (compressed offset {}, decompressed offset {}), in-memory {}, on disk {}, total marks {}",
-                            data_part->getDataPartStorage().getFullPath(),
+                            getDataPartStorage().getFullPath(),
                             mark_num, offset_in_compressed_file, offset_in_decompressed_block,
                             index_granularity.getMarkRows(mark_num), index_granularity_rows,
                             index_granularity.getMarksCount());
@@ -617,12 +622,11 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai
     }
 }
 
-void MergeTreeDataPartWriterWide::fillDataChecksums(IMergeTreeDataPart::Checksums & checksums, NameSet & checksums_to_remove)
+void MergeTreeDataPartWriterWide::fillDataChecksums(MergeTreeDataPartChecksums & checksums, NameSet & checksums_to_remove)
 {
-    const auto & global_settings = storage.getContext()->getSettingsRef();
     ISerialization::SerializeBinaryBulkSettings serialize_settings;
-    serialize_settings.low_cardinality_max_dictionary_size = global_settings.low_cardinality_max_dictionary_size;
-    serialize_settings.low_cardinality_use_single_dictionary_for_part = global_settings.low_cardinality_use_single_dictionary_for_part != 0;
+    serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size;
+    serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part;
     WrittenOffsetColumns offset_columns;
     if (rows_written_in_last_mark > 0)
     {
@@ -646,7 +650,7 @@ void MergeTreeDataPartWriterWide::fillDataChecksums(IMergeTreeDataPart::Checksum
             {
                 serialize_settings.getter = createStreamGetter(*it, written_offset_columns ? *written_offset_columns : offset_columns);
                 serialize_settings.dynamic_write_statistics = ISerialization::SerializeBinaryBulkSettings::DynamicStatisticsMode::SUFFIX;
-                data_part->getSerialization(it->name)->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[it->name]);
+                getSerialization(it->name)->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[it->name]);
             }
 
             if (write_final_mark)
@@ -689,7 +693,7 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(bool sync)
     {
         if (column.type->isValueRepresentedByNumber()
             && !column.type->haveSubtypes()
-            && data_part->getSerialization(column.name)->getKind() == ISerialization::Kind::DEFAULT)
+            && getSerialization(column.name)->getKind() == ISerialization::Kind::DEFAULT)
         {
             validateColumnOfFixedSize(column);
         }
@@ -698,7 +702,7 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(bool sync)
 
 }
 
-void MergeTreeDataPartWriterWide::fillChecksums(IMergeTreeDataPart::Checksums & checksums, NameSet & checksums_to_remove)
+void MergeTreeDataPartWriterWide::fillChecksums(MergeTreeDataPartChecksums & checksums, NameSet & checksums_to_remove)
 {
     // If we don't have anything to write, skip finalization.
     if (!columns_list.empty())
@@ -732,7 +736,7 @@ void MergeTreeDataPartWriterWide::writeFinalMark(
 {
     writeSingleMark(name_and_type, offset_columns, 0);
     /// Memoize information about offsets
-    data_part->getSerialization(name_and_type.name)->enumerateStreams([&] (const ISerialization::SubstreamPath & substream_path)
+    getSerialization(name_and_type.name)->enumerateStreams([&] (const ISerialization::SubstreamPath & substream_path)
     {
         bool is_offsets = !substream_path.empty() && substream_path.back().type == ISerialization::Substream::ArraySizes;
         if (is_offsets)

src/Storages/MergeTree/MergeTreeDataPartWriterWide.h

@@ -21,9 +21,15 @@ class MergeTreeDataPartWriterWide : public MergeTreeDataPartWriterOnDisk
 {
 public:
     MergeTreeDataPartWriterWide(
-        const MergeTreeMutableDataPartPtr & data_part,
+        const String & data_part_name_,
+        const String & logger_name_,
+        const SerializationByName & serializations_,
+        MutableDataPartStoragePtr data_part_storage_,
+        const MergeTreeIndexGranularityInfo & index_granularity_info_,
+        const MergeTreeSettingsPtr & storage_settings_,
         const NamesAndTypesList & columns_list,
         const StorageMetadataPtr & metadata_snapshot,
+        const VirtualsDescriptionPtr & virtual_columns_,
         const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
         const Statistics & stats_to_recalc_,
         const String & marks_file_extension,
@@ -33,14 +39,14 @@ public:
 
     void write(const Block & block, const IColumn::Permutation * permutation) override;
 
-    void fillChecksums(IMergeTreeDataPart::Checksums & checksums, NameSet & checksums_to_remove) final;
+    void fillChecksums(MergeTreeDataPartChecksums & checksums, NameSet & checksums_to_remove) final;
 
     void finish(bool sync) final;
 
 private:
     /// Finish serialization of data: write final mark if required and compute checksums
     /// Also validate written data in debug mode
-    void fillDataChecksums(IMergeTreeDataPart::Checksums & checksums, NameSet & checksums_to_remove);
+    void fillDataChecksums(MergeTreeDataPartChecksums & checksums, NameSet & checksums_to_remove);
     void finishDataSerialization(bool sync);
 
     /// Write data of one column.