From 961bf074daf0c901a3e9d14b6caa4ba6cb37cc7c Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 20 Nov 2023 10:56:10 +0100 Subject: [PATCH 001/103] Initial draft version of adding backup support to AzureBlobStorage --- src/Backups/BackupFactory.cpp | 2 + src/Backups/BackupIO_AzureBlobStorage.cpp | 336 ++++++++++++++++++ src/Backups/BackupIO_AzureBlobStorage.h | 69 ++++ src/Backups/BackupImpl.cpp | 8 +- .../registerBackupEngineAzureBlobStorage.cpp | 134 +++++++ src/CMakeLists.txt | 3 + src/Common/ProfileEvents.cpp | 4 + .../copyAzureBlobStorageFile.cpp | 324 +++++++++++++++++ .../copyAzureBlobStorageFile.h | 58 +++ src/Storages/StorageAzureBlob.cpp | 11 + src/Storages/StorageAzureBlob.h | 1 + .../__init__.py | 1 + .../configs/config.xml | 11 + .../configs/disable_profilers.xml | 13 + .../configs/users.xml | 8 + .../test.py | 151 ++++++++ 16 files changed, 1132 insertions(+), 2 deletions(-) create mode 100644 src/Backups/BackupIO_AzureBlobStorage.cpp create mode 100644 src/Backups/BackupIO_AzureBlobStorage.h create mode 100644 src/Backups/registerBackupEngineAzureBlobStorage.cpp create mode 100644 src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp create mode 100644 src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h create mode 100644 tests/integration/test_backup_restore_azure_blob_storage/__init__.py create mode 100644 tests/integration/test_backup_restore_azure_blob_storage/configs/config.xml create mode 100644 tests/integration/test_backup_restore_azure_blob_storage/configs/disable_profilers.xml create mode 100644 tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml create mode 100644 tests/integration/test_backup_restore_azure_blob_storage/test.py diff --git a/src/Backups/BackupFactory.cpp b/src/Backups/BackupFactory.cpp index 898ac7bc490..31e87a21fc2 100644 --- a/src/Backups/BackupFactory.cpp +++ b/src/Backups/BackupFactory.cpp @@ -33,11 +33,13 @@ void BackupFactory::registerBackupEngine(const String & engine_name, const Creat void registerBackupEnginesFileAndDisk(BackupFactory &); void registerBackupEngineS3(BackupFactory &); +void registerBackupEngineAzureBlobStorage(BackupFactory &); void registerBackupEngines(BackupFactory & factory) { registerBackupEnginesFileAndDisk(factory); registerBackupEngineS3(factory); + registerBackupEngineAzureBlobStorage(factory); } BackupFactory::BackupFactory() diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp new file mode 100644 index 00000000000..d41d23e3c36 --- /dev/null +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -0,0 +1,336 @@ +#include + +#if USE_AZURE_BLOB_STORAGE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + + +namespace fs = std::filesystem; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int AZURE_BLOB_STORAGE_ERROR; + extern const int LOGICAL_ERROR; +} + +//using AzureClientPtr = std::shared_ptr; + +BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( + StorageAzureBlob::Configuration configuration_, + const ReadSettings & read_settings_, + const WriteSettings & write_settings_, + const ContextPtr & context_) + : BackupReaderDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupReaderAzureBlobStorage")) + , data_source_description{DataSourceType::AzureBlobStorage, "AzureBlobStorage", false, false} + , configuration(configuration_) +{ + client = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); + settings = StorageAzureBlob::createSettingsAsSharedPtr(context_); + auto settings_as_unique_ptr = StorageAzureBlob::createSettings(context_); + object_storage = std::make_unique("BackupReaderAzureBlobStorage", + std::make_unique(*client.get()), + std::move(settings_as_unique_ptr)); +} + +BackupReaderAzureBlobStorage::~BackupReaderAzureBlobStorage() = default; + +bool BackupReaderAzureBlobStorage::fileExists(const String & file_name) +{ + String key; + if (startsWith(file_name, ".")) + { + key= configuration.blob_path + file_name; + } + else + { + key = file_name; + } + return object_storage->exists(StoredObject(key)); +} + +UInt64 BackupReaderAzureBlobStorage::getFileSize(const String & file_name) +{ + String key; + if (startsWith(file_name, ".")) + { + key= configuration.blob_path + file_name; + } + else + { + key = file_name; + } + ObjectMetadata object_metadata = object_storage->getObjectMetadata(key); + return object_metadata.size_bytes; +} + +std::unique_ptr BackupReaderAzureBlobStorage::readFile(const String & file_name) +{ + String key; + if (startsWith(file_name, ".")) + { + key= configuration.blob_path + file_name; + } + else + { + key = file_name; + } + return std::make_unique( + client, key, read_settings, settings->max_single_read_retries, + settings->max_single_download_retries); +} + +void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup, + DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) +{ + LOG_INFO(&Poco::Logger::get("BackupReaderAzureBlobStorage"), "Enter copyFileToDisk"); + + /// Use the native copy as a more optimal way to copy a file from AzureBlobStorage to AzureBlobStorage if it's possible. + /// We don't check for `has_throttling` here because the native copy almost doesn't use network. + auto destination_data_source_description = destination_disk->getDataSourceDescription(); + if (destination_data_source_description.sameKind(data_source_description) + && (destination_data_source_description.is_encrypted == encrypted_in_backup)) + { + LOG_TRACE(log, "Copying {} from AzureBlobStorage to disk {}", path_in_backup, destination_disk->getName()); + auto write_blob_function = [&](const Strings & blob_path, WriteMode mode, const std::optional & object_attributes) -> size_t + { + /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files. + if (blob_path.size() != 2 || mode != WriteMode::Rewrite) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Blob writing function called with unexpected blob_path.size={} or mode={}", + blob_path.size(), mode); + + std::shared_ptr dest_client; + if (configuration.container == blob_path[1]) + { + dest_client = client; + } + else + { + StorageAzureBlob::Configuration dest_configuration = configuration; + dest_configuration.container = blob_path[1]; + dest_configuration.blob_path = blob_path[0]; + dest_client = StorageAzureBlob::createClient(dest_configuration, /* is_read_only */ false); + } + + + copyAzureBlobStorageFile( + client, + dest_client, + configuration.container, + fs::path(configuration.blob_path) / path_in_backup, + 0, + file_size, + /* dest_bucket= */ blob_path[1], + /* dest_key= */ blob_path[0], + settings, + read_settings, + object_attributes, + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupReaderAzureBlobStorage"), + /* for_disk_azure_blob_storage= */ true); + + return file_size; + }; + + destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function); + return; /// copied! + } + + /// Fallback to copy through buffers. + BackupReaderDefault::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode); +} + + +BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( + StorageAzureBlob::Configuration configuration_, + const ReadSettings & read_settings_, + const WriteSettings & write_settings_, + const ContextPtr & context_) + : BackupWriterDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupWriterAzureBlobStorage")) + , data_source_description{DataSourceType::AzureBlobStorage, "AzureBlobStorage", false, false} + , configuration(configuration_) +{ + client = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); + settings = StorageAzureBlob::createSettingsAsSharedPtr(context_); + auto settings_as_unique_ptr = StorageAzureBlob::createSettings(context_); + object_storage = std::make_unique("BackupWriterAzureBlobStorage", + std::make_unique(*client.get()), + std::move(settings_as_unique_ptr)); +} + +void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, + bool copy_encrypted, UInt64 start_pos, UInt64 length) +{ + /// Use the native copy as a more optimal way to copy a file from AzureBlobStorage to AzureBlobStorage if it's possible. + auto source_data_source_description = src_disk->getDataSourceDescription(); + if (source_data_source_description.sameKind(data_source_description) && (source_data_source_description.is_encrypted == copy_encrypted)) + { + /// getBlobPath() can return more than 3 elements if the file is stored as multiple objects in AzureBlobStorage bucket. + /// In this case we can't use the native copy. + if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 2) + { + + std::shared_ptr src_client; + if (configuration.container == blob_path[1]) + { + src_client = client; + } + else + { + StorageAzureBlob::Configuration src_configuration = configuration; + src_configuration.container = blob_path[1]; + src_configuration.blob_path = blob_path[0]; + src_client = StorageAzureBlob::createClient(src_configuration, /* is_read_only */ false); + } + + LOG_TRACE(log, "Copying file {} from disk {} to AzureBlobStorag", src_path, src_disk->getName()); + copyAzureBlobStorageFile( + src_client, + client, + /* src_bucket */ blob_path[1], + /* src_key= */ blob_path[0], + start_pos, + length, + configuration.container, + fs::path(configuration.blob_path) / path_in_backup, + settings, + read_settings, + {}, + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterAzureBlobStorage")); + return; /// copied! + } + } + + /// Fallback to copy through buffers. + BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length); +} + +void BackupWriterAzureBlobStorage::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) +{ + copyDataToAzureBlobStorageFile(create_read_buffer, start_pos, length, client, configuration.container, path_in_backup, settings, {}, + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterAzureBlobStorage")); +} + +BackupWriterAzureBlobStorage::~BackupWriterAzureBlobStorage() = default; + +bool BackupWriterAzureBlobStorage::fileExists(const String & file_name) +{ + String key; + if (startsWith(file_name, ".")) + { + key= configuration.blob_path + file_name; + } + else + { + key = file_name; + } + LOG_INFO(&Poco::Logger::get("BackupWriterAzureBlobStorage"), "Result fileExists {} ", object_storage->exists(StoredObject(key))); + + return object_storage->exists(StoredObject(key)); +} + +UInt64 BackupWriterAzureBlobStorage::getFileSize(const String & file_name) +{ + LOG_INFO(&Poco::Logger::get("BackupWriterAzureBlobStorage"), "Enter getFileSize"); + String key; + if (startsWith(file_name, ".")) + { + key= configuration.blob_path + file_name; + } + else + { + key = file_name; + } + RelativePathsWithMetadata children; + object_storage->listObjects(key,children,/*max_keys*/0); + if (children.empty()) + throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Object {} must exist"); + return children[0].metadata.size_bytes; +} + +std::unique_ptr BackupWriterAzureBlobStorage::readFile(const String & file_name, size_t /*expected_file_size*/) +{ + String key; + if (startsWith(file_name, ".")) + { + key= configuration.blob_path + file_name; + } + else + { + key = file_name; + } + + return std::make_unique( + client, key, read_settings, settings->max_single_read_retries, + settings->max_single_download_retries); +} + +std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const String & file_name) +{ + String key; + if (startsWith(file_name, ".")) + { + key= configuration.blob_path + file_name; + } + else + { + key = file_name; + } + return std::make_unique( + client, + key, + settings->max_single_part_upload_size, + DBMS_DEFAULT_BUFFER_SIZE, + write_settings); +} + +void BackupWriterAzureBlobStorage::removeFile(const String & file_name) +{ + String key; + if (startsWith(file_name, ".")) + { + key= configuration.blob_path + file_name; + } + else + { + key = file_name; + } + StoredObject object(key); + object_storage->removeObjectIfExists(object); +} + +void BackupWriterAzureBlobStorage::removeFiles(const Strings & keys) +{ + StoredObjects objects; + for (const auto & key : keys) + objects.emplace_back(key); + + object_storage->removeObjectsIfExist(objects); + +} + +void BackupWriterAzureBlobStorage::removeFilesBatch(const Strings & keys) +{ + StoredObjects objects; + for (const auto & key : keys) + objects.emplace_back(key); + + object_storage->removeObjectsIfExist(objects); +} + +} + +#endif diff --git a/src/Backups/BackupIO_AzureBlobStorage.h b/src/Backups/BackupIO_AzureBlobStorage.h new file mode 100644 index 00000000000..6ef66fc432d --- /dev/null +++ b/src/Backups/BackupIO_AzureBlobStorage.h @@ -0,0 +1,69 @@ +#pragma once + +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE +#include +#include +#include +#include + + +namespace DB +{ + +// using AzureClientPtr = std::shared_ptr; + +/// Represents a backup stored to Azure + class BackupReaderAzureBlobStorage : public BackupReaderDefault + { + public: + BackupReaderAzureBlobStorage(StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_); + ~BackupReaderAzureBlobStorage() override; + + bool fileExists(const String & file_name) override; + UInt64 getFileSize(const String & file_name) override; + std::unique_ptr readFile(const String & file_name) override; + + void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup, + DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override; + + private: + const DataSourceDescription data_source_description; + std::shared_ptr client; + StorageAzureBlob::Configuration configuration; + std::unique_ptr object_storage; + std::shared_ptr settings; + }; + + + class BackupWriterAzureBlobStorage : public BackupWriterDefault + { + public: + BackupWriterAzureBlobStorage(StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_); + ~BackupWriterAzureBlobStorage() override; + + bool fileExists(const String & file_name) override; + UInt64 getFileSize(const String & file_name) override; + std::unique_ptr writeFile(const String & file_name) override; + + void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) override; + void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, + bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + + void removeFile(const String & file_name) override; + void removeFiles(const Strings & file_names) override; + + private: + std::unique_ptr readFile(const String & file_name, size_t expected_file_size) override; + void removeFilesBatch(const Strings & file_names); + const DataSourceDescription data_source_description; + std::shared_ptr client; + StorageAzureBlob::Configuration configuration; + std::unique_ptr object_storage; + std::shared_ptr settings; + }; + +} + +#endif diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index bb97335d8fb..9363ca5e7a7 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -492,6 +492,7 @@ void BackupImpl::checkBackupDoesntExist() const else file_name_to_check_existence = ".backup"; + LOG_INFO(&Poco::Logger::get("BackupImpl"), "checkBackupDoesntExist 1"); if (writer->fileExists(file_name_to_check_existence)) throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} already exists", backup_name_for_logging); @@ -499,6 +500,7 @@ void BackupImpl::checkBackupDoesntExist() const if (!is_internal_backup) { assert(!lock_file_name.empty()); + LOG_INFO(&Poco::Logger::get("BackupImpl"), "checkBackupDoesntExist 2"); if (writer->fileExists(lock_file_name)) throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} is being written already", backup_name_for_logging); } @@ -522,6 +524,8 @@ bool BackupImpl::checkLockFile(bool throw_if_failed) const if (throw_if_failed) { + LOG_INFO(&Poco::Logger::get("BackupImpl"), "checkLockFile"); + if (!writer->fileExists(lock_file_name)) { throw Exception( @@ -886,12 +890,12 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry) } else if (src_disk && from_immutable_file) { - LOG_TRACE(log, "Writing backup for file {} from {} (disk {}): data file #{}", info.data_file_name, src_file_desc, src_disk->getName(), info.data_file_index); + LOG_INFO(log, "Writing backup for file {} from {} (disk {}): data file #{}", info.data_file_name, src_file_desc, src_disk->getName(), info.data_file_index); writer->copyFileFromDisk(info.data_file_name, src_disk, src_file_path, info.encrypted_by_disk, info.base_size, info.size - info.base_size); } else { - LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}", info.data_file_name, src_file_desc, info.data_file_index); + LOG_INFO(log, "Writing backup for file {} from {}: data file #{}", info.data_file_name, src_file_desc, info.data_file_index); auto create_read_buffer = [entry, read_settings = writer->getReadSettings()] { return entry->getReadBuffer(read_settings); }; writer->copyDataToFile(info.data_file_name, create_read_buffer, info.base_size, info.size - info.base_size); } diff --git a/src/Backups/registerBackupEngineAzureBlobStorage.cpp b/src/Backups/registerBackupEngineAzureBlobStorage.cpp new file mode 100644 index 00000000000..6f7b5f38c28 --- /dev/null +++ b/src/Backups/registerBackupEngineAzureBlobStorage.cpp @@ -0,0 +1,134 @@ +#include "config.h" + +#include +#include + +#if USE_AZURE_BLOB_STORAGE +#include +#include +#include +#include +#include +#include +#include +#endif + + +namespace DB +{ +namespace fs = std::filesystem; + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int SUPPORT_IS_DISABLED; +} + +#if USE_AZURE_BLOB_STORAGE +namespace +{ + String removeFileNameFromURL(String & url) + { + Poco::URI url2{url}; + String path = url2.getPath(); + size_t slash_pos = path.find_last_of('/'); + String file_name = path.substr(slash_pos + 1); + path.resize(slash_pos + 1); + url2.setPath(path); + url = url2.toString(); + return file_name; + } +} +#endif + + +void registerBackupEngineAzureBlobStorage(BackupFactory & factory) +{ + auto creator_fn = []([[maybe_unused]] const BackupFactory::CreateParams & params) -> std::unique_ptr + { +#if USE_AZURE_BLOB_STORAGE + const String & id_arg = params.backup_info.id_arg; + const auto & args = params.backup_info.args; + + LOG_INFO(&Poco::Logger::get("registerBackupEngineAzureBlobStorage"), "Begin id_arg={} args.size={}", id_arg, args.size()); + + StorageAzureBlob::Configuration configuration; + + if (args.size() == 4) + { + configuration.connection_url = args[0].safeGet(); + configuration.is_connection_string = true; + + configuration.container = args[1].safeGet(); + configuration.blob_path = args[2].safeGet(); + configuration.format = args[3].safeGet(); + + LOG_TRACE(&Poco::Logger::get("registerBackupEngineAzureBlobStorage"), "configuration.connection_url = {}" + "configuration.container = {}" + "configuration.blob_path = {}" + "configuration.format = {}", + configuration.connection_url, configuration.container, configuration.blob_path, configuration.format); + } + + + BackupImpl::ArchiveParams archive_params; + if (hasRegisteredArchiveFileExtension(configuration.blob_path)) + { + if (params.is_internal_backup) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Using archives with backups on clusters is disabled"); + + archive_params.archive_name = removeFileNameFromURL(configuration.blob_path); + archive_params.compression_method = params.compression_method; + archive_params.compression_level = params.compression_level; + archive_params.password = params.password; + } + else + { + if (!params.password.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Password is not applicable, backup cannot be encrypted"); + } + + + if (params.open_mode == IBackup::OpenMode::READ) + { + auto reader = std::make_shared(configuration, + params.read_settings, + params.write_settings, + params.context); + + return std::make_unique( + params.backup_info, + archive_params, + params.base_backup_info, + reader, + params.context, + /*params.use_same_s3_credentials_for_base_backup*/ false); + } + else + { + auto writer = std::make_shared(configuration, + params.read_settings, + params.write_settings, + params.context); + + return std::make_unique( + params.backup_info, + archive_params, + params.base_backup_info, + writer, + params.context, + params.is_internal_backup, + params.backup_coordination, + params.backup_uuid, + params.deduplicate_files, + /*params.use_same_s3_credentials_for_base_backup*/ false); + } +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "AzureBlobStorage support is disabled"); +#endif + }; + + factory.registerBackupEngine("AzureBlobStorage", creator_fn); +} + +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0257b7d329b..984594a6541 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -87,6 +87,7 @@ add_headers_and_sources(clickhouse_common_io IO) add_headers_and_sources(clickhouse_common_io IO/Archives) add_headers_and_sources(clickhouse_common_io IO/Resource) add_headers_and_sources(clickhouse_common_io IO/S3) +add_headers_and_sources(clickhouse_common_io IO/AzureBlobStorage) list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/new_delete.cpp) @@ -139,6 +140,7 @@ endif() if (TARGET ch_contrib::azure_sdk) add_headers_and_sources(dbms Disks/ObjectStorages/AzureBlobStorage) + add_headers_and_sources(dbms IO/AzureBlobStorage) endif() if (TARGET ch_contrib::hdfs) @@ -485,6 +487,7 @@ if (TARGET ch_contrib::aws_s3) endif() if (TARGET ch_contrib::azure_sdk) + target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::azure_sdk) dbms_target_link_libraries (PRIVATE ch_contrib::azure_sdk) endif() diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 58e860ebcaf..1655d19986a 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -361,6 +361,10 @@ The server successfully detected this situation and will download merged part fr M(S3PutObject, "Number of S3 API PutObject calls.") \ M(S3GetObject, "Number of S3 API GetObject calls.") \ \ + M(AzureUploadPart, "Number of Azure blob storage API UploadPart calls") \ + M(DiskAzureUploadPart, "Number of Disk Azure blob storage API UploadPart calls") \ + M(AzureCopyObject, "Number of Azure blob storage API CopyObject calls") \ + M(DiskAzureCopyObject, "Number of Disk Azure blob storage API CopyObject calls") \ M(AzureDeleteObjects, "Number of Azure blob storage API DeleteObject(s) calls.") \ M(AzureListObjects, "Number of Azure blob storage API ListObjects calls.") \ \ diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp new file mode 100644 index 00000000000..bf0bcac664b --- /dev/null +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -0,0 +1,324 @@ +#include + +#if USE_AZURE_BLOB_STORAGE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event AzureCopyObject; + extern const Event AzureUploadPart; + + extern const Event DiskAzureCopyObject; + extern const Event DiskAzureUploadPart; +} + + +namespace DB +{ + +size_t max_single_operation_copy_size = 256 * 1024 * 1024; + + +namespace +{ + class UploadHelper + { + public: + UploadHelper( + const CreateReadBuffer & create_read_buffer_, + std::shared_ptr client_, + size_t offset_, + size_t total_size_, + const String & dest_bucket_, + const String & dest_key_, + std::shared_ptr settings_, + const std::optional> & object_metadata_, + ThreadPoolCallbackRunner schedule_, + bool for_disk_azure_blob_storage_) + : create_read_buffer(create_read_buffer_) + , client(client_) + , offset (offset_) + , total_size (total_size_) + , dest_bucket(dest_bucket_) + , dest_key(dest_key_) + , settings(settings_) + , object_metadata(object_metadata_) + , schedule(schedule_) + , for_disk_azure_blob_storage(for_disk_azure_blob_storage_) + , log(&Poco::Logger::get("azureBlobStorageUploadHelper")) + , max_single_part_upload_size(settings_.get()->max_single_part_upload_size) + { + } + + ~UploadHelper() {} + + protected: + std::function()> create_read_buffer; + std::shared_ptr client; + size_t offset; + size_t total_size; + const String & dest_bucket; + const String & dest_key; + std::shared_ptr settings; + const std::optional> & object_metadata; + ThreadPoolCallbackRunner schedule; + bool for_disk_azure_blob_storage; + const Poco::Logger * log; + size_t max_single_part_upload_size; + + struct UploadPartTask + { + char *data = nullptr; + size_t size = 0; + std::string block_id; + bool is_finished = false; + std::exception_ptr exception; + + ~UploadPartTask() + { + if (data != nullptr) + free(data); + } + }; + + size_t normal_part_size; + std::vector block_ids; + + std::list TSA_GUARDED_BY(bg_tasks_mutex) bg_tasks; + int num_added_bg_tasks TSA_GUARDED_BY(bg_tasks_mutex) = 0; + int num_finished_bg_tasks TSA_GUARDED_BY(bg_tasks_mutex) = 0; + std::mutex bg_tasks_mutex; + std::condition_variable bg_tasks_condvar; + + public: + void performCopy() + { + performMultipartUpload(); + } + + void completeMultipartUpload() + { + auto block_blob_client = client->GetBlockBlobClient(dest_key); + block_blob_client.CommitBlockList(block_ids); + } + + void performMultipartUpload() + { + normal_part_size = 1024; + + size_t position = offset; + size_t end_position = offset + total_size; + + try + { + while (position < end_position) + { + size_t next_position = std::min(position + normal_part_size, end_position); + size_t part_size = next_position - position; /// `part_size` is either `normal_part_size` or smaller if it's the final part. + + uploadPart(position, part_size); + + position = next_position; + } + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + waitForAllBackgroundTasks(); + throw; + } + + waitForAllBackgroundTasks(); + completeMultipartUpload(); + } + + + void uploadPart(size_t part_offset, size_t part_size) + { + LOG_TRACE(log, "Writing part. Bucket: {}, Key: {}, Size: {}", dest_bucket, dest_key, part_size); + + if (!part_size) + { + LOG_TRACE(log, "Skipping writing an empty part."); + return; + } + + if (schedule) + { + UploadPartTask * task = nullptr; + + { + std::lock_guard lock(bg_tasks_mutex); + task = &bg_tasks.emplace_back(); + ++num_added_bg_tasks; + } + + /// Notify waiting thread when task finished + auto task_finish_notify = [this, task]() + { + std::lock_guard lock(bg_tasks_mutex); + task->is_finished = true; + ++num_finished_bg_tasks; + + /// Notification under mutex is important here. + /// Otherwise, WriteBuffer could be destroyed in between + /// Releasing lock and condvar notification. + bg_tasks_condvar.notify_one(); + }; + + try + { + auto read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); + auto buffer = std::make_unique(std::move(read_buffer), part_size); + task->data = new char[part_size]; + task->size = part_size; + buffer->read(task->data,part_size); + task->block_id = getRandomASCIIString(64); + + schedule([this, task, task_finish_notify]() + { + try + { + processUploadTask(*task); + } + catch (...) + { + task->exception = std::current_exception(); + } + task_finish_notify(); + }, Priority{}); + } + catch (...) + { + task_finish_notify(); + throw; + } + } + else + { + UploadPartTask task; + auto read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); + auto buffer = std::make_unique(std::move(read_buffer), part_size); + task.data = new char[part_size]; + buffer->read(task.data,part_size); + task.size = part_size; + processUploadTask(task); + block_ids.emplace_back(task.block_id); + } + } + + void processUploadTask(UploadPartTask & task) + { + auto block_id = processUploadPartRequest(task); + + std::lock_guard lock(bg_tasks_mutex); /// Protect bg_tasks from race + task.block_id = block_id; + LOG_TRACE(log, "Writing part finished. Bucket: {}, Key: {}, block_id: {}, Parts: {}", dest_bucket, dest_key, block_id, bg_tasks.size()); + } + + String processUploadPartRequest(UploadPartTask & task) + { + ProfileEvents::increment(ProfileEvents::AzureUploadPart); + if (for_disk_azure_blob_storage) + ProfileEvents::increment(ProfileEvents::DiskAzureUploadPart); + + auto block_blob_client = client->GetBlockBlobClient(dest_key); + task.block_id = getRandomASCIIString(64); + Azure::Core::IO::MemoryBodyStream memory(reinterpret_cast(task.data), task.size); + block_blob_client.StageBlock(task.block_id, memory); + + return task.block_id; + } + + + void waitForAllBackgroundTasks() + { + if (!schedule) + return; + + std::unique_lock lock(bg_tasks_mutex); + /// Suppress warnings because bg_tasks_mutex is actually hold, but tsa annotations do not understand std::unique_lock + bg_tasks_condvar.wait(lock, [this]() {return TSA_SUPPRESS_WARNING_FOR_READ(num_added_bg_tasks) == TSA_SUPPRESS_WARNING_FOR_READ(num_finished_bg_tasks); }); + + auto & tasks = TSA_SUPPRESS_WARNING_FOR_WRITE(bg_tasks); + for (auto & task : tasks) + { + if (task.exception) + std::rethrow_exception(task.exception); + block_ids.emplace_back(task.block_id); + } + } + }; +} + + +void copyDataToAzureBlobStorageFile( + const std::function()> & create_read_buffer, + size_t offset, + size_t size, + std::shared_ptr & dest_client, + const String & dest_bucket, + const String & dest_key, + std::shared_ptr settings, + const std::optional> & object_metadata, + ThreadPoolCallbackRunner schedule, + bool for_disk_azure_blob_storage) +{ + UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_azure_blob_storage}; + helper.performCopy(); +} + + +void copyAzureBlobStorageFile( + std::shared_ptr src_client, + std::shared_ptr dest_client, + const String & src_bucket, + const String & src_key, + size_t offset, + size_t size, + const String & dest_bucket, + const String & dest_key, + std::shared_ptr settings, + const ReadSettings & read_settings, + const std::optional> & object_metadata, + ThreadPoolCallbackRunner schedule, + bool for_disk_azure_blob_storage) +{ + + if (size < max_single_operation_copy_size) + { + ProfileEvents::increment(ProfileEvents::AzureCopyObject); + if (for_disk_azure_blob_storage) + ProfileEvents::increment(ProfileEvents::DiskAzureCopyObject); + auto block_blob_client_src = src_client->GetBlockBlobClient(src_key); + auto block_blob_client_dest = dest_client->GetBlockBlobClient(dest_key); + auto uri = block_blob_client_src.GetUrl(); + block_blob_client_dest.CopyFromUri(uri); + } + else + { + LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Bucket: {}, Key: {}", src_bucket, src_key); + auto create_read_buffer = [&] + { + return std::make_unique(src_client, src_key, read_settings, settings->max_single_read_retries, + settings->max_single_download_retries); + }; + + UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_azure_blob_storage}; + helper.performCopy(); + } +} + +} + +#endif diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h new file mode 100644 index 00000000000..31228fbcb23 --- /dev/null +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h @@ -0,0 +1,58 @@ +#pragma once + +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +class SeekableReadBuffer; + +using CreateReadBuffer = std::function()>; + +/// Copies a file from AzureBlobStorage to AzureBlobStorage. +/// The parameters `src_offset` and `src_size` specify a part in the source to copy. +void copyAzureBlobStorageFile( + std::shared_ptr src_client, + std::shared_ptr dest_client, + const String & src_bucket, + const String & src_key, + size_t src_offset, + size_t src_size, + const String & dest_bucket, + const String & dest_key, + std::shared_ptr settings, + const ReadSettings & read_settings, + const std::optional> & object_metadata = std::nullopt, + ThreadPoolCallbackRunner schedule_ = {}, + bool for_disk_azure_blob_storage = false); + + +/// Copies data from any seekable source to AzureBlobStorage. +/// The same functionality can be done by using the function copyData() and the class WriteBufferFromS3 +/// however copyDataToS3File() is faster and spends less memory. +/// The callback `create_read_buffer` can be called from multiple threads in parallel, so that should be thread-safe. +/// The parameters `offset` and `size` specify a part in the source to copy. +void copyDataToAzureBlobStorageFile( + const std::function()> & create_read_buffer, + size_t offset, + size_t size, + std::shared_ptr & client, + const String & dest_bucket, + const String & dest_key, + std::shared_ptr settings, + const std::optional> & object_metadata = std::nullopt, + ThreadPoolCallbackRunner schedule_ = {}, + bool for_disk_azure_blob_storage = false); + +} + +#endif diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp index 2e0703a8df3..e36604cfb1a 100644 --- a/src/Storages/StorageAzureBlob.cpp +++ b/src/Storages/StorageAzureBlob.cpp @@ -258,6 +258,17 @@ AzureObjectStorage::SettingsPtr StorageAzureBlob::createSettings(ContextPtr loca return settings_ptr; } +std::shared_ptr StorageAzureBlob::createSettingsAsSharedPtr(ContextPtr local_context) +{ + const auto & context_settings = local_context->getSettingsRef(); + auto settings_ptr = std::make_shared(); + settings_ptr->max_single_part_upload_size = context_settings.azure_max_single_part_upload_size; + settings_ptr->max_single_read_retries = context_settings.azure_max_single_read_retries; + settings_ptr->list_object_keys_size = static_cast(context_settings.azure_list_object_keys_size); + + return settings_ptr; +} + void registerStorageAzureBlob(StorageFactory & factory) { factory.registerStorage("AzureBlobStorage", [](const StorageFactory::Arguments & args) diff --git a/src/Storages/StorageAzureBlob.h b/src/Storages/StorageAzureBlob.h index b97dee0caed..570e4124d73 100644 --- a/src/Storages/StorageAzureBlob.h +++ b/src/Storages/StorageAzureBlob.h @@ -80,6 +80,7 @@ public: static AzureClientPtr createClient(StorageAzureBlob::Configuration configuration, bool is_read_only); static AzureObjectStorage::SettingsPtr createSettings(ContextPtr local_context); + static std::shared_ptr createSettingsAsSharedPtr(ContextPtr local_context); static void processNamedCollectionResult(StorageAzureBlob::Configuration & configuration, const NamedCollection & collection); diff --git a/tests/integration/test_backup_restore_azure_blob_storage/__init__.py b/tests/integration/test_backup_restore_azure_blob_storage/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_backup_restore_azure_blob_storage/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_backup_restore_azure_blob_storage/configs/config.xml b/tests/integration/test_backup_restore_azure_blob_storage/configs/config.xml new file mode 100644 index 00000000000..5725dce40cd --- /dev/null +++ b/tests/integration/test_backup_restore_azure_blob_storage/configs/config.xml @@ -0,0 +1,11 @@ + + 1 + 0 + 0.0 + 0 + 1 + 1 + 0 + 16 + 16 + \ No newline at end of file diff --git a/tests/integration/test_backup_restore_azure_blob_storage/configs/disable_profilers.xml b/tests/integration/test_backup_restore_azure_blob_storage/configs/disable_profilers.xml new file mode 100644 index 00000000000..b74bb1502ce --- /dev/null +++ b/tests/integration/test_backup_restore_azure_blob_storage/configs/disable_profilers.xml @@ -0,0 +1,13 @@ + + + + + 0 + 0 + 0 + 1000 + 1 + 1 + + + diff --git a/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml b/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml new file mode 100644 index 00000000000..c12eb2f79f4 --- /dev/null +++ b/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml @@ -0,0 +1,8 @@ + + + + + default + + + diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py new file mode 100644 index 00000000000..2ecf08a4f40 --- /dev/null +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +import gzip +import json +import logging +import os +import io +import random +import threading +import time + +from azure.storage.blob import BlobServiceClient +import helpers.client +import pytest +from helpers.cluster import ClickHouseCluster, ClickHouseInstance +from helpers.network import PartitionManager +from helpers.mock_servers import start_mock_servers +from helpers.test_tools import exec_query_with_retry + + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node", + main_configs=["configs/config.xml"], + user_configs=["configs/disable_profilers.xml", "configs/users.xml"], + with_azurite=True, + ) + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +def azure_query( + node, query, expect_error="false", try_num=10, settings={}, query_on_retry=None +): + for i in range(try_num): + try: + if expect_error == "true": + return node.query_and_get_error(query, settings=settings) + else: + return node.query(query, settings=settings) + except Exception as ex: + retriable_errors = [ + "DB::Exception: Azure::Core::Http::TransportException: Connection was closed by the server while trying to read a response", + "DB::Exception: Azure::Core::Http::TransportException: Connection closed before getting full response or response is less than expected", + "DB::Exception: Azure::Core::Http::TransportException: Connection was closed by the server while trying to read a response", + "DB::Exception: Azure::Core::Http::TransportException: Error while polling for socket ready read", + "Azure::Core::Http::TransportException, e.what() = Connection was closed by the server while trying to read a response", + "Azure::Core::Http::TransportException, e.what() = Connection closed before getting full response or response is less than expected", + "Azure::Core::Http::TransportException, e.what() = Connection was closed by the server while trying to read a response", + "Azure::Core::Http::TransportException, e.what() = Error while polling for socket ready read", + ] + retry = False + for error in retriable_errors: + if error in str(ex): + retry = True + print(f"Try num: {i}. Having retriable error: {ex}") + time.sleep(i) + break + if not retry or i == try_num - 1: + raise Exception(ex) + if query_on_retry is not None: + node.query(query_on_retry) + continue + + +def get_azure_file_content(filename, port): + container_name = "cont" + connection_string = ( + f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + f"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + f"BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;" + ) + blob_service_client = BlobServiceClient.from_connection_string( + str(connection_string) + ) + container_client = blob_service_client.get_container_client(container_name) + blob_client = container_client.get_blob_client(filename) + download_stream = blob_client.download_blob() + return download_stream.readall().decode("utf-8") + + +def put_azure_file_content(filename, port, data): + container_name = "cont" + connection_string = ( + f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + f"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + f"BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;" + ) + blob_service_client = BlobServiceClient.from_connection_string(connection_string) + try: + container_client = blob_service_client.create_container(container_name) + except: + container_client = blob_service_client.get_container_client(container_name) + + blob_client = container_client.get_blob_client(filename) + buf = io.BytesIO(data) + blob_client.upload_blob(buf) + +@pytest.fixture(autouse=True, scope="function") +def delete_all_files(cluster): + port = cluster.env_variables["AZURITE_PORT"] + connection_string = ( + f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + f"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + f"BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;" + ) + blob_service_client = BlobServiceClient.from_connection_string(connection_string) + containers = blob_service_client.list_containers() + for container in containers: + container_client = blob_service_client.get_container_client(container) + blob_list = container_client.list_blobs() + for blob in blob_list: + print(blob) + blob_client = container_client.get_blob_client(blob) + blob_client.delete_blob() + + assert len(list(container_client.list_blobs())) == 0 + + yield + + +def test_create_table_connection_string(cluster): + node = cluster.instances["node"] + azure_query( + node, + f"CREATE TABLE test_create_table_conn_string (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_create_connection_string', 'CSV')", + ) + +def test_backup_restore(cluster): + node = cluster.instances["node"] + port = cluster.env_variables["AZURITE_PORT"] + azure_query( + node, + f"CREATE TABLE test_simple_write_connection_string (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write_c.csv', 'CSV')", + ) + azure_query(node, f"INSERT INTO test_simple_write_connection_string VALUES (1, 'a')") + print(get_azure_file_content("test_simple_write_c.csv", port)) + assert get_azure_file_content("test_simple_write_c.csv", port) == '1,"a"\n' + + backup_destination = f"AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write_c_backup.csv', 'CSV')" + azure_query(node,f"BACKUP TABLE test_simple_write_connection_string TO {backup_destination}") + print (get_azure_file_content("test_simple_write_c_backup.csv.backup", port)) + azure_query(node, f"RESTORE TABLE test_simple_write_connection_string AS test_simple_write_connection_string_restored FROM {backup_destination};") + assert(azure_query(node,f"SELECT * from test_simple_write_connection_string_restored") == "1\ta\n") \ No newline at end of file From 05b608cd76da8995086887f812e1ab3fceb99551 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Nov 2023 10:12:45 +0000 Subject: [PATCH 002/103] Automatic style fix --- .../test.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index 2ecf08a4f40..cda3cab07e4 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -18,7 +18,6 @@ from helpers.mock_servers import start_mock_servers from helpers.test_tools import exec_query_with_retry - @pytest.fixture(scope="module") def cluster(): try: @@ -103,6 +102,7 @@ def put_azure_file_content(filename, port, data): buf = io.BytesIO(data) blob_client.upload_blob(buf) + @pytest.fixture(autouse=True, scope="function") def delete_all_files(cluster): port = cluster.env_variables["AZURITE_PORT"] @@ -133,6 +133,7 @@ def test_create_table_connection_string(cluster): f"CREATE TABLE test_create_table_conn_string (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_create_connection_string', 'CSV')", ) + def test_backup_restore(cluster): node = cluster.instances["node"] port = cluster.env_variables["AZURITE_PORT"] @@ -140,12 +141,23 @@ def test_backup_restore(cluster): node, f"CREATE TABLE test_simple_write_connection_string (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write_c.csv', 'CSV')", ) - azure_query(node, f"INSERT INTO test_simple_write_connection_string VALUES (1, 'a')") + azure_query( + node, f"INSERT INTO test_simple_write_connection_string VALUES (1, 'a')" + ) print(get_azure_file_content("test_simple_write_c.csv", port)) assert get_azure_file_content("test_simple_write_c.csv", port) == '1,"a"\n' backup_destination = f"AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write_c_backup.csv', 'CSV')" - azure_query(node,f"BACKUP TABLE test_simple_write_connection_string TO {backup_destination}") - print (get_azure_file_content("test_simple_write_c_backup.csv.backup", port)) - azure_query(node, f"RESTORE TABLE test_simple_write_connection_string AS test_simple_write_connection_string_restored FROM {backup_destination};") - assert(azure_query(node,f"SELECT * from test_simple_write_connection_string_restored") == "1\ta\n") \ No newline at end of file + azure_query( + node, + f"BACKUP TABLE test_simple_write_connection_string TO {backup_destination}", + ) + print(get_azure_file_content("test_simple_write_c_backup.csv.backup", port)) + azure_query( + node, + f"RESTORE TABLE test_simple_write_connection_string AS test_simple_write_connection_string_restored FROM {backup_destination};", + ) + assert ( + azure_query(node, f"SELECT * from test_simple_write_connection_string_restored") + == "1\ta\n" + ) From 6dfb1c25ec6a4a61a4fe329191c10263eb19ad07 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 20 Nov 2023 11:37:06 +0100 Subject: [PATCH 003/103] Added docs --- docs/en/operations/backup.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index 6068b185ede..15d953249a0 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -451,3 +451,24 @@ To disallow concurrent backup/restore, you can use these settings respectively. The default value for both is true, so by default concurrent backup/restores are allowed. When these settings are false on a cluster, only 1 backup/restore is allowed to run on a cluster at a time. + +## Configuring BACKUP/RESTORE to use an AzureBlobStorage Endpoint + +To write backups to an AzureBlobStorage container you need the following pieces of information: +- AzureBlobStorage endpoint connection string / url, +- Container, +- Path, +- Account name (if url is specified) +- Account Key (if url is specified) + +The destination for a backup will be specified like this: +``` +AzureBlobStorage('/', '', '', '', ') +``` + +```sql +BACKUP TABLE data TO AzureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;', + 'test_container', 'data_backup'); +RESTORE TABLE data AS data_restored FROM AzureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;', + 'test_container', 'data_backup'); +``` From d0827e3ea77ff432c4a6a66145827428bcd62b5e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 1 Dec 2023 17:45:23 +0000 Subject: [PATCH 004/103] Add a test. --- .../0_stateless/02932_set_ttl_where.reference | 0 .../0_stateless/02932_set_ttl_where.sql | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/02932_set_ttl_where.reference create mode 100644 tests/queries/0_stateless/02932_set_ttl_where.sql diff --git a/tests/queries/0_stateless/02932_set_ttl_where.reference b/tests/queries/0_stateless/02932_set_ttl_where.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02932_set_ttl_where.sql b/tests/queries/0_stateless/02932_set_ttl_where.sql new file mode 100644 index 00000000000..85fddf613e8 --- /dev/null +++ b/tests/queries/0_stateless/02932_set_ttl_where.sql @@ -0,0 +1,22 @@ +create or replace table temp ( + a UInt32 +) +engine = MergeTree +order by a; + +insert into temp select number from system.numbers limit 100_000; + +create or replace table t_temp ( + a UInt32, + timestamp DateTime +) +engine = MergeTree +order by a +TTL timestamp + INTERVAL 2 SECOND WHERE a in (select a from temp); + +select sleep(1); +insert into t_temp select rand(), now() from system.numbers limit 1_000_000; +select sleep(1); +insert into t_temp select rand(), now() from system.numbers limit 1_000_000; +select sleep(1); +optimize table t_temp final; From 508046e6922c0cb163ce5611f1e6ef6a22f8b7f1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 1 Dec 2023 20:31:26 +0000 Subject: [PATCH 005/103] Attempt to support subqueries in TTL. --- src/Interpreters/PreparedSets.cpp | 8 ++- src/Interpreters/PreparedSets.h | 1 + src/Processors/TTL/ITTLAlgorithm.cpp | 5 +- src/Processors/TTL/ITTLAlgorithm.h | 9 ++- .../TTL/TTLAggregationAlgorithm.cpp | 11 ++-- src/Processors/TTL/TTLAggregationAlgorithm.h | 1 + src/Processors/TTL/TTLColumnAlgorithm.cpp | 5 +- src/Processors/TTL/TTLColumnAlgorithm.h | 1 + src/Processors/TTL/TTLDeleteAlgorithm.cpp | 10 +-- src/Processors/TTL/TTLDeleteAlgorithm.h | 2 +- src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp | 5 +- src/Processors/TTL/TTLUpdateInfoAlgorithm.h | 1 + src/Processors/Transforms/TTLTransform.cpp | 2 +- .../MergeTree/MergeTreeDataWriter.cpp | 19 +++--- src/Storages/StorageInMemoryMetadata.cpp | 21 +++---- src/Storages/TTLDescription.cpp | 62 ++++++++++++------- src/Storages/TTLDescription.h | 15 ++++- 17 files changed, 116 insertions(+), 62 deletions(-) diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index 955d8892284..ea8d9a62b8b 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -189,11 +189,17 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) } } + set_and_key->set->fillSetElements(); + + return buildSetInplace(context); +} + +SetPtr FutureSetFromSubquery::buildSetInplace(const ContextPtr & context) +{ auto plan = build(context); if (!plan) return nullptr; - set_and_key->set->fillSetElements(); auto builder = plan->buildQueryPipeline(QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context)); auto pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); pipeline.complete(std::make_shared(Block())); diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index e237789c63c..3e751d309ba 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -107,6 +107,7 @@ public: SetPtr get() const override; DataTypes getTypes() const override; SetPtr buildOrderedSetInplace(const ContextPtr & context) override; + SetPtr buildSetInplace(const ContextPtr & context); std::unique_ptr build(const ContextPtr & context); diff --git a/src/Processors/TTL/ITTLAlgorithm.cpp b/src/Processors/TTL/ITTLAlgorithm.cpp index 79140137df8..af6c4e4ac35 100644 --- a/src/Processors/TTL/ITTLAlgorithm.cpp +++ b/src/Processors/TTL/ITTLAlgorithm.cpp @@ -11,8 +11,9 @@ namespace ErrorCodes } ITTLAlgorithm::ITTLAlgorithm( - const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) - : description(description_) + const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) + : ttl_expressions(ttl_expressions_) + , description(description_) , old_ttl_info(old_ttl_info_) , current_time(current_time_) , force(force_) diff --git a/src/Processors/TTL/ITTLAlgorithm.h b/src/Processors/TTL/ITTLAlgorithm.h index 49cd2c46d9d..6e73286b564 100644 --- a/src/Processors/TTL/ITTLAlgorithm.h +++ b/src/Processors/TTL/ITTLAlgorithm.h @@ -8,6 +8,12 @@ namespace DB { +struct TTlExpressions +{ + ExpressionActionsPtr expression; + ExpressionActionsPtr where_expression; +}; + /** * Represents the actions, which are required to do * with data, when TTL is expired: delete, aggregate, etc. @@ -18,7 +24,7 @@ public: using TTLInfo = IMergeTreeDataPart::TTLInfo; using MutableDataPartPtr = MergeTreeMutableDataPartPtr; - ITTLAlgorithm(const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_); + ITTLAlgorithm(const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_); virtual ~ITTLAlgorithm() = default; virtual void execute(Block & block) = 0; @@ -39,6 +45,7 @@ protected: bool isTTLExpired(time_t ttl) const; UInt32 getTimestampByIndex(const IColumn * column, size_t index) const; + const TTlExpressions ttl_expressions; const TTLDescription description; const TTLInfo old_ttl_info; const time_t current_time; diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index fa3436ec55d..ab2ba5f58fc 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -5,13 +5,14 @@ namespace DB { TTLAggregationAlgorithm::TTLAggregationAlgorithm( + const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_, const Block & header_, const MergeTreeData & storage_) - : ITTLAlgorithm(description_, old_ttl_info_, current_time_, force_) + : ITTLAlgorithm(ttl_expressions_, description_, old_ttl_info_, current_time_, force_) , header(header_) { current_key_value.resize(description.group_by_keys.size()); @@ -73,8 +74,8 @@ void TTLAggregationAlgorithm::execute(Block & block) const auto & column_names = header.getNames(); MutableColumns aggregate_columns = header.cloneEmptyColumns(); - auto ttl_column = executeExpressionAndGetColumn(description.expression, block, description.result_column); - auto where_column = executeExpressionAndGetColumn(description.where_expression, block, description.where_result_column); + auto ttl_column = executeExpressionAndGetColumn(ttl_expressions.expression, block, description.result_column); + auto where_column = executeExpressionAndGetColumn(ttl_expressions.where_expression, block, description.where_result_column); size_t rows_aggregated = 0; size_t current_key_start = 0; @@ -145,8 +146,8 @@ void TTLAggregationAlgorithm::execute(Block & block) /// If some rows were aggregated we have to recalculate ttl info's if (some_rows_were_aggregated) { - auto ttl_column_after_aggregation = executeExpressionAndGetColumn(description.expression, block, description.result_column); - auto where_column_after_aggregation = executeExpressionAndGetColumn(description.where_expression, block, description.where_result_column); + auto ttl_column_after_aggregation = executeExpressionAndGetColumn(ttl_expressions.expression, block, description.result_column); + auto where_column_after_aggregation = executeExpressionAndGetColumn(ttl_expressions.where_expression, block, description.where_result_column); for (size_t i = 0; i < block.rows(); ++i) { bool where_filter_passed = !where_column_after_aggregation || where_column_after_aggregation->getBool(i); diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.h b/src/Processors/TTL/TTLAggregationAlgorithm.h index 0e4bf092ed6..9fd074efba8 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.h +++ b/src/Processors/TTL/TTLAggregationAlgorithm.h @@ -13,6 +13,7 @@ class TTLAggregationAlgorithm final : public ITTLAlgorithm { public: TTLAggregationAlgorithm( + const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, diff --git a/src/Processors/TTL/TTLColumnAlgorithm.cpp b/src/Processors/TTL/TTLColumnAlgorithm.cpp index 04c4d7b9348..cb99dcf99b1 100644 --- a/src/Processors/TTL/TTLColumnAlgorithm.cpp +++ b/src/Processors/TTL/TTLColumnAlgorithm.cpp @@ -4,6 +4,7 @@ namespace DB { TTLColumnAlgorithm::TTLColumnAlgorithm( + const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, @@ -12,7 +13,7 @@ TTLColumnAlgorithm::TTLColumnAlgorithm( const ExpressionActionsPtr & default_expression_, const String & default_column_name_, bool is_compact_part_) - : ITTLAlgorithm(description_, old_ttl_info_, current_time_, force_) + : ITTLAlgorithm(ttl_expressions_, description_, old_ttl_info_, current_time_, force_) , column_name(column_name_) , default_expression(default_expression_) , default_column_name(default_column_name_) @@ -49,7 +50,7 @@ void TTLColumnAlgorithm::execute(Block & block) if (default_column) default_column = default_column->convertToFullColumnIfConst(); - auto ttl_column = executeExpressionAndGetColumn(description.expression, block, description.result_column); + auto ttl_column = executeExpressionAndGetColumn(ttl_expressions.expression, block, description.result_column); auto & column_with_type = block.getByName(column_name); const IColumn * values_column = column_with_type.column.get(); diff --git a/src/Processors/TTL/TTLColumnAlgorithm.h b/src/Processors/TTL/TTLColumnAlgorithm.h index 30de77dcc2a..efcd7c74454 100644 --- a/src/Processors/TTL/TTLColumnAlgorithm.h +++ b/src/Processors/TTL/TTLColumnAlgorithm.h @@ -11,6 +11,7 @@ class TTLColumnAlgorithm final : public ITTLAlgorithm { public: TTLColumnAlgorithm( + const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, diff --git a/src/Processors/TTL/TTLDeleteAlgorithm.cpp b/src/Processors/TTL/TTLDeleteAlgorithm.cpp index f176df2d003..6a172e9c3c3 100644 --- a/src/Processors/TTL/TTLDeleteAlgorithm.cpp +++ b/src/Processors/TTL/TTLDeleteAlgorithm.cpp @@ -4,8 +4,8 @@ namespace DB { TTLDeleteAlgorithm::TTLDeleteAlgorithm( - const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) - : ITTLAlgorithm(description_, old_ttl_info_, current_time_, force_) + const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) + : ITTLAlgorithm(ttl_expressions_, description_, old_ttl_info_, current_time_, force_) { if (!isMinTTLExpired()) new_ttl_info = old_ttl_info; @@ -19,8 +19,8 @@ void TTLDeleteAlgorithm::execute(Block & block) if (!block || !isMinTTLExpired()) return; - auto ttl_column = executeExpressionAndGetColumn(description.expression, block, description.result_column); - auto where_column = executeExpressionAndGetColumn(description.where_expression, block, description.where_result_column); + auto ttl_column = executeExpressionAndGetColumn(ttl_expressions.expression, block, description.result_column); + auto where_column = executeExpressionAndGetColumn(ttl_expressions.where_expression, block, description.where_result_column); MutableColumns result_columns; const auto & column_names = block.getNames(); @@ -54,7 +54,7 @@ void TTLDeleteAlgorithm::execute(Block & block) void TTLDeleteAlgorithm::finalize(const MutableDataPartPtr & data_part) const { - if (description.where_expression) + if (ttl_expressions.where_expression) data_part->ttl_infos.rows_where_ttl[description.result_column] = new_ttl_info; else data_part->ttl_infos.table_ttl = new_ttl_info; diff --git a/src/Processors/TTL/TTLDeleteAlgorithm.h b/src/Processors/TTL/TTLDeleteAlgorithm.h index 292a29bfa27..23389070774 100644 --- a/src/Processors/TTL/TTLDeleteAlgorithm.h +++ b/src/Processors/TTL/TTLDeleteAlgorithm.h @@ -10,7 +10,7 @@ namespace DB class TTLDeleteAlgorithm final : public ITTLAlgorithm { public: - TTLDeleteAlgorithm(const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_); + TTLDeleteAlgorithm(const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_); void execute(Block & block) override; void finalize(const MutableDataPartPtr & data_part) const override; diff --git a/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp b/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp index eba364aa2b8..34c0cad70ea 100644 --- a/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp +++ b/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp @@ -4,13 +4,14 @@ namespace DB { TTLUpdateInfoAlgorithm::TTLUpdateInfoAlgorithm( + const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLUpdateField ttl_update_field_, const String ttl_update_key_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) - : ITTLAlgorithm(description_, old_ttl_info_, current_time_, force_) + : ITTLAlgorithm(ttl_expressions_, description_, old_ttl_info_, current_time_, force_) , ttl_update_field(ttl_update_field_) , ttl_update_key(ttl_update_key_) { @@ -21,7 +22,7 @@ void TTLUpdateInfoAlgorithm::execute(Block & block) if (!block) return; - auto ttl_column = executeExpressionAndGetColumn(description.expression, block, description.result_column); + auto ttl_column = executeExpressionAndGetColumn(ttl_expressions.expression, block, description.result_column); for (size_t i = 0; i < block.rows(); ++i) { UInt32 cur_ttl = ITTLAlgorithm::getTimestampByIndex(ttl_column.get(), i); diff --git a/src/Processors/TTL/TTLUpdateInfoAlgorithm.h b/src/Processors/TTL/TTLUpdateInfoAlgorithm.h index 45eecbde3d0..e9bcfcdec88 100644 --- a/src/Processors/TTL/TTLUpdateInfoAlgorithm.h +++ b/src/Processors/TTL/TTLUpdateInfoAlgorithm.h @@ -20,6 +20,7 @@ class TTLUpdateInfoAlgorithm : public ITTLAlgorithm { public: TTLUpdateInfoAlgorithm( + const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLUpdateField ttl_update_field_, const String ttl_update_key_, diff --git a/src/Processors/Transforms/TTLTransform.cpp b/src/Processors/Transforms/TTLTransform.cpp index 7cde86098c7..d3d45f68d46 100644 --- a/src/Processors/Transforms/TTLTransform.cpp +++ b/src/Processors/Transforms/TTLTransform.cpp @@ -36,7 +36,7 @@ TTLTransform::TTLTransform( rows_ttl, old_ttl_infos.table_ttl, current_time_, force_); /// Skip all data if table ttl is expired for part - if (algorithm->isMaxTTLExpired() && !rows_ttl.where_expression) + if (algorithm->isMaxTTLExpired() && !rows_ttl.where_expression_ast) all_data_dropped = true; delete_algorithm = algorithm.get(); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 2a381afa805..d080240b066 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -125,13 +125,18 @@ void buildScatterSelector( /// Computes ttls and updates ttl infos void updateTTL( + const ContextPtr context, const TTLDescription & ttl_entry, IMergeTreeDataPart::TTLInfos & ttl_infos, DB::MergeTreeDataPartTTLInfo & ttl_info, const Block & block, bool update_part_min_max_ttls) { - auto ttl_column = ITTLAlgorithm::executeExpressionAndGetColumn(ttl_entry.expression, block, ttl_entry.result_column); + auto expr_and_set = ttl_entry.buildExpression(); + for (auto & subquery : expr_and_set.sets->getSubqueries()) + subquery->buildSetInplace(context); + + auto ttl_column = ITTLAlgorithm::executeExpressionAndGetColumn(expr_and_set.expression, block, ttl_entry.result_column); if (const ColumnUInt16 * column_date = typeid_cast(ttl_column.get())) { @@ -488,7 +493,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( DB::IMergeTreeDataPart::TTLInfos move_ttl_infos; const auto & move_ttl_entries = metadata_snapshot->getMoveTTLs(); for (const auto & ttl_entry : move_ttl_entries) - updateTTL(ttl_entry, move_ttl_infos, move_ttl_infos.moves_ttl[ttl_entry.result_column], block, false); + updateTTL(context, ttl_entry, move_ttl_infos, move_ttl_infos.moves_ttl[ttl_entry.result_column], block, false); ReservationPtr reservation = data.reserveSpacePreferringTTLRules(metadata_snapshot, expected_size, move_ttl_infos, time(nullptr), 0, true); VolumePtr volume = data.getStoragePolicy()->getVolume(0); @@ -543,20 +548,20 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( } if (metadata_snapshot->hasRowsTTL()) - updateTTL(metadata_snapshot->getRowsTTL(), new_data_part->ttl_infos, new_data_part->ttl_infos.table_ttl, block, true); + updateTTL(context, metadata_snapshot->getRowsTTL(), new_data_part->ttl_infos, new_data_part->ttl_infos.table_ttl, block, true); for (const auto & ttl_entry : metadata_snapshot->getGroupByTTLs()) - updateTTL(ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.group_by_ttl[ttl_entry.result_column], block, true); + updateTTL(context, ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.group_by_ttl[ttl_entry.result_column], block, true); for (const auto & ttl_entry : metadata_snapshot->getRowsWhereTTLs()) - updateTTL(ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.rows_where_ttl[ttl_entry.result_column], block, true); + updateTTL(context, ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.rows_where_ttl[ttl_entry.result_column], block, true); for (const auto & [name, ttl_entry] : metadata_snapshot->getColumnTTLs()) - updateTTL(ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.columns_ttl[name], block, true); + updateTTL(context, ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.columns_ttl[name], block, true); const auto & recompression_ttl_entries = metadata_snapshot->getRecompressionTTLs(); for (const auto & ttl_entry : recompression_ttl_entries) - updateTTL(ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.recompression_ttl[ttl_entry.result_column], block, false); + updateTTL(context, ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.recompression_ttl[ttl_entry.result_column], block, false); new_data_part->ttl_infos.update(move_ttl_infos); diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index af285a953dc..7db5af82e0b 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -193,7 +193,7 @@ TTLDescription StorageInMemoryMetadata::getRowsTTL() const bool StorageInMemoryMetadata::hasRowsTTL() const { - return table_ttl.rows_ttl.expression != nullptr; + return table_ttl.rows_ttl.expression_ast != nullptr; } TTLDescriptions StorageInMemoryMetadata::getRowsWhereTTLs() const @@ -251,9 +251,8 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies( NameSet required_ttl_columns; NameSet updated_ttl_columns; - auto add_dependent_columns = [&updated_columns](const auto & expression, auto & to_set) + auto add_dependent_columns = [&updated_columns](const Names & required_columns, auto & to_set) { - auto required_columns = expression->getRequiredColumns(); for (const auto & dependency : required_columns) { if (updated_columns.contains(dependency)) @@ -269,13 +268,13 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies( for (const auto & index : getSecondaryIndices()) { if (has_dependency(index.name, ColumnDependency::SKIP_INDEX)) - add_dependent_columns(index.expression, indices_columns); + add_dependent_columns(index.expression->getRequiredColumns(), indices_columns); } for (const auto & projection : getProjections()) { if (has_dependency(projection.name, ColumnDependency::PROJECTION)) - add_dependent_columns(&projection, projections_columns); + add_dependent_columns(projection.getRequiredColumns(), projections_columns); } auto add_for_rows_ttl = [&](const auto & expression, auto & to_set) @@ -289,25 +288,25 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies( }; if (hasRowsTTL()) - add_for_rows_ttl(getRowsTTL().expression, required_ttl_columns); + add_for_rows_ttl(getRowsTTL().expression_columns, required_ttl_columns); for (const auto & entry : getRowsWhereTTLs()) - add_for_rows_ttl(entry.expression, required_ttl_columns); + add_for_rows_ttl(entry.expression_columns, required_ttl_columns); for (const auto & entry : getGroupByTTLs()) - add_for_rows_ttl(entry.expression, required_ttl_columns); + add_for_rows_ttl(entry.expression_columns, required_ttl_columns); for (const auto & entry : getRecompressionTTLs()) - add_dependent_columns(entry.expression, required_ttl_columns); + add_dependent_columns(entry.expression_columns, required_ttl_columns); for (const auto & [name, entry] : getColumnTTLs()) { - if (add_dependent_columns(entry.expression, required_ttl_columns) && include_ttl_target) + if (add_dependent_columns(entry.expression_columns, required_ttl_columns) && include_ttl_target) updated_ttl_columns.insert(name); } for (const auto & entry : getMoveTTLs()) - add_dependent_columns(entry.expression, required_ttl_columns); + add_dependent_columns(entry.expression_columns, required_ttl_columns); //TODO what about rows_where_ttl and group_by_ttl ?? diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index f601fed06ac..47138f30e4f 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -113,11 +113,11 @@ TTLDescription::TTLDescription(const TTLDescription & other) , if_exists(other.if_exists) , recompression_codec(other.recompression_codec) { - if (other.expression) - expression = other.expression->clone(); + // if (other.expression) + // expression = other.expression->clone(); - if (other.where_expression) - where_expression = other.where_expression->clone(); + // if (other.where_expression) + // where_expression = other.where_expression->clone(); } TTLDescription & TTLDescription::operator=(const TTLDescription & other) @@ -131,16 +131,16 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) else expression_ast.reset(); - if (other.expression) - expression = other.expression->clone(); - else - expression.reset(); + // if (other.expression) + // expression = other.expression->clone(); + // else + // expression.reset(); result_column = other.result_column; - if (other.where_expression) - where_expression = other.where_expression->clone(); - else - where_expression.reset(); + // if (other.where_expression) + // where_expression = other.where_expression->clone(); + // else + // where_expression.reset(); where_result_column = other.where_result_column; group_by_keys = other.group_by_keys; @@ -158,6 +158,17 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) return * this; } +static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndTypesList & columns, const ContextPtr & context) +{ + ExpressionAndSets result; + auto syntax_analyzer_result = TreeRewriter(context).analyze(ast, columns); + ExpressionAnalyzer analyzer(ast, syntax_analyzer_result, context); + result.expression = analyzer.getActions(false); + result.sets = analyzer.getPreparedSets(); + + return result; +} + TTLDescription TTLDescription::getTTLFromAST( const ASTPtr & definition_ast, const ColumnsDescription & columns, @@ -174,10 +185,15 @@ TTLDescription TTLDescription::getTTLFromAST( result.expression_ast = definition_ast->clone(); auto ttl_ast = result.expression_ast->clone(); - auto syntax_analyzer_result = TreeRewriter(context).analyze(ttl_ast, columns.getAllPhysical()); - result.expression = ExpressionAnalyzer(ttl_ast, syntax_analyzer_result, context).getActions(false); + auto expression = buildExpressionAndSets(ttl_ast, columns.getAllPhysical(), context).expression; + result.expression_columns = expression->getRequiredColumns(); + + // auto syntax_analyzer_result = TreeRewriter(context).analyze(ttl_ast, columns.getAllPhysical()); + // result.expression = ExpressionAnalyzer(ttl_ast, syntax_analyzer_result, context).getActions(false); result.result_column = ttl_ast->getColumnName(); + ExpressionActionsPtr where_expression; + if (ttl_element == nullptr) /// columns TTL { result.destination_type = DataDestinationType::DELETE; @@ -194,8 +210,10 @@ TTLDescription TTLDescription::getTTLFromAST( { if (ASTPtr where_expr_ast = ttl_element->where()) { - auto where_syntax_result = TreeRewriter(context).analyze(where_expr_ast, columns.getAllPhysical()); - result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, context).getActions(false); + result.where_expression_ast = where_expr_ast->clone(); + where_expression = buildExpressionAndSets(where_expr_ast, columns.getAllPhysical(), context).expression; + // auto where_syntax_result = TreeRewriter(context).analyze(where_expr_ast, columns.getAllPhysical()); + // result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, context).getActions(false); result.where_result_column = where_expr_ast->getColumnName(); } } @@ -221,17 +239,17 @@ TTLDescription TTLDescription::getTTLFromAST( for (const auto & ast : ttl_element->group_by_assignments) { const auto assignment = ast->as(); - auto expression = assignment.expression(); + auto ass_expression = assignment.expression(); FindAggregateFunctionVisitor::Data data{false}; - FindAggregateFunctionVisitor(data).visit(expression); + FindAggregateFunctionVisitor(data).visit(ass_expression); if (!data.has_aggregate_function) throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "Invalid expression for assignment of column {}. Should contain an aggregate function", assignment.column_name); - expression = addTypeConversionToAST(std::move(expression), columns.getPhysical(assignment.column_name).type->getName()); - aggregations.emplace_back(assignment.column_name, std::move(expression)); + ass_expression = addTypeConversionToAST(std::move(ass_expression), columns.getPhysical(assignment.column_name).type->getName()); + aggregations.emplace_back(assignment.column_name, std::move(ass_expression)); aggregation_columns_set.insert(assignment.column_name); } @@ -289,7 +307,7 @@ TTLDescription TTLDescription::getTTLFromAST( } } - checkTTLExpression(result.expression, result.result_column); + checkTTLExpression(expression, result.result_column); return result; } @@ -341,7 +359,7 @@ TTLTableDescription TTLTableDescription::getTTLForTableFromAST( auto ttl = TTLDescription::getTTLFromAST(ttl_element_ptr, columns, context, primary_key); if (ttl.mode == TTLMode::DELETE) { - if (!ttl.where_expression) + if (!ttl.where_expression_ast) { if (have_unconditional_delete_ttl) throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "More than one DELETE TTL expression without WHERE expression is not allowed"); diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h index 8f60eb604b5..5ea243424cb 100644 --- a/src/Storages/TTLDescription.h +++ b/src/Storages/TTLDescription.h @@ -33,6 +33,15 @@ struct TTLAggregateDescription using TTLAggregateDescriptions = std::vector; +class PreparedSets; +using PreparedSetsPtr = std::shared_ptr; + +struct ExpressionAndSets +{ + ExpressionActionsPtr expression; + PreparedSetsPtr sets; +}; + /// Common struct for TTL record in storage struct TTLDescription { @@ -42,9 +51,10 @@ struct TTLDescription /// TTL d + INTERVAL 1 DAY /// ^~~~~~~~~~~~~~~~~~~^ ASTPtr expression_ast; + Names expression_columns; /// Expression actions evaluated from AST - ExpressionActionsPtr expression; + ExpressionAndSets buildExpression() const; /// Result column of this TTL expression String result_column; @@ -52,7 +62,8 @@ struct TTLDescription /// WHERE part in TTL expression /// TTL ... WHERE x % 10 == 0 and y > 5 /// ^~~~~~~~~~~~~~~~~~~~~~^ - ExpressionActionsPtr where_expression; + ASTPtr where_expression_ast; + ExpressionAndSets buildWhereExpression() const; /// Name of result column from WHERE expression String where_result_column; From 7ab4af06df0d78e6728e3cc5c727e5c9e4cc33ef Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 4 Dec 2023 18:04:42 +0000 Subject: [PATCH 006/103] Attempt to support subqueries in TTL. (2) --- src/Processors/QueryPlan/CreatingSetsStep.cpp | 29 +++++++++++ src/Processors/QueryPlan/CreatingSetsStep.h | 2 + src/Processors/TTL/ITTLAlgorithm.cpp | 2 +- src/Processors/TTL/ITTLAlgorithm.h | 6 +-- .../TTL/TTLAggregationAlgorithm.cpp | 2 +- src/Processors/TTL/TTLAggregationAlgorithm.h | 2 +- src/Processors/TTL/TTLColumnAlgorithm.cpp | 2 +- src/Processors/TTL/TTLColumnAlgorithm.h | 2 +- src/Processors/TTL/TTLDeleteAlgorithm.cpp | 2 +- src/Processors/TTL/TTLDeleteAlgorithm.h | 2 +- src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp | 2 +- src/Processors/TTL/TTLUpdateInfoAlgorithm.h | 2 +- .../Transforms/TTLCalcTransform.cpp | 33 ++++++++++--- src/Processors/Transforms/TTLCalcTransform.h | 4 ++ src/Processors/Transforms/TTLTransform.cpp | 33 ++++++++++--- src/Processors/Transforms/TTLTransform.h | 5 ++ src/Storages/MergeTree/MergeTask.cpp | 36 +++++++++----- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 49 ++++++++++++++----- src/Storages/StorageInMemoryMetadata.cpp | 8 +-- src/Storages/TTLDescription.cpp | 21 +++++++- src/Storages/TTLDescription.h | 7 +-- 22 files changed, 197 insertions(+), 56 deletions(-) diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 3e4dfb0c7d1..11415e8d815 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -157,6 +157,35 @@ void addCreatingSetsStep(QueryPlan & query_plan, PreparedSets::Subqueries subque query_plan.unitePlans(std::move(creating_sets), std::move(plans)); } +QueryPipelineBuilderPtr addCreatingSetsTransform(QueryPipelineBuilderPtr pipeline, PreparedSets::Subqueries subqueries, ContextPtr context) +{ + DataStreams input_streams; + input_streams.emplace_back(DataStream{pipeline->getHeader()}); + + QueryPipelineBuilders pipelines; + pipelines.reserve(1 + subqueries.size()); + pipelines.push_back(std::move(pipeline)); + + auto plan_settings = QueryPlanOptimizationSettings::fromContext(context); + auto pipeline_settings = BuildQueryPipelineSettings::fromContext(context); + + for (auto & future_set : subqueries) + { + if (future_set->get()) + continue; + + auto plan = future_set->build(context); + if (!plan) + continue; + + input_streams.emplace_back(plan->getCurrentDataStream()); + pipelines.emplace_back(plan->buildQueryPipeline(plan_settings, pipeline_settings)); + } + + CreatingSetsStep(input_streams).updatePipeline(std::move(pipelines), pipeline_settings); + return std::move(pipelines.front()); +} + std::vector> DelayedCreatingSetsStep::makePlansForSets(DelayedCreatingSetsStep && step) { std::vector> plans; diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index a90b70a2fa4..292ec19914c 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -72,4 +72,6 @@ void addCreatingSetsStep(QueryPlan & query_plan, PreparedSets::Subqueries subque void addCreatingSetsStep(QueryPlan & query_plan, PreparedSetsPtr prepared_sets, ContextPtr context); +QueryPipelineBuilderPtr addCreatingSetsTransform(QueryPipelineBuilderPtr pipeline, PreparedSets::Subqueries subqueries, ContextPtr context); + } diff --git a/src/Processors/TTL/ITTLAlgorithm.cpp b/src/Processors/TTL/ITTLAlgorithm.cpp index af6c4e4ac35..761f43e2422 100644 --- a/src/Processors/TTL/ITTLAlgorithm.cpp +++ b/src/Processors/TTL/ITTLAlgorithm.cpp @@ -11,7 +11,7 @@ namespace ErrorCodes } ITTLAlgorithm::ITTLAlgorithm( - const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) + const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) : ttl_expressions(ttl_expressions_) , description(description_) , old_ttl_info(old_ttl_info_) diff --git a/src/Processors/TTL/ITTLAlgorithm.h b/src/Processors/TTL/ITTLAlgorithm.h index 6e73286b564..d79aa8a8dfc 100644 --- a/src/Processors/TTL/ITTLAlgorithm.h +++ b/src/Processors/TTL/ITTLAlgorithm.h @@ -8,7 +8,7 @@ namespace DB { -struct TTlExpressions +struct TTLExpressions { ExpressionActionsPtr expression; ExpressionActionsPtr where_expression; @@ -24,7 +24,7 @@ public: using TTLInfo = IMergeTreeDataPart::TTLInfo; using MutableDataPartPtr = MergeTreeMutableDataPartPtr; - ITTLAlgorithm(const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_); + ITTLAlgorithm(const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_); virtual ~ITTLAlgorithm() = default; virtual void execute(Block & block) = 0; @@ -45,7 +45,7 @@ protected: bool isTTLExpired(time_t ttl) const; UInt32 getTimestampByIndex(const IColumn * column, size_t index) const; - const TTlExpressions ttl_expressions; + const TTLExpressions ttl_expressions; const TTLDescription description; const TTLInfo old_ttl_info; const time_t current_time; diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index ab2ba5f58fc..0c6184a56e5 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -5,7 +5,7 @@ namespace DB { TTLAggregationAlgorithm::TTLAggregationAlgorithm( - const TTlExpressions & ttl_expressions_, + const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.h b/src/Processors/TTL/TTLAggregationAlgorithm.h index 9fd074efba8..f7bf19a202b 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.h +++ b/src/Processors/TTL/TTLAggregationAlgorithm.h @@ -13,7 +13,7 @@ class TTLAggregationAlgorithm final : public ITTLAlgorithm { public: TTLAggregationAlgorithm( - const TTlExpressions & ttl_expressions_, + const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, diff --git a/src/Processors/TTL/TTLColumnAlgorithm.cpp b/src/Processors/TTL/TTLColumnAlgorithm.cpp index cb99dcf99b1..e27050564ce 100644 --- a/src/Processors/TTL/TTLColumnAlgorithm.cpp +++ b/src/Processors/TTL/TTLColumnAlgorithm.cpp @@ -4,7 +4,7 @@ namespace DB { TTLColumnAlgorithm::TTLColumnAlgorithm( - const TTlExpressions & ttl_expressions_, + const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, diff --git a/src/Processors/TTL/TTLColumnAlgorithm.h b/src/Processors/TTL/TTLColumnAlgorithm.h index efcd7c74454..f34dae952d1 100644 --- a/src/Processors/TTL/TTLColumnAlgorithm.h +++ b/src/Processors/TTL/TTLColumnAlgorithm.h @@ -11,7 +11,7 @@ class TTLColumnAlgorithm final : public ITTLAlgorithm { public: TTLColumnAlgorithm( - const TTlExpressions & ttl_expressions_, + const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, diff --git a/src/Processors/TTL/TTLDeleteAlgorithm.cpp b/src/Processors/TTL/TTLDeleteAlgorithm.cpp index 6a172e9c3c3..6f9bc315276 100644 --- a/src/Processors/TTL/TTLDeleteAlgorithm.cpp +++ b/src/Processors/TTL/TTLDeleteAlgorithm.cpp @@ -4,7 +4,7 @@ namespace DB { TTLDeleteAlgorithm::TTLDeleteAlgorithm( - const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) + const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_) : ITTLAlgorithm(ttl_expressions_, description_, old_ttl_info_, current_time_, force_) { if (!isMinTTLExpired()) diff --git a/src/Processors/TTL/TTLDeleteAlgorithm.h b/src/Processors/TTL/TTLDeleteAlgorithm.h index 23389070774..622e45acecb 100644 --- a/src/Processors/TTL/TTLDeleteAlgorithm.h +++ b/src/Processors/TTL/TTLDeleteAlgorithm.h @@ -10,7 +10,7 @@ namespace DB class TTLDeleteAlgorithm final : public ITTLAlgorithm { public: - TTLDeleteAlgorithm(const TTlExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_); + TTLDeleteAlgorithm(const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_); void execute(Block & block) override; void finalize(const MutableDataPartPtr & data_part) const override; diff --git a/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp b/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp index 34c0cad70ea..b7cddf3c165 100644 --- a/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp +++ b/src/Processors/TTL/TTLUpdateInfoAlgorithm.cpp @@ -4,7 +4,7 @@ namespace DB { TTLUpdateInfoAlgorithm::TTLUpdateInfoAlgorithm( - const TTlExpressions & ttl_expressions_, + const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLUpdateField ttl_update_field_, const String ttl_update_key_, diff --git a/src/Processors/TTL/TTLUpdateInfoAlgorithm.h b/src/Processors/TTL/TTLUpdateInfoAlgorithm.h index e9bcfcdec88..0cf31765aef 100644 --- a/src/Processors/TTL/TTLUpdateInfoAlgorithm.h +++ b/src/Processors/TTL/TTLUpdateInfoAlgorithm.h @@ -20,7 +20,7 @@ class TTLUpdateInfoAlgorithm : public ITTLAlgorithm { public: TTLUpdateInfoAlgorithm( - const TTlExpressions & ttl_expressions_, + const TTLExpressions & ttl_expressions_, const TTLDescription & description_, const TTLUpdateField ttl_update_field_, const String ttl_update_key_, diff --git a/src/Processors/Transforms/TTLCalcTransform.cpp b/src/Processors/Transforms/TTLCalcTransform.cpp index 31fb61239ef..204dfe21733 100644 --- a/src/Processors/Transforms/TTLCalcTransform.cpp +++ b/src/Processors/Transforms/TTLCalcTransform.cpp @@ -4,7 +4,22 @@ namespace DB { +static TTLExpressions getExpressions(const TTLDescription & ttl_descr, PreparedSets::Subqueries & subqueries_for_sets, const ContextPtr & context) +{ + auto expr = ttl_descr.buildExpression(context); + auto where_expr = ttl_descr.buildWhereExpression(context); + + auto expr_queries = expr.sets->getSubqueries(); + auto where_expr_queries = expr.sets->getSubqueries(); + + subqueries_for_sets.insert(subqueries_for_sets.end(), expr_queries.begin(), expr_queries.end()); + subqueries_for_sets.insert(subqueries_for_sets.end(), where_expr_queries.begin(), where_expr_queries.end()); + + return {expr.expression, where_expr.expression}; +} + TTLCalcTransform::TTLCalcTransform( + const ContextPtr & context, const Block & header_, const MergeTreeData & storage_, const StorageMetadataPtr & metadata_snapshot_, @@ -21,33 +36,39 @@ TTLCalcTransform::TTLCalcTransform( { const auto & rows_ttl = metadata_snapshot_->getRowsTTL(); algorithms.emplace_back(std::make_unique( - rows_ttl, TTLUpdateField::TABLE_TTL, rows_ttl.result_column, old_ttl_infos.table_ttl, current_time_, force_)); + getExpressions(rows_ttl, subqueries_for_sets, context), rows_ttl, + TTLUpdateField::TABLE_TTL, rows_ttl.result_column, old_ttl_infos.table_ttl, current_time_, force_)); } for (const auto & where_ttl : metadata_snapshot_->getRowsWhereTTLs()) algorithms.emplace_back(std::make_unique( - where_ttl, TTLUpdateField::ROWS_WHERE_TTL, where_ttl.result_column, old_ttl_infos.rows_where_ttl[where_ttl.result_column], current_time_, force_)); + getExpressions(where_ttl, subqueries_for_sets, context), where_ttl, + TTLUpdateField::ROWS_WHERE_TTL, where_ttl.result_column, old_ttl_infos.rows_where_ttl[where_ttl.result_column], current_time_, force_)); for (const auto & group_by_ttl : metadata_snapshot_->getGroupByTTLs()) algorithms.emplace_back(std::make_unique( - group_by_ttl, TTLUpdateField::GROUP_BY_TTL, group_by_ttl.result_column, old_ttl_infos.group_by_ttl[group_by_ttl.result_column], current_time_, force_)); + getExpressions(group_by_ttl, subqueries_for_sets, context), group_by_ttl, + TTLUpdateField::GROUP_BY_TTL, group_by_ttl.result_column, old_ttl_infos.group_by_ttl[group_by_ttl.result_column], current_time_, force_)); if (metadata_snapshot_->hasAnyColumnTTL()) { for (const auto & [name, description] : metadata_snapshot_->getColumnTTLs()) { algorithms.emplace_back(std::make_unique( - description, TTLUpdateField::COLUMNS_TTL, name, old_ttl_infos.columns_ttl[name], current_time_, force_)); + getExpressions(description, subqueries_for_sets, context), description, + TTLUpdateField::COLUMNS_TTL, name, old_ttl_infos.columns_ttl[name], current_time_, force_)); } } for (const auto & move_ttl : metadata_snapshot_->getMoveTTLs()) algorithms.emplace_back(std::make_unique( - move_ttl, TTLUpdateField::MOVES_TTL, move_ttl.result_column, old_ttl_infos.moves_ttl[move_ttl.result_column], current_time_, force_)); + getExpressions(move_ttl, subqueries_for_sets, context), move_ttl, + TTLUpdateField::MOVES_TTL, move_ttl.result_column, old_ttl_infos.moves_ttl[move_ttl.result_column], current_time_, force_)); for (const auto & recompression_ttl : metadata_snapshot_->getRecompressionTTLs()) algorithms.emplace_back(std::make_unique( - recompression_ttl, TTLUpdateField::RECOMPRESSION_TTL, recompression_ttl.result_column, old_ttl_infos.recompression_ttl[recompression_ttl.result_column], current_time_, force_)); + getExpressions(recompression_ttl, subqueries_for_sets, context), recompression_ttl, + TTLUpdateField::RECOMPRESSION_TTL, recompression_ttl.result_column, old_ttl_infos.recompression_ttl[recompression_ttl.result_column], current_time_, force_)); } void TTLCalcTransform::consume(Chunk chunk) diff --git a/src/Processors/Transforms/TTLCalcTransform.h b/src/Processors/Transforms/TTLCalcTransform.h index 495879400dc..960438f5f2b 100644 --- a/src/Processors/Transforms/TTLCalcTransform.h +++ b/src/Processors/Transforms/TTLCalcTransform.h @@ -15,6 +15,7 @@ class TTLCalcTransform : public IAccumulatingTransform { public: TTLCalcTransform( + const ContextPtr & context, const Block & header_, const MergeTreeData & storage_, const StorageMetadataPtr & metadata_snapshot_, @@ -23,6 +24,8 @@ public: bool force_ ); + PreparedSets::Subqueries getSubqueries() { return std::move(subqueries_for_sets); } + String getName() const override { return "TTL_CALC"; } Status prepare() override; @@ -35,6 +38,7 @@ protected: private: std::vector algorithms; + PreparedSets::Subqueries subqueries_for_sets; /// ttl_infos and empty_columns are updating while reading const MergeTreeData::MutableDataPartPtr & data_part; diff --git a/src/Processors/Transforms/TTLTransform.cpp b/src/Processors/Transforms/TTLTransform.cpp index d3d45f68d46..69e2e6e5fc0 100644 --- a/src/Processors/Transforms/TTLTransform.cpp +++ b/src/Processors/Transforms/TTLTransform.cpp @@ -16,7 +16,22 @@ namespace DB { +static TTLExpressions getExpressions(const TTLDescription & ttl_descr, PreparedSets::Subqueries & subqueries_for_sets, const ContextPtr & context) +{ + auto expr = ttl_descr.buildExpression(context); + auto where_expr = ttl_descr.buildWhereExpression(context); + + auto expr_queries = expr.sets->getSubqueries(); + auto where_expr_queries = expr.sets->getSubqueries(); + + subqueries_for_sets.insert(subqueries_for_sets.end(), expr_queries.begin(), expr_queries.end()); + subqueries_for_sets.insert(subqueries_for_sets.end(), where_expr_queries.begin(), where_expr_queries.end()); + + return {expr.expression, where_expr.expression}; +} + TTLTransform::TTLTransform( + const ContextPtr & context, const Block & header_, const MergeTreeData & storage_, const StorageMetadataPtr & metadata_snapshot_, @@ -33,7 +48,8 @@ TTLTransform::TTLTransform( { const auto & rows_ttl = metadata_snapshot_->getRowsTTL(); auto algorithm = std::make_unique( - rows_ttl, old_ttl_infos.table_ttl, current_time_, force_); + getExpressions(rows_ttl, subqueries_for_sets, context), rows_ttl, + old_ttl_infos.table_ttl, current_time_, force_); /// Skip all data if table ttl is expired for part if (algorithm->isMaxTTLExpired() && !rows_ttl.where_expression_ast) @@ -45,11 +61,13 @@ TTLTransform::TTLTransform( for (const auto & where_ttl : metadata_snapshot_->getRowsWhereTTLs()) algorithms.emplace_back(std::make_unique( - where_ttl, old_ttl_infos.rows_where_ttl[where_ttl.result_column], current_time_, force_)); + getExpressions(where_ttl, subqueries_for_sets, context), where_ttl, + old_ttl_infos.rows_where_ttl[where_ttl.result_column], current_time_, force_)); for (const auto & group_by_ttl : metadata_snapshot_->getGroupByTTLs()) algorithms.emplace_back(std::make_unique( - group_by_ttl, old_ttl_infos.group_by_ttl[group_by_ttl.result_column], current_time_, force_, + getExpressions(group_by_ttl, subqueries_for_sets, context), group_by_ttl, + old_ttl_infos.group_by_ttl[group_by_ttl.result_column], current_time_, force_, getInputPort().getHeader(), storage_)); if (metadata_snapshot_->hasAnyColumnTTL()) @@ -75,18 +93,21 @@ TTLTransform::TTLTransform( } algorithms.emplace_back(std::make_unique( - description, old_ttl_infos.columns_ttl[name], current_time_, + getExpressions(description, subqueries_for_sets, context), description, + old_ttl_infos.columns_ttl[name], current_time_, force_, name, default_expression, default_column_name, isCompactPart(data_part))); } } for (const auto & move_ttl : metadata_snapshot_->getMoveTTLs()) algorithms.emplace_back(std::make_unique( - move_ttl, TTLUpdateField::MOVES_TTL, move_ttl.result_column, old_ttl_infos.moves_ttl[move_ttl.result_column], current_time_, force_)); + getExpressions(move_ttl, subqueries_for_sets, context), move_ttl, + TTLUpdateField::MOVES_TTL, move_ttl.result_column, old_ttl_infos.moves_ttl[move_ttl.result_column], current_time_, force_)); for (const auto & recompression_ttl : metadata_snapshot_->getRecompressionTTLs()) algorithms.emplace_back(std::make_unique( - recompression_ttl, TTLUpdateField::RECOMPRESSION_TTL, recompression_ttl.result_column, old_ttl_infos.recompression_ttl[recompression_ttl.result_column], current_time_, force_)); + getExpressions(recompression_ttl, subqueries_for_sets, context), recompression_ttl, + TTLUpdateField::RECOMPRESSION_TTL, recompression_ttl.result_column, old_ttl_infos.recompression_ttl[recompression_ttl.result_column], current_time_, force_)); } Block reorderColumns(Block block, const Block & header) diff --git a/src/Processors/Transforms/TTLTransform.h b/src/Processors/Transforms/TTLTransform.h index 3f0dffd1998..47da456a2e3 100644 --- a/src/Processors/Transforms/TTLTransform.h +++ b/src/Processors/Transforms/TTLTransform.h @@ -16,6 +16,7 @@ class TTLTransform : public IAccumulatingTransform { public: TTLTransform( + const ContextPtr & context, const Block & header_, const MergeTreeData & storage_, const StorageMetadataPtr & metadata_snapshot_, @@ -28,6 +29,8 @@ public: Status prepare() override; + PreparedSets::Subqueries getSubqueries() { return std::move(subqueries_for_sets); } + protected: void consume(Chunk chunk) override; Chunk generate() override; @@ -40,6 +43,8 @@ private: const TTLDeleteAlgorithm * delete_algorithm = nullptr; bool all_data_dropped = false; + PreparedSets::Subqueries subqueries_for_sets; + /// ttl_infos and empty_columns are updating while reading const MergeTreeData::MutableDataPartPtr & data_part; Poco::Logger * log; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index e8e307bb148..26b290d33d5 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -31,6 +31,9 @@ #include #include #include +#include +#include +#include namespace DB { @@ -1004,8 +1007,9 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() break; } - auto res_pipe = Pipe::unitePipes(std::move(pipes)); - res_pipe.addTransform(std::move(merged_transform)); + auto builder = std::make_unique(); + builder->init(Pipe::unitePipes(std::move(pipes))); + builder->addTransform(std::move(merged_transform)); if (global_ctx->deduplicate) { @@ -1021,26 +1025,34 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() } if (DistinctSortedTransform::isApplicable(header, sort_description, global_ctx->deduplicate_by_columns)) - res_pipe.addTransform(std::make_shared( - res_pipe.getHeader(), sort_description, SizeLimits(), 0 /*limit_hint*/, global_ctx->deduplicate_by_columns)); + builder->addTransform(std::make_shared( + builder->getHeader(), sort_description, SizeLimits(), 0 /*limit_hint*/, global_ctx->deduplicate_by_columns)); else - res_pipe.addTransform(std::make_shared( - res_pipe.getHeader(), SizeLimits(), 0 /*limit_hint*/, global_ctx->deduplicate_by_columns)); + builder->addTransform(std::make_shared( + builder->getHeader(), SizeLimits(), 0 /*limit_hint*/, global_ctx->deduplicate_by_columns)); } + PreparedSets::Subqueries subqueries; + if (ctx->need_remove_expired_values) - res_pipe.addTransform(std::make_shared( - res_pipe.getHeader(), *global_ctx->data, global_ctx->metadata_snapshot, global_ctx->new_data_part, global_ctx->time_of_merge, ctx->force_ttl)); + { + auto transform = std::make_shared(global_ctx->context, builder->getHeader(), *global_ctx->data, global_ctx->metadata_snapshot, global_ctx->new_data_part, global_ctx->time_of_merge, ctx->force_ttl); + subqueries = transform->getSubqueries(); + builder->addTransform(std::move(transform)); + } if (global_ctx->metadata_snapshot->hasSecondaryIndices()) { const auto & indices = global_ctx->metadata_snapshot->getSecondaryIndices(); - res_pipe.addTransform(std::make_shared( - res_pipe.getHeader(), indices.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext()))); - res_pipe.addTransform(std::make_shared(res_pipe.getHeader())); + builder->addTransform(std::make_shared( + builder->getHeader(), indices.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext()))); + builder->addTransform(std::make_shared(builder->getHeader())); } - global_ctx->merged_pipeline = QueryPipeline(std::move(res_pipe)); + if (!subqueries.empty()) + builder = addCreatingSetsTransform(std::move(builder), std::move(subqueries), global_ctx->context); + + global_ctx->merged_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); /// Dereference unique_ptr and pass horizontal_stage_progress by reference global_ctx->merged_pipeline.setProgressCallback(MergeProgressCallback(global_ctx->merge_list_element_ptr, global_ctx->watch_prev_elapsed, *global_ctx->horizontal_stage_progress)); /// Is calculated inside MergeProgressCallback. diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index d080240b066..ce9e5762cb4 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -132,7 +132,7 @@ void updateTTL( const Block & block, bool update_part_min_max_ttls) { - auto expr_and_set = ttl_entry.buildExpression(); + auto expr_and_set = ttl_entry.buildExpression(context); for (auto & subquery : expr_and_set.sets->getSubqueries()) subquery->buildSetInplace(context); diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 6b6b5947581..61849f94e44 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1507,21 +1509,34 @@ private: if (!ctx->mutating_pipeline_builder.initialized()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot mutate part columns with uninitialized mutations stream. It's a bug"); - QueryPipelineBuilder builder(std::move(ctx->mutating_pipeline_builder)); + auto builder = std::make_unique(std::move(ctx->mutating_pipeline_builder)); if (ctx->metadata_snapshot->hasPrimaryKey() || ctx->metadata_snapshot->hasSecondaryIndices()) { - builder.addTransform(std::make_shared( - builder.getHeader(), ctx->data->getPrimaryKeyAndSkipIndicesExpression(ctx->metadata_snapshot, skip_indices))); + builder->addTransform(std::make_shared( + builder->getHeader(), ctx->data->getPrimaryKeyAndSkipIndicesExpression(ctx->metadata_snapshot, skip_indices))); - builder.addTransform(std::make_shared(builder.getHeader())); + builder->addTransform(std::make_shared(builder->getHeader())); } + PreparedSets::Subqueries subqueries; + if (ctx->execute_ttl_type == ExecuteTTLType::NORMAL) - builder.addTransform(std::make_shared(builder.getHeader(), *ctx->data, ctx->metadata_snapshot, ctx->new_data_part, ctx->time_of_mutation, true)); + { + auto transform = std::make_shared(ctx->context, builder->getHeader(), *ctx->data, ctx->metadata_snapshot, ctx->new_data_part, ctx->time_of_mutation, true); + subqueries = transform->getSubqueries(); + builder->addTransform(std::move(transform)); + } if (ctx->execute_ttl_type == ExecuteTTLType::RECALCULATE) - builder.addTransform(std::make_shared(builder.getHeader(), *ctx->data, ctx->metadata_snapshot, ctx->new_data_part, ctx->time_of_mutation, true)); + { + auto transform = std::make_shared(ctx->context, builder->getHeader(), *ctx->data, ctx->metadata_snapshot, ctx->new_data_part, ctx->time_of_mutation, true); + subqueries = transform->getSubqueries(); + builder->addTransform(std::move(transform)); + } + + if (!subqueries.empty()) + builder = addCreatingSetsTransform(std::move(builder), std::move(subqueries), ctx->context); ctx->minmax_idx = std::make_shared(); @@ -1537,7 +1552,7 @@ private: /*blocks_are_granules_size=*/ false, ctx->context->getWriteSettings()); - ctx->mutating_pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); + ctx->mutating_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); ctx->mutating_pipeline.setProgressCallback(ctx->progress_callback); /// Is calculated inside MergeProgressCallback. ctx->mutating_pipeline.disableProfileEventUpdate(); @@ -1712,13 +1727,25 @@ private: if (ctx->mutating_pipeline_builder.initialized()) { - QueryPipelineBuilder builder(std::move(ctx->mutating_pipeline_builder)); + auto builder = std::make_unique(std::move(ctx->mutating_pipeline_builder)); + PreparedSets::Subqueries subqueries; if (ctx->execute_ttl_type == ExecuteTTLType::NORMAL) - builder.addTransform(std::make_shared(builder.getHeader(), *ctx->data, ctx->metadata_snapshot, ctx->new_data_part, ctx->time_of_mutation, true)); + { + auto transform = std::make_shared(ctx->context, builder->getHeader(), *ctx->data, ctx->metadata_snapshot, ctx->new_data_part, ctx->time_of_mutation, true); + subqueries = transform->getSubqueries(); + builder->addTransform(std::move(transform)); + } if (ctx->execute_ttl_type == ExecuteTTLType::RECALCULATE) - builder.addTransform(std::make_shared(builder.getHeader(), *ctx->data, ctx->metadata_snapshot, ctx->new_data_part, ctx->time_of_mutation, true)); + { + auto transform = std::make_shared(ctx->context, builder->getHeader(), *ctx->data, ctx->metadata_snapshot, ctx->new_data_part, ctx->time_of_mutation, true); + subqueries = transform->getSubqueries(); + builder->addTransform(std::move(transform)); + } + + if (!subqueries.empty()) + builder = addCreatingSetsTransform(std::move(builder), std::move(subqueries), ctx->context); ctx->out = std::make_shared( ctx->new_data_part, @@ -1732,7 +1759,7 @@ private: &ctx->source_part->index_granularity_info ); - ctx->mutating_pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); + ctx->mutating_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); ctx->mutating_pipeline.setProgressCallback(ctx->progress_callback); /// Is calculated inside MergeProgressCallback. ctx->mutating_pipeline.disableProfileEventUpdate(); diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 7db5af82e0b..158c13b653d 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -279,7 +279,7 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies( auto add_for_rows_ttl = [&](const auto & expression, auto & to_set) { - if (add_dependent_columns(expression, to_set) && include_ttl_target) + if (add_dependent_columns(expression.getNames(), to_set) && include_ttl_target) { /// Filter all columns, if rows TTL expression have to be recalculated. for (const auto & column : getColumns().getAllPhysical()) @@ -297,16 +297,16 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies( add_for_rows_ttl(entry.expression_columns, required_ttl_columns); for (const auto & entry : getRecompressionTTLs()) - add_dependent_columns(entry.expression_columns, required_ttl_columns); + add_dependent_columns(entry.expression_columns.getNames(), required_ttl_columns); for (const auto & [name, entry] : getColumnTTLs()) { - if (add_dependent_columns(entry.expression_columns, required_ttl_columns) && include_ttl_target) + if (add_dependent_columns(entry.expression_columns.getNames(), required_ttl_columns) && include_ttl_target) updated_ttl_columns.insert(name); } for (const auto & entry : getMoveTTLs()) - add_dependent_columns(entry.expression_columns, required_ttl_columns); + add_dependent_columns(entry.expression_columns.getNames(), required_ttl_columns); //TODO what about rows_where_ttl and group_by_ttl ?? diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 47138f30e4f..e02ac933028 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -169,6 +169,23 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType return result; } +ExpressionAndSets TTLDescription::buildExpression(const ContextPtr & context) const +{ + auto ast = expression_ast->clone(); + return buildExpressionAndSets(ast, expression_columns, context); +} + +ExpressionAndSets TTLDescription::buildWhereExpression(const ContextPtr & context) const +{ + if (where_expression_ast) + { + auto ast = where_expression_ast->clone(); + return buildExpressionAndSets(ast, where_expression_columns, context); + } + + return {}; +} + TTLDescription TTLDescription::getTTLFromAST( const ASTPtr & definition_ast, const ColumnsDescription & columns, @@ -186,7 +203,7 @@ TTLDescription TTLDescription::getTTLFromAST( auto ttl_ast = result.expression_ast->clone(); auto expression = buildExpressionAndSets(ttl_ast, columns.getAllPhysical(), context).expression; - result.expression_columns = expression->getRequiredColumns(); + result.expression_columns = expression->getRequiredColumnsWithTypes(); // auto syntax_analyzer_result = TreeRewriter(context).analyze(ttl_ast, columns.getAllPhysical()); // result.expression = ExpressionAnalyzer(ttl_ast, syntax_analyzer_result, context).getActions(false); @@ -214,6 +231,8 @@ TTLDescription TTLDescription::getTTLFromAST( where_expression = buildExpressionAndSets(where_expr_ast, columns.getAllPhysical(), context).expression; // auto where_syntax_result = TreeRewriter(context).analyze(where_expr_ast, columns.getAllPhysical()); // result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, context).getActions(false); + + result.where_expression_columns = where_expression->getRequiredColumnsWithTypes(); result.where_result_column = where_expr_ast->getColumnName(); } } diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h index 5ea243424cb..7dfc736ded2 100644 --- a/src/Storages/TTLDescription.h +++ b/src/Storages/TTLDescription.h @@ -51,10 +51,10 @@ struct TTLDescription /// TTL d + INTERVAL 1 DAY /// ^~~~~~~~~~~~~~~~~~~^ ASTPtr expression_ast; - Names expression_columns; + NamesAndTypesList expression_columns; /// Expression actions evaluated from AST - ExpressionAndSets buildExpression() const; + ExpressionAndSets buildExpression(const ContextPtr & context) const; /// Result column of this TTL expression String result_column; @@ -63,7 +63,8 @@ struct TTLDescription /// TTL ... WHERE x % 10 == 0 and y > 5 /// ^~~~~~~~~~~~~~~~~~~~~~^ ASTPtr where_expression_ast; - ExpressionAndSets buildWhereExpression() const; + NamesAndTypesList where_expression_columns; + ExpressionAndSets buildWhereExpression(const ContextPtr & context) const; /// Name of result column from WHERE expression String where_result_column; From 16558ccc840d7a15efb2ab0fe691a79c38dd5086 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 4 Dec 2023 18:13:34 +0000 Subject: [PATCH 007/103] Fix some tests --- src/Storages/TTLDescription.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index e02ac933028..e32ff11860b 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -103,7 +103,10 @@ using FindAggregateFunctionVisitor = InDepthNodeVisitorclone() : nullptr) + , expression_columns(other.expression_columns) , result_column(other.result_column) + , where_expression_ast(other.where_expression_ast ? other.where_expression_ast->clone() : nullptr) + , where_expression_columns(other.where_expression_columns) , where_result_column(other.where_result_column) , group_by_keys(other.group_by_keys) , set_parts(other.set_parts) @@ -136,12 +139,20 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) // else // expression.reset(); + expression_columns = other.expression_columns; result_column = other.result_column; + + if (other.where_expression_ast) + where_expression_ast = other.where_expression_ast->clone(); + else + where_expression_ast.reset(); + // if (other.where_expression) // where_expression = other.where_expression->clone(); // else // where_expression.reset(); + where_expression_columns = other.where_expression_columns; where_result_column = other.where_result_column; group_by_keys = other.group_by_keys; set_parts = other.set_parts; From 6a821f9e737373b28bc98f25e10439dd04e7bdb8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 4 Dec 2023 19:24:27 +0000 Subject: [PATCH 008/103] Fix some staff --- src/Processors/QueryPlan/CreatingSetsStep.cpp | 3 +-- src/Processors/Transforms/TTLCalcTransform.cpp | 12 +++++++----- src/Processors/Transforms/TTLTransform.cpp | 12 +++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 11415e8d815..f13a717004f 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -182,8 +182,7 @@ QueryPipelineBuilderPtr addCreatingSetsTransform(QueryPipelineBuilderPtr pipelin pipelines.emplace_back(plan->buildQueryPipeline(plan_settings, pipeline_settings)); } - CreatingSetsStep(input_streams).updatePipeline(std::move(pipelines), pipeline_settings); - return std::move(pipelines.front()); + return CreatingSetsStep(input_streams).updatePipeline(std::move(pipelines), pipeline_settings); } std::vector> DelayedCreatingSetsStep::makePlansForSets(DelayedCreatingSetsStep && step) diff --git a/src/Processors/Transforms/TTLCalcTransform.cpp b/src/Processors/Transforms/TTLCalcTransform.cpp index 204dfe21733..0af9f38b20f 100644 --- a/src/Processors/Transforms/TTLCalcTransform.cpp +++ b/src/Processors/Transforms/TTLCalcTransform.cpp @@ -7,13 +7,15 @@ namespace DB static TTLExpressions getExpressions(const TTLDescription & ttl_descr, PreparedSets::Subqueries & subqueries_for_sets, const ContextPtr & context) { auto expr = ttl_descr.buildExpression(context); - auto where_expr = ttl_descr.buildWhereExpression(context); - auto expr_queries = expr.sets->getSubqueries(); - auto where_expr_queries = expr.sets->getSubqueries(); - subqueries_for_sets.insert(subqueries_for_sets.end(), expr_queries.begin(), expr_queries.end()); - subqueries_for_sets.insert(subqueries_for_sets.end(), where_expr_queries.begin(), where_expr_queries.end()); + + auto where_expr = ttl_descr.buildWhereExpression(context); + if (where_expr.sets) + { + auto where_expr_queries = where_expr.sets->getSubqueries(); + subqueries_for_sets.insert(subqueries_for_sets.end(), where_expr_queries.begin(), where_expr_queries.end()); + } return {expr.expression, where_expr.expression}; } diff --git a/src/Processors/Transforms/TTLTransform.cpp b/src/Processors/Transforms/TTLTransform.cpp index 69e2e6e5fc0..69b7d80c563 100644 --- a/src/Processors/Transforms/TTLTransform.cpp +++ b/src/Processors/Transforms/TTLTransform.cpp @@ -19,13 +19,15 @@ namespace DB static TTLExpressions getExpressions(const TTLDescription & ttl_descr, PreparedSets::Subqueries & subqueries_for_sets, const ContextPtr & context) { auto expr = ttl_descr.buildExpression(context); - auto where_expr = ttl_descr.buildWhereExpression(context); - auto expr_queries = expr.sets->getSubqueries(); - auto where_expr_queries = expr.sets->getSubqueries(); - subqueries_for_sets.insert(subqueries_for_sets.end(), expr_queries.begin(), expr_queries.end()); - subqueries_for_sets.insert(subqueries_for_sets.end(), where_expr_queries.begin(), where_expr_queries.end()); + + auto where_expr = ttl_descr.buildWhereExpression(context); + if (where_expr.sets) + { + auto where_expr_queries = where_expr.sets->getSubqueries(); + subqueries_for_sets.insert(subqueries_for_sets.end(), where_expr_queries.begin(), where_expr_queries.end()); + } return {expr.expression, where_expr.expression}; } From 0015ec28f9f70548c31e220f2dd826e4ac21f007 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 5 Dec 2023 12:45:25 +0000 Subject: [PATCH 009/103] Fixing test. --- src/Storages/TTLDescription.cpp | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index e32ff11860b..bfd3afc30d8 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB @@ -172,11 +173,26 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndTypesList & columns, const ContextPtr & context) { ExpressionAndSets result; + auto ttl_string = queryToString(ast); auto syntax_analyzer_result = TreeRewriter(context).analyze(ast, columns); ExpressionAnalyzer analyzer(ast, syntax_analyzer_result, context); - result.expression = analyzer.getActions(false); + auto dag = analyzer.getActionsDAG(false); + + const auto * col = &dag->findInOutputs(ast->getColumnName()); + // std::cerr << "buildExpressionAndSets " << ttl_string << std::endl; + if (col->result_name != ttl_string) + col = &dag->addAlias(*col, ttl_string); + + dag->getOutputs() = {col}; + dag->removeUnusedActions(); + + result.expression = std::make_shared(dag, ExpressionActionsSettings::fromContext(context)); result.sets = analyzer.getPreparedSets(); + // std::cerr << "--------- buildExpressionAndSets\n"; + // std::cerr << result.expression->dumpActions() << std::endl; + // std::cerr << result.sets->getSubqueries().size() << std::endl; + return result; } @@ -218,7 +234,7 @@ TTLDescription TTLDescription::getTTLFromAST( // auto syntax_analyzer_result = TreeRewriter(context).analyze(ttl_ast, columns.getAllPhysical()); // result.expression = ExpressionAnalyzer(ttl_ast, syntax_analyzer_result, context).getActions(false); - result.result_column = ttl_ast->getColumnName(); + result.result_column = expression->getSampleBlock().safeGetByPosition(0).name; ExpressionActionsPtr where_expression; @@ -244,7 +260,7 @@ TTLDescription TTLDescription::getTTLFromAST( // result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, context).getActions(false); result.where_expression_columns = where_expression->getRequiredColumnsWithTypes(); - result.where_result_column = where_expr_ast->getColumnName(); + result.where_result_column = where_expression->getSampleBlock().safeGetByPosition(0).name; } } else if (ttl_element->mode == TTLMode::GROUP_BY) From 43a23898e0ddb71fe810dafd850cef911dace902 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 5 Dec 2023 14:20:07 +0000 Subject: [PATCH 010/103] Updating the tests. --- .../0_stateless/01465_ttl_recompression.reference | 6 +++--- .../queries/0_stateless/02932_set_ttl_where.reference | 3 +++ tests/queries/0_stateless/02932_set_ttl_where.sql | 10 +--------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/queries/0_stateless/01465_ttl_recompression.reference b/tests/queries/0_stateless/01465_ttl_recompression.reference index 108df565669..90661a5dc78 100644 --- a/tests/queries/0_stateless/01465_ttl_recompression.reference +++ b/tests/queries/0_stateless/01465_ttl_recompression.reference @@ -13,9 +13,9 @@ CREATE TABLE default.recompression_table\n(\n `dt` DateTime,\n `key` UInt6 1_1_1 LZ4 2_2_2 ZSTD(12) 3_3_3 ZSTD(12) -1_1_1 ['plus(dt, toIntervalDay(1))'] -2_2_2 ['plus(dt, toIntervalDay(1))'] -3_3_3 ['plus(dt, toIntervalDay(1))'] +1_1_1 ['dt + toIntervalDay(1)'] +2_2_2 ['dt + toIntervalDay(1)'] +3_3_3 ['dt + toIntervalDay(1)'] 1_1_1 LZ4 2_2_2 LZ4 3_3_3 LZ4 diff --git a/tests/queries/0_stateless/02932_set_ttl_where.reference b/tests/queries/0_stateless/02932_set_ttl_where.reference index e69de29bb2d..bb0b1cf658d 100644 --- a/tests/queries/0_stateless/02932_set_ttl_where.reference +++ b/tests/queries/0_stateless/02932_set_ttl_where.reference @@ -0,0 +1,3 @@ +0 +0 +0 diff --git a/tests/queries/0_stateless/02932_set_ttl_where.sql b/tests/queries/0_stateless/02932_set_ttl_where.sql index 85fddf613e8..bf2b317c4bf 100644 --- a/tests/queries/0_stateless/02932_set_ttl_where.sql +++ b/tests/queries/0_stateless/02932_set_ttl_where.sql @@ -1,18 +1,10 @@ -create or replace table temp ( - a UInt32 -) -engine = MergeTree -order by a; - -insert into temp select number from system.numbers limit 100_000; - create or replace table t_temp ( a UInt32, timestamp DateTime ) engine = MergeTree order by a -TTL timestamp + INTERVAL 2 SECOND WHERE a in (select a from temp); +TTL timestamp + INTERVAL 2 SECOND WHERE a in (select number from system.numbers limit 100_000); select sleep(1); insert into t_temp select rand(), now() from system.numbers limit 1_000_000; From 7dc7062dadd5ddf3bed3dea4364cabfa97bcd61a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 6 Dec 2023 12:53:14 +0000 Subject: [PATCH 011/103] Fixing test. --- src/Interpreters/PreparedSets.cpp | 3 ++- src/Interpreters/Set.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index ea8d9a62b8b..9f646825d9f 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -189,7 +189,8 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) } } - set_and_key->set->fillSetElements(); + if (!set_and_key->set->hasSetElements()) + set_and_key->set->fillSetElements(); return buildSetInplace(context); } diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index 7136b090c42..7e8e0f2371b 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -77,6 +77,7 @@ public: const DataTypes & getElementsTypes() const { return set_elements_types; } bool hasExplicitSetElements() const { return fill_set_elements || (!set_elements.empty() && set_elements.front()->size() == data.getTotalRowCount()); } + bool hasSetElements() const { return !set_elements.empty(); } Columns getSetElements() const { checkIsCreated(); return { set_elements.begin(), set_elements.end() }; } void checkColumnsNumber(size_t num_key_columns) const; From 3e22f29b4529b6fefd5e92616ce9ef1ac33966d0 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sat, 23 Dec 2023 11:40:58 +0100 Subject: [PATCH 012/103] Fixed parameters --- docs/en/operations/backup.md | 2 +- .../registerBackupEngineAzureBlobStorage.cpp | 25 +++++++++++++++---- .../test.py | 2 +- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index 15d953249a0..4871f97c270 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -463,7 +463,7 @@ To write backups to an AzureBlobStorage container you need the following pieces The destination for a backup will be specified like this: ``` -AzureBlobStorage('/', '', '', '', ') +AzureBlobStorage('/', '', '', '', '') ``` ```sql diff --git a/src/Backups/registerBackupEngineAzureBlobStorage.cpp b/src/Backups/registerBackupEngineAzureBlobStorage.cpp index 6f7b5f38c28..ef95206831f 100644 --- a/src/Backups/registerBackupEngineAzureBlobStorage.cpp +++ b/src/Backups/registerBackupEngineAzureBlobStorage.cpp @@ -22,6 +22,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int SUPPORT_IS_DISABLED; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } #if USE_AZURE_BLOB_STORAGE @@ -54,20 +55,34 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory) StorageAzureBlob::Configuration configuration; - if (args.size() == 4) + if (args.size() == 3) { configuration.connection_url = args[0].safeGet(); configuration.is_connection_string = true; configuration.container = args[1].safeGet(); configuration.blob_path = args[2].safeGet(); - configuration.format = args[3].safeGet(); LOG_TRACE(&Poco::Logger::get("registerBackupEngineAzureBlobStorage"), "configuration.connection_url = {}" "configuration.container = {}" - "configuration.blob_path = {}" - "configuration.format = {}", - configuration.connection_url, configuration.container, configuration.blob_path, configuration.format); + "configuration.blob_path = {}", + configuration.connection_url, configuration.container, configuration.blob_path); + } + else if (args.size() == 5) + { + configuration.connection_url = args[0].safeGet(); + configuration.is_connection_string = false; + + configuration.container = args[1].safeGet(); + configuration.blob_path = args[2].safeGet(); + configuration.account_name = args[3].safeGet(); + configuration.account_key = args[4].safeGet(); + + } + else + { + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Backup AzureBlobStorage requires 3 or 5 arguments: connection string>/ Date: Wed, 27 Dec 2023 10:28:52 +0100 Subject: [PATCH 013/103] Updated thread name --- src/Backups/BackupIO_AzureBlobStorage.cpp | 8 +- src/Backups/BackupIO_AzureBlobStorage.h | 81 +++++++++---------- .../copyAzureBlobStorageFile.cpp | 25 +++--- 3 files changed, 59 insertions(+), 55 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index d41d23e3c36..a1fd5bd8327 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -35,7 +35,7 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( const WriteSettings & write_settings_, const ContextPtr & context_) : BackupReaderDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupReaderAzureBlobStorage")) - , data_source_description{DataSourceType::AzureBlobStorage, "AzureBlobStorage", false, false} + , data_source_description{DataSourceType::AzureBlobStorage, configuration_.container, false, false} , configuration(configuration_) { client = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); @@ -160,7 +160,7 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( const WriteSettings & write_settings_, const ContextPtr & context_) : BackupWriterDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupWriterAzureBlobStorage")) - , data_source_description{DataSourceType::AzureBlobStorage, "AzureBlobStorage", false, false} + , data_source_description{DataSourceType::AzureBlobStorage,configuration_.container, false, false} , configuration(configuration_) { client = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); @@ -209,7 +209,7 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu settings, read_settings, {}, - threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterAzureBlobStorage")); + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); return; /// copied! } } @@ -221,7 +221,7 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu void BackupWriterAzureBlobStorage::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { copyDataToAzureBlobStorageFile(create_read_buffer, start_pos, length, client, configuration.container, path_in_backup, settings, {}, - threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterAzureBlobStorage")); + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); } BackupWriterAzureBlobStorage::~BackupWriterAzureBlobStorage() = default; diff --git a/src/Backups/BackupIO_AzureBlobStorage.h b/src/Backups/BackupIO_AzureBlobStorage.h index 6ef66fc432d..65affb9f079 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.h +++ b/src/Backups/BackupIO_AzureBlobStorage.h @@ -12,57 +12,54 @@ namespace DB { -// using AzureClientPtr = std::shared_ptr; - /// Represents a backup stored to Azure - class BackupReaderAzureBlobStorage : public BackupReaderDefault - { - public: - BackupReaderAzureBlobStorage(StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_); - ~BackupReaderAzureBlobStorage() override; +class BackupReaderAzureBlobStorage : public BackupReaderDefault +{ +public: + BackupReaderAzureBlobStorage(StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_); + ~BackupReaderAzureBlobStorage() override; - bool fileExists(const String & file_name) override; - UInt64 getFileSize(const String & file_name) override; - std::unique_ptr readFile(const String & file_name) override; + bool fileExists(const String & file_name) override; + UInt64 getFileSize(const String & file_name) override; + std::unique_ptr readFile(const String & file_name) override; - void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup, - DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override; + void copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup, + DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) override; - private: - const DataSourceDescription data_source_description; - std::shared_ptr client; - StorageAzureBlob::Configuration configuration; - std::unique_ptr object_storage; - std::shared_ptr settings; - }; +private: + const DataSourceDescription data_source_description; + std::shared_ptr client; + StorageAzureBlob::Configuration configuration; + std::unique_ptr object_storage; + std::shared_ptr settings; +}; +class BackupWriterAzureBlobStorage : public BackupWriterDefault +{ +public: + BackupWriterAzureBlobStorage(StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_); + ~BackupWriterAzureBlobStorage() override; - class BackupWriterAzureBlobStorage : public BackupWriterDefault - { - public: - BackupWriterAzureBlobStorage(StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_); - ~BackupWriterAzureBlobStorage() override; + bool fileExists(const String & file_name) override; + UInt64 getFileSize(const String & file_name) override; + std::unique_ptr writeFile(const String & file_name) override; - bool fileExists(const String & file_name) override; - UInt64 getFileSize(const String & file_name) override; - std::unique_ptr writeFile(const String & file_name) override; + void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) override; + void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, + bool copy_encrypted, UInt64 start_pos, UInt64 length) override; - void copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) override; - void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, - bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + void removeFile(const String & file_name) override; + void removeFiles(const Strings & file_names) override; - void removeFile(const String & file_name) override; - void removeFiles(const Strings & file_names) override; - - private: - std::unique_ptr readFile(const String & file_name, size_t expected_file_size) override; - void removeFilesBatch(const Strings & file_names); - const DataSourceDescription data_source_description; - std::shared_ptr client; - StorageAzureBlob::Configuration configuration; - std::unique_ptr object_storage; - std::shared_ptr settings; - }; +private: + std::unique_ptr readFile(const String & file_name, size_t expected_file_size) override; + void removeFilesBatch(const Strings & file_names); + const DataSourceDescription data_source_description; + std::shared_ptr client; + StorageAzureBlob::Configuration configuration; + std::unique_ptr object_storage; + std::shared_ptr settings; +}; } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index bf0bcac664b..0a0a080b5cb 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -22,6 +22,11 @@ namespace ProfileEvents extern const Event DiskAzureUploadPart; } +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace DB { @@ -44,7 +49,8 @@ namespace std::shared_ptr settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, - bool for_disk_azure_blob_storage_) + bool for_disk_azure_blob_storage_, + const Poco::Logger * log_) : create_read_buffer(create_read_buffer_) , client(client_) , offset (offset_) @@ -55,7 +61,7 @@ namespace , object_metadata(object_metadata_) , schedule(schedule_) , for_disk_azure_blob_storage(for_disk_azure_blob_storage_) - , log(&Poco::Logger::get("azureBlobStorageUploadHelper")) + , log(log_) , max_single_part_upload_size(settings_.get()->max_single_part_upload_size) { } @@ -179,11 +185,11 @@ namespace try { auto read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); - auto buffer = std::make_unique(std::move(read_buffer), part_size); task->data = new char[part_size]; task->size = part_size; - buffer->read(task->data,part_size); - task->block_id = getRandomASCIIString(64); + size_t n = read_buffer->read(task->data,part_size); + if (n != part_size) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected size"); schedule([this, task, task_finish_notify]() { @@ -208,9 +214,10 @@ namespace { UploadPartTask task; auto read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); - auto buffer = std::make_unique(std::move(read_buffer), part_size); task.data = new char[part_size]; - buffer->read(task.data,part_size); + size_t n = read_buffer->read(task.data,part_size); + if (n != part_size) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected size"); task.size = part_size; processUploadTask(task); block_ids.emplace_back(task.block_id); @@ -274,7 +281,7 @@ void copyDataToAzureBlobStorageFile( ThreadPoolCallbackRunner schedule, bool for_disk_azure_blob_storage) { - UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_azure_blob_storage}; + UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyDataToAzureBlobStorageFile")}; helper.performCopy(); } @@ -314,7 +321,7 @@ void copyAzureBlobStorageFile( settings->max_single_download_retries); }; - UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_azure_blob_storage}; + UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyAzureBlobStorageFile")}; helper.performCopy(); } } From b70ff6d8ea71d4633cdcdbe3ef486707e70c1abb Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 1 Jan 2024 11:02:57 +0100 Subject: [PATCH 014/103] Fix build --- src/Backups/BackupIO_AzureBlobStorage.cpp | 33 +++++++++++++++++++++-- src/Backups/BackupIO_AzureBlobStorage.h | 2 ++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index a1fd5bd8327..bd4efcf63ae 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -218,10 +218,39 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length); } +void BackupWriterAzureBlobStorage::copyFile(const String & destination, const String & source, size_t size) +{ + std::shared_ptr src_client; + std::shared_ptr dest_client; + StorageAzureBlob::Configuration src_configuration = configuration; + src_configuration.container = source; + src_client = StorageAzureBlob::createClient(src_configuration, /* is_read_only */ false); + + StorageAzureBlob::Configuration dest_configuration = configuration; + dest_configuration.container = destination; + dest_client = StorageAzureBlob::createClient(dest_configuration, /* is_read_only */ false); + + LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); + copyAzureBlobStorageFile( + src_client, + dest_client, + configuration.container, + fs::path(configuration.blob_path), + 0, + size, + /* dest_bucket= */ destination, + /* dest_key= */ configuration.blob_path, + settings, + read_settings, + {}, + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupRDAzure"), + /* for_disk_azure_blob_storage= */ true); +} + void BackupWriterAzureBlobStorage::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { copyDataToAzureBlobStorageFile(create_read_buffer, start_pos, length, client, configuration.container, path_in_backup, settings, {}, - threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWRAzure")); } BackupWriterAzureBlobStorage::~BackupWriterAzureBlobStorage() = default; @@ -257,7 +286,7 @@ UInt64 BackupWriterAzureBlobStorage::getFileSize(const String & file_name) RelativePathsWithMetadata children; object_storage->listObjects(key,children,/*max_keys*/0); if (children.empty()) - throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Object {} must exist"); + throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Object must exist"); return children[0].metadata.size_bytes; } diff --git a/src/Backups/BackupIO_AzureBlobStorage.h b/src/Backups/BackupIO_AzureBlobStorage.h index 65affb9f079..87a6c3ef675 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.h +++ b/src/Backups/BackupIO_AzureBlobStorage.h @@ -48,6 +48,8 @@ public: void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + void copyFile(const String & destination, const String & source, size_t size) override; + void removeFile(const String & file_name) override; void removeFiles(const Strings & file_names) override; From 4122de97213d835de5202d4ca741b4972973884b Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 2 Jan 2024 20:19:01 +0100 Subject: [PATCH 015/103] Updated tests and added settings --- src/Backups/BackupIO_AzureBlobStorage.cpp | 6 +- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 5 +- .../AzureBlobStorage/AzureObjectStorage.h | 11 ++- .../copyAzureBlobStorageFile.cpp | 68 +++++++++++++++++-- src/Storages/StorageAzureBlob.cpp | 2 +- .../configs/config.xml | 11 --- .../configs/disable_profilers.xml | 13 ---- .../configs/users.xml | 8 --- .../test.py | 2 - 9 files changed, 80 insertions(+), 46 deletions(-) delete mode 100644 tests/integration/test_backup_restore_azure_blob_storage/configs/config.xml delete mode 100644 tests/integration/test_backup_restore_azure_blob_storage/configs/disable_profilers.xml delete mode 100644 tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index bd4efcf63ae..15e8e92a85d 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -139,7 +139,7 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, settings, read_settings, object_attributes, - threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupReaderAzureBlobStorage"), + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupRDAzure"), /* for_disk_azure_blob_storage= */ true); return file_size; @@ -209,7 +209,7 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu settings, read_settings, {}, - threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWRAzure")); return; /// copied! } } @@ -243,7 +243,7 @@ void BackupWriterAzureBlobStorage::copyFile(const String & destination, const St settings, read_settings, {}, - threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupRDAzure"), + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWRAzure"), /* for_disk_azure_blob_storage= */ true); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 6075b385a6c..9e703d6fc5e 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -164,7 +164,10 @@ std::unique_ptr getAzureBlobStorageSettings(const Po config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), config.getInt(config_prefix + ".max_single_read_retries", 3), config.getInt(config_prefix + ".max_single_download_retries", 3), - config.getInt(config_prefix + ".list_object_keys_size", 1000) + config.getInt(config_prefix + ".list_object_keys_size", 1000), + config.getUInt64(config_prefix + ".min_upload_part_size", 16 * 1024 * 1024), + config.getUInt64(config_prefix + ".max_upload_part_size", 5ULL * 1024 * 1024 * 1024), + config.getUInt64(config_prefix + ".max_part_number", 10000) ); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 8e3d50418d3..55c81b4b7d9 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -24,12 +24,18 @@ struct AzureObjectStorageSettings uint64_t min_bytes_for_seek_, int max_single_read_retries_, int max_single_download_retries_, - int list_object_keys_size_) + int list_object_keys_size_, + size_t min_upload_part_size_, + size_t max_upload_part_size_, + size_t max_part_number_) : max_single_part_upload_size(max_single_part_upload_size_) , min_bytes_for_seek(min_bytes_for_seek_) , max_single_read_retries(max_single_read_retries_) , max_single_download_retries(max_single_download_retries_) , list_object_keys_size(list_object_keys_size_) + , min_upload_part_size(min_upload_part_size_) + , max_upload_part_size(max_upload_part_size_) + , max_part_number(max_part_number_) { } @@ -40,6 +46,9 @@ struct AzureObjectStorageSettings size_t max_single_read_retries = 3; size_t max_single_download_retries = 3; int list_object_keys_size = 1000; + size_t min_upload_part_size = 16 * 1024 * 1024; + size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; + size_t max_part_number = 10000; }; using AzureClient = Azure::Storage::Blobs::BlobContainerClient; diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 0a0a080b5cb..5ca30fa8071 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -22,15 +22,17 @@ namespace ProfileEvents extern const Event DiskAzureUploadPart; } -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int INVALID_CONFIG_PARAMETER; +} + + size_t max_single_operation_copy_size = 256 * 1024 * 1024; @@ -106,6 +108,60 @@ namespace std::mutex bg_tasks_mutex; std::condition_variable bg_tasks_condvar; + void calculatePartSize() + { + if (!total_size) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Chosen multipart upload for an empty file. This must not happen"); + + auto max_part_number = settings.get()->max_part_number; + auto min_upload_part_size = settings.get()->min_upload_part_size; + auto max_upload_part_size = settings.get()->max_upload_part_size; + + if (!max_part_number) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_part_number must not be 0"); + else if (!min_upload_part_size) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "min_upload_part_size must not be 0"); + else if (max_upload_part_size < min_upload_part_size) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_upload_part_size must not be less than min_upload_part_size"); + + size_t part_size = min_upload_part_size; + size_t num_parts = (total_size + part_size - 1) / part_size; + + if (num_parts > max_part_number) + { + part_size = (total_size + max_part_number - 1) / max_part_number; + num_parts = (total_size + part_size - 1) / part_size; + } + + if (part_size > max_upload_part_size) + { + part_size = max_upload_part_size; + num_parts = (total_size + part_size - 1) / part_size; + } + + if (num_parts < 1 || num_parts > max_part_number || part_size < min_upload_part_size || part_size > max_upload_part_size) + { + String msg; + if (num_parts < 1) + msg = "Number of parts is zero"; + else if (num_parts > max_part_number) + msg = fmt::format("Number of parts exceeds {}", num_parts, max_part_number); + else if (part_size < min_upload_part_size) + msg = fmt::format("Size of a part is less than {}", part_size, min_upload_part_size); + else + msg = fmt::format("Size of a part exceeds {}", part_size, max_upload_part_size); + + throw Exception( + ErrorCodes::INVALID_CONFIG_PARAMETER, + "{} while writing {} bytes to AzureBlobStorage. Check max_part_number = {}, " + "min_upload_part_size = {}, max_upload_part_size = {}", + msg, total_size, max_part_number, min_upload_part_size, max_upload_part_size); + } + + /// We've calculated the size of a normal part (the final part can be smaller). + normal_part_size = part_size; + } + public: void performCopy() { @@ -120,7 +176,7 @@ namespace void performMultipartUpload() { - normal_part_size = 1024; + calculatePartSize(); size_t position = offset; size_t end_position = offset + total_size; diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp index 1b28a2c2fac..f1070c8c31e 100644 --- a/src/Storages/StorageAzureBlob.cpp +++ b/src/Storages/StorageAzureBlob.cpp @@ -1139,7 +1139,7 @@ StorageAzureBlobSource::ReaderHolder StorageAzureBlobSource::createReader() QueryPipelineBuilder builder; std::shared_ptr source; std::unique_ptr read_buf; - std::optional num_rows_from_cache = need_only_count && getContext()->getSettingsRef().use_cache_for_count_from_files + std::optional num_rows_from_cache = need_only_count && getContext()->getSettingsRef().use_cache_for_count_from_files ? tryGetNumRowsFromCache(path_with_metadata) : std::nullopt; if (num_rows_from_cache) { diff --git a/tests/integration/test_backup_restore_azure_blob_storage/configs/config.xml b/tests/integration/test_backup_restore_azure_blob_storage/configs/config.xml deleted file mode 100644 index 5725dce40cd..00000000000 --- a/tests/integration/test_backup_restore_azure_blob_storage/configs/config.xml +++ /dev/null @@ -1,11 +0,0 @@ - - 1 - 0 - 0.0 - 0 - 1 - 1 - 0 - 16 - 16 - \ No newline at end of file diff --git a/tests/integration/test_backup_restore_azure_blob_storage/configs/disable_profilers.xml b/tests/integration/test_backup_restore_azure_blob_storage/configs/disable_profilers.xml deleted file mode 100644 index b74bb1502ce..00000000000 --- a/tests/integration/test_backup_restore_azure_blob_storage/configs/disable_profilers.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - 0 - 0 - 0 - 1000 - 1 - 1 - - - diff --git a/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml b/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml deleted file mode 100644 index c12eb2f79f4..00000000000 --- a/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - default - - - diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index 0a48d3523f0..06c18d7468f 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -24,8 +24,6 @@ def cluster(): cluster = ClickHouseCluster(__file__) cluster.add_instance( "node", - main_configs=["configs/config.xml"], - user_configs=["configs/disable_profilers.xml", "configs/users.xml"], with_azurite=True, ) cluster.start() From df221f7db65fd17af6a71704f756e47ceec7a928 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 3 Jan 2024 11:35:06 +0100 Subject: [PATCH 016/103] Renamed Bucket-Key to Container-Blob --- src/Backups/BackupIO_AzureBlobStorage.cpp | 14 +++--- .../copyAzureBlobStorageFile.cpp | 44 +++++++++---------- .../copyAzureBlobStorageFile.h | 10 ++--- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 15e8e92a85d..de40fc6b33b 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -134,8 +134,8 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, fs::path(configuration.blob_path) / path_in_backup, 0, file_size, - /* dest_bucket= */ blob_path[1], - /* dest_key= */ blob_path[0], + /* dest_container */ blob_path[1], + /* dest_path */ blob_path[0], settings, read_settings, object_attributes, @@ -178,7 +178,7 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu auto source_data_source_description = src_disk->getDataSourceDescription(); if (source_data_source_description.sameKind(data_source_description) && (source_data_source_description.is_encrypted == copy_encrypted)) { - /// getBlobPath() can return more than 3 elements if the file is stored as multiple objects in AzureBlobStorage bucket. + /// getBlobPath() can return more than 3 elements if the file is stored as multiple objects in AzureBlobStorage container. /// In this case we can't use the native copy. if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 2) { @@ -200,8 +200,8 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu copyAzureBlobStorageFile( src_client, client, - /* src_bucket */ blob_path[1], - /* src_key= */ blob_path[0], + /* src_container */ blob_path[1], + /* src_path */ blob_path[0], start_pos, length, configuration.container, @@ -238,8 +238,8 @@ void BackupWriterAzureBlobStorage::copyFile(const String & destination, const St fs::path(configuration.blob_path), 0, size, - /* dest_bucket= */ destination, - /* dest_key= */ configuration.blob_path, + /* dest_container */ destination, + /* dest_path */ configuration.blob_path, settings, read_settings, {}, diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 5ca30fa8071..df1341efdd1 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -46,8 +46,8 @@ namespace std::shared_ptr client_, size_t offset_, size_t total_size_, - const String & dest_bucket_, - const String & dest_key_, + const String & dest_container_, + const String & dest_blob_, std::shared_ptr settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, @@ -57,8 +57,8 @@ namespace , client(client_) , offset (offset_) , total_size (total_size_) - , dest_bucket(dest_bucket_) - , dest_key(dest_key_) + , dest_container(dest_container_) + , dest_blob(dest_blob_) , settings(settings_) , object_metadata(object_metadata_) , schedule(schedule_) @@ -75,8 +75,8 @@ namespace std::shared_ptr client; size_t offset; size_t total_size; - const String & dest_bucket; - const String & dest_key; + const String & dest_container; + const String & dest_blob; std::shared_ptr settings; const std::optional> & object_metadata; ThreadPoolCallbackRunner schedule; @@ -170,7 +170,7 @@ namespace void completeMultipartUpload() { - auto block_blob_client = client->GetBlockBlobClient(dest_key); + auto block_blob_client = client->GetBlockBlobClient(dest_blob); block_blob_client.CommitBlockList(block_ids); } @@ -207,7 +207,7 @@ namespace void uploadPart(size_t part_offset, size_t part_size) { - LOG_TRACE(log, "Writing part. Bucket: {}, Key: {}, Size: {}", dest_bucket, dest_key, part_size); + LOG_TRACE(log, "Writing part. Container: {}, Blob: {}, Size: {}", dest_container, dest_blob, part_size); if (!part_size) { @@ -286,7 +286,7 @@ namespace std::lock_guard lock(bg_tasks_mutex); /// Protect bg_tasks from race task.block_id = block_id; - LOG_TRACE(log, "Writing part finished. Bucket: {}, Key: {}, block_id: {}, Parts: {}", dest_bucket, dest_key, block_id, bg_tasks.size()); + LOG_TRACE(log, "Writing part finished. Container: {}, Blob: {}, block_id: {}, Parts: {}", dest_container, dest_blob, block_id, bg_tasks.size()); } String processUploadPartRequest(UploadPartTask & task) @@ -295,7 +295,7 @@ namespace if (for_disk_azure_blob_storage) ProfileEvents::increment(ProfileEvents::DiskAzureUploadPart); - auto block_blob_client = client->GetBlockBlobClient(dest_key); + auto block_blob_client = client->GetBlockBlobClient(dest_blob); task.block_id = getRandomASCIIString(64); Azure::Core::IO::MemoryBodyStream memory(reinterpret_cast(task.data), task.size); block_blob_client.StageBlock(task.block_id, memory); @@ -330,14 +330,14 @@ void copyDataToAzureBlobStorageFile( size_t offset, size_t size, std::shared_ptr & dest_client, - const String & dest_bucket, - const String & dest_key, + const String & dest_container, + const String & dest_blob, std::shared_ptr settings, const std::optional> & object_metadata, ThreadPoolCallbackRunner schedule, bool for_disk_azure_blob_storage) { - UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyDataToAzureBlobStorageFile")}; + UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container, dest_blob, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyDataToAzureBlobStorageFile")}; helper.performCopy(); } @@ -345,12 +345,12 @@ void copyDataToAzureBlobStorageFile( void copyAzureBlobStorageFile( std::shared_ptr src_client, std::shared_ptr dest_client, - const String & src_bucket, - const String & src_key, + const String & src_container, + const String & src_blob, size_t offset, size_t size, - const String & dest_bucket, - const String & dest_key, + const String & dest_container, + const String & dest_blob, std::shared_ptr settings, const ReadSettings & read_settings, const std::optional> & object_metadata, @@ -363,21 +363,21 @@ void copyAzureBlobStorageFile( ProfileEvents::increment(ProfileEvents::AzureCopyObject); if (for_disk_azure_blob_storage) ProfileEvents::increment(ProfileEvents::DiskAzureCopyObject); - auto block_blob_client_src = src_client->GetBlockBlobClient(src_key); - auto block_blob_client_dest = dest_client->GetBlockBlobClient(dest_key); + auto block_blob_client_src = src_client->GetBlockBlobClient(src_blob); + auto block_blob_client_dest = dest_client->GetBlockBlobClient(dest_blob); auto uri = block_blob_client_src.GetUrl(); block_blob_client_dest.CopyFromUri(uri); } else { - LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Bucket: {}, Key: {}", src_bucket, src_key); + LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Container: {}, Blob: {}", src_container, src_blob); auto create_read_buffer = [&] { - return std::make_unique(src_client, src_key, read_settings, settings->max_single_read_retries, + return std::make_unique(src_client, src_blob, read_settings, settings->max_single_read_retries, settings->max_single_download_retries); }; - UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyAzureBlobStorageFile")}; + UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container, dest_blob, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyAzureBlobStorageFile")}; helper.performCopy(); } } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h index 31228fbcb23..059d0318f57 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h @@ -23,12 +23,12 @@ using CreateReadBuffer = std::function()>; void copyAzureBlobStorageFile( std::shared_ptr src_client, std::shared_ptr dest_client, - const String & src_bucket, - const String & src_key, + const String & src_container, + const String & src_path, size_t src_offset, size_t src_size, - const String & dest_bucket, - const String & dest_key, + const String & dest_container, + const String & dest_path, std::shared_ptr settings, const ReadSettings & read_settings, const std::optional> & object_metadata = std::nullopt, @@ -46,8 +46,8 @@ void copyDataToAzureBlobStorageFile( size_t offset, size_t size, std::shared_ptr & client, + const String & dest_container, const String & dest_bucket, - const String & dest_key, std::shared_ptr settings, const std::optional> & object_metadata = std::nullopt, ThreadPoolCallbackRunner schedule_ = {}, From 91bad5bc39963e9450f284dfc6b45fd69fa146de Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 4 Jan 2024 16:06:36 +0100 Subject: [PATCH 017/103] Updated to use MultiVersion for BlobContainerClient in Backups and updated to get client from disk --- src/Backups/BackupIO_AzureBlobStorage.cpp | 72 +++++-------------- src/Backups/BackupIO_AzureBlobStorage.h | 4 +- .../AzureBlobStorage/AzureObjectStorage.h | 5 ++ .../copyAzureBlobStorageFile.cpp | 20 +++--- .../copyAzureBlobStorageFile.h | 6 +- 5 files changed, 37 insertions(+), 70 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index de40fc6b33b..968a60c566f 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -27,8 +27,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -//using AzureClientPtr = std::shared_ptr; - BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, @@ -38,12 +36,13 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( , data_source_description{DataSourceType::AzureBlobStorage, configuration_.container, false, false} , configuration(configuration_) { - client = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); + auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); settings = StorageAzureBlob::createSettingsAsSharedPtr(context_); auto settings_as_unique_ptr = StorageAzureBlob::createSettings(context_); object_storage = std::make_unique("BackupReaderAzureBlobStorage", - std::make_unique(*client.get()), + std::move(client_ptr), std::move(settings_as_unique_ptr)); + client = object_storage->getClient(); } BackupReaderAzureBlobStorage::~BackupReaderAzureBlobStorage() = default; @@ -89,7 +88,7 @@ std::unique_ptr BackupReaderAzureBlobStorage::readFile(const key = file_name; } return std::make_unique( - client, key, read_settings, settings->max_single_read_retries, + client.get(), key, read_settings, settings->max_single_read_retries, settings->max_single_download_retries); } @@ -113,23 +112,9 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, "Blob writing function called with unexpected blob_path.size={} or mode={}", blob_path.size(), mode); - std::shared_ptr dest_client; - if (configuration.container == blob_path[1]) - { - dest_client = client; - } - else - { - StorageAzureBlob::Configuration dest_configuration = configuration; - dest_configuration.container = blob_path[1]; - dest_configuration.blob_path = blob_path[0]; - dest_client = StorageAzureBlob::createClient(dest_configuration, /* is_read_only */ false); - } - - copyAzureBlobStorageFile( client, - dest_client, + reinterpret_cast(destination_disk->getObjectStorage().get())->getClient(), configuration.container, fs::path(configuration.blob_path) / path_in_backup, 0, @@ -163,12 +148,13 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( , data_source_description{DataSourceType::AzureBlobStorage,configuration_.container, false, false} , configuration(configuration_) { - client = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); + auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); settings = StorageAzureBlob::createSettingsAsSharedPtr(context_); auto settings_as_unique_ptr = StorageAzureBlob::createSettings(context_); object_storage = std::make_unique("BackupWriterAzureBlobStorage", - std::make_unique(*client.get()), - std::move(settings_as_unique_ptr)); + std::move(client_ptr), + std::move(settings_as_unique_ptr)); + client = object_storage->getClient(); } void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, @@ -182,23 +168,9 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu /// In this case we can't use the native copy. if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 2) { - - std::shared_ptr src_client; - if (configuration.container == blob_path[1]) - { - src_client = client; - } - else - { - StorageAzureBlob::Configuration src_configuration = configuration; - src_configuration.container = blob_path[1]; - src_configuration.blob_path = blob_path[0]; - src_client = StorageAzureBlob::createClient(src_configuration, /* is_read_only */ false); - } - LOG_TRACE(log, "Copying file {} from disk {} to AzureBlobStorag", src_path, src_disk->getName()); copyAzureBlobStorageFile( - src_client, + reinterpret_cast(src_disk->getObjectStorage().get())->getClient(), client, /* src_container */ blob_path[1], /* src_path */ blob_path[0], @@ -220,26 +192,16 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu void BackupWriterAzureBlobStorage::copyFile(const String & destination, const String & source, size_t size) { - std::shared_ptr src_client; - std::shared_ptr dest_client; - StorageAzureBlob::Configuration src_configuration = configuration; - src_configuration.container = source; - src_client = StorageAzureBlob::createClient(src_configuration, /* is_read_only */ false); - - StorageAzureBlob::Configuration dest_configuration = configuration; - dest_configuration.container = destination; - dest_client = StorageAzureBlob::createClient(dest_configuration, /* is_read_only */ false); - LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); copyAzureBlobStorageFile( - src_client, - dest_client, + client, + client, configuration.container, - fs::path(configuration.blob_path), + fs::path(source), 0, size, - /* dest_container */ destination, - /* dest_path */ configuration.blob_path, + /* dest_container */ configuration.container, + /* dest_path */ destination, settings, read_settings, {}, @@ -303,7 +265,7 @@ std::unique_ptr BackupWriterAzureBlobStorage::readFile(const String } return std::make_unique( - client, key, read_settings, settings->max_single_read_retries, + client.get(), key, read_settings, settings->max_single_read_retries, settings->max_single_download_retries); } @@ -319,7 +281,7 @@ std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const Strin key = file_name; } return std::make_unique( - client, + client.get(), key, settings->max_single_part_upload_size, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Backups/BackupIO_AzureBlobStorage.h b/src/Backups/BackupIO_AzureBlobStorage.h index 87a6c3ef675..12bf073cd08 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.h +++ b/src/Backups/BackupIO_AzureBlobStorage.h @@ -28,7 +28,7 @@ public: private: const DataSourceDescription data_source_description; - std::shared_ptr client; + MultiVersion client; StorageAzureBlob::Configuration configuration; std::unique_ptr object_storage; std::shared_ptr settings; @@ -57,7 +57,7 @@ private: std::unique_ptr readFile(const String & file_name, size_t expected_file_size) override; void removeFilesBatch(const Strings & file_names); const DataSourceDescription data_source_description; - std::shared_ptr client; + MultiVersion client; StorageAzureBlob::Configuration configuration; std::unique_ptr object_storage; std::shared_ptr settings; diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 55c81b4b7d9..1ff4537742f 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -134,6 +134,11 @@ public: bool isRemote() const override { return true; } + MultiVersion & getClient() + { + return client; + } + private: const String name; /// client used to access the files in the Blob Storage cloud diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index df1341efdd1..4ec90d2830e 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -43,7 +43,7 @@ namespace public: UploadHelper( const CreateReadBuffer & create_read_buffer_, - std::shared_ptr client_, + MultiVersion & client_, size_t offset_, size_t total_size_, const String & dest_container_, @@ -72,7 +72,7 @@ namespace protected: std::function()> create_read_buffer; - std::shared_ptr client; + MultiVersion & client; size_t offset; size_t total_size; const String & dest_container; @@ -170,7 +170,7 @@ namespace void completeMultipartUpload() { - auto block_blob_client = client->GetBlockBlobClient(dest_blob); + auto block_blob_client = client.get()->GetBlockBlobClient(dest_blob); block_blob_client.CommitBlockList(block_ids); } @@ -295,7 +295,7 @@ namespace if (for_disk_azure_blob_storage) ProfileEvents::increment(ProfileEvents::DiskAzureUploadPart); - auto block_blob_client = client->GetBlockBlobClient(dest_blob); + auto block_blob_client = client.get()->GetBlockBlobClient(dest_blob); task.block_id = getRandomASCIIString(64); Azure::Core::IO::MemoryBodyStream memory(reinterpret_cast(task.data), task.size); block_blob_client.StageBlock(task.block_id, memory); @@ -329,7 +329,7 @@ void copyDataToAzureBlobStorageFile( const std::function()> & create_read_buffer, size_t offset, size_t size, - std::shared_ptr & dest_client, + MultiVersion & dest_client, const String & dest_container, const String & dest_blob, std::shared_ptr settings, @@ -343,8 +343,8 @@ void copyDataToAzureBlobStorageFile( void copyAzureBlobStorageFile( - std::shared_ptr src_client, - std::shared_ptr dest_client, + MultiVersion & src_client, + MultiVersion & dest_client, const String & src_container, const String & src_blob, size_t offset, @@ -363,8 +363,8 @@ void copyAzureBlobStorageFile( ProfileEvents::increment(ProfileEvents::AzureCopyObject); if (for_disk_azure_blob_storage) ProfileEvents::increment(ProfileEvents::DiskAzureCopyObject); - auto block_blob_client_src = src_client->GetBlockBlobClient(src_blob); - auto block_blob_client_dest = dest_client->GetBlockBlobClient(dest_blob); + auto block_blob_client_src = src_client.get()->GetBlockBlobClient(src_blob); + auto block_blob_client_dest = dest_client.get()->GetBlockBlobClient(dest_blob); auto uri = block_blob_client_src.GetUrl(); block_blob_client_dest.CopyFromUri(uri); } @@ -373,7 +373,7 @@ void copyAzureBlobStorageFile( LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Container: {}, Blob: {}", src_container, src_blob); auto create_read_buffer = [&] { - return std::make_unique(src_client, src_blob, read_settings, settings->max_single_read_retries, + return std::make_unique(src_client.get(), src_blob, read_settings, settings->max_single_read_retries, settings->max_single_download_retries); }; diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h index 059d0318f57..a6502541db1 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h @@ -21,8 +21,8 @@ using CreateReadBuffer = std::function()>; /// Copies a file from AzureBlobStorage to AzureBlobStorage. /// The parameters `src_offset` and `src_size` specify a part in the source to copy. void copyAzureBlobStorageFile( - std::shared_ptr src_client, - std::shared_ptr dest_client, + MultiVersion & src_client, + MultiVersion & dest_client, const String & src_container, const String & src_path, size_t src_offset, @@ -45,7 +45,7 @@ void copyDataToAzureBlobStorageFile( const std::function()> & create_read_buffer, size_t offset, size_t size, - std::shared_ptr & client, + MultiVersion & client, const String & dest_container, const String & dest_bucket, std::shared_ptr settings, From c14605caa7f403531a6ff0663c242aa5d466ab07 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 4 Jan 2024 18:27:54 +0100 Subject: [PATCH 018/103] Added flag use_native_copy and updated to use StartCopyFromUri for native copy with large files --- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 3 ++- .../AzureBlobStorage/AzureObjectStorage.h | 10 +++---- .../copyAzureBlobStorageFile.cpp | 26 ++++++++++++++++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 9e703d6fc5e..e29def06363 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -167,7 +167,8 @@ std::unique_ptr getAzureBlobStorageSettings(const Po config.getInt(config_prefix + ".list_object_keys_size", 1000), config.getUInt64(config_prefix + ".min_upload_part_size", 16 * 1024 * 1024), config.getUInt64(config_prefix + ".max_upload_part_size", 5ULL * 1024 * 1024 * 1024), - config.getUInt64(config_prefix + ".max_part_number", 10000) + config.getUInt64(config_prefix + ".max_part_number", 10000), + config.getBool(config_prefix + ".use_native_copy", false) ); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 1ff4537742f..436b48c0ad4 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -27,7 +27,8 @@ struct AzureObjectStorageSettings int list_object_keys_size_, size_t min_upload_part_size_, size_t max_upload_part_size_, - size_t max_part_number_) + size_t max_part_number_, + bool use_native_copy_) : max_single_part_upload_size(max_single_part_upload_size_) , min_bytes_for_seek(min_bytes_for_seek_) , max_single_read_retries(max_single_read_retries_) @@ -36,6 +37,7 @@ struct AzureObjectStorageSettings , min_upload_part_size(min_upload_part_size_) , max_upload_part_size(max_upload_part_size_) , max_part_number(max_part_number_) + , use_native_copy(use_native_copy_) { } @@ -49,6 +51,7 @@ struct AzureObjectStorageSettings size_t min_upload_part_size = 16 * 1024 * 1024; size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; size_t max_part_number = 10000; + bool use_native_copy = false; }; using AzureClient = Azure::Storage::Blobs::BlobContainerClient; @@ -134,10 +137,7 @@ public: bool isRemote() const override { return true; } - MultiVersion & getClient() - { - return client; - } + MultiVersion & getClient() { return client; } private: const String name; diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 4ec90d2830e..9db5ddb476a 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -30,6 +30,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int INVALID_CONFIG_PARAMETER; + extern const int AZURE_BLOB_STORAGE_ERROR; } @@ -358,15 +359,34 @@ void copyAzureBlobStorageFile( bool for_disk_azure_blob_storage) { - if (size < max_single_operation_copy_size) + if (settings->use_native_copy ) { ProfileEvents::increment(ProfileEvents::AzureCopyObject); if (for_disk_azure_blob_storage) ProfileEvents::increment(ProfileEvents::DiskAzureCopyObject); + auto block_blob_client_src = src_client.get()->GetBlockBlobClient(src_blob); auto block_blob_client_dest = dest_client.get()->GetBlockBlobClient(dest_blob); - auto uri = block_blob_client_src.GetUrl(); - block_blob_client_dest.CopyFromUri(uri); + auto source_uri = block_blob_client_src.GetUrl(); + + if (size < max_single_operation_copy_size) + { + block_blob_client_dest.CopyFromUri(source_uri); + } + else + { + Azure::Storage::Blobs::StartBlobCopyOperation operation = block_blob_client_dest.StartCopyFromUri(source_uri); + + // Wait for the operation to finish, checking for status every 100 second. + auto copy_response = operation.PollUntilDone(std::chrono::milliseconds(100)); + auto properties_model = copy_response.Value; + + if (properties_model.CopySource.HasValue()) + { + throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Copy failed"); + } + + } } else { From 2ee68933123583fe585093868e65c3562d36d66a Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 5 Jan 2024 10:58:04 +0100 Subject: [PATCH 019/103] Updated to return container for getObjectsNamespace --- src/Backups/BackupIO_AzureBlobStorage.cpp | 6 ++++-- .../ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp | 7 +++++-- .../ObjectStorages/AzureBlobStorage/AzureObjectStorage.h | 6 ++++-- .../AzureBlobStorage/registerDiskAzureBlobStorage.cpp | 4 +++- src/Storages/StorageAzureBlob.cpp | 2 +- src/TableFunctions/TableFunctionAzureBlobStorage.cpp | 4 ++-- .../TableFunctionAzureBlobStorageCluster.cpp | 4 ++-- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 968a60c566f..5ddbb42e2c0 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -41,7 +41,8 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( auto settings_as_unique_ptr = StorageAzureBlob::createSettings(context_); object_storage = std::make_unique("BackupReaderAzureBlobStorage", std::move(client_ptr), - std::move(settings_as_unique_ptr)); + std::move(settings_as_unique_ptr), + configuration_.container); client = object_storage->getClient(); } @@ -153,7 +154,8 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( auto settings_as_unique_ptr = StorageAzureBlob::createSettings(context_); object_storage = std::make_unique("BackupWriterAzureBlobStorage", std::move(client_ptr), - std::move(settings_as_unique_ptr)); + std::move(settings_as_unique_ptr), + configuration_.container); client = object_storage->getClient(); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 068e2aebab1..1f92ef48350 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -92,10 +92,12 @@ private: AzureObjectStorage::AzureObjectStorage( const String & name_, AzureClientPtr && client_, - SettingsPtr && settings_) + SettingsPtr && settings_, + const String & container_) : name(name_) , client(std::move(client_)) , settings(std::move(settings_)) + , container(container_) , log(&Poco::Logger::get("AzureObjectStorage")) { data_source_description.type = DataSourceType::AzureBlobStorage; @@ -379,7 +381,8 @@ std::unique_ptr AzureObjectStorage::cloneObjectStorage(const std return std::make_unique( name, getAzureBlobContainerClient(config, config_prefix), - getAzureBlobStorageSettings(config, config_prefix, context) + getAzureBlobStorageSettings(config, config_prefix, context), + container ); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 436b48c0ad4..660d4a30889 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -66,7 +66,8 @@ public: AzureObjectStorage( const String & name_, AzureClientPtr && client_, - SettingsPtr && settings_); + SettingsPtr && settings_, + const String & container_); void listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const override; @@ -125,7 +126,7 @@ public: const std::string & config_prefix, ContextPtr context) override; - String getObjectsNamespace() const override { return ""; } + String getObjectsNamespace() const override { return container ; } std::unique_ptr cloneObjectStorage( const std::string & new_namespace, @@ -144,6 +145,7 @@ private: /// client used to access the files in the Blob Storage cloud MultiVersion client; MultiVersion settings; + const String container; Poco::Logger * log; diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp index 7ba9d21db62..2ffd910f92a 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp @@ -26,10 +26,12 @@ void registerDiskAzureBlobStorage(DiskFactory & factory, bool global_skip_access { auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context); + String container_name = config.getString(config_prefix + ".container_name", "default-container"); ObjectStoragePtr azure_object_storage = std::make_unique( name, getAzureBlobContainerClient(config, config_prefix), - getAzureBlobStorageSettings(config, config_prefix, context)); + getAzureBlobStorageSettings(config, config_prefix, context), + container_name); String key_prefix; auto metadata_storage = std::make_shared(metadata_disk, key_prefix); diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp index f1070c8c31e..fcd7074b9d2 100644 --- a/src/Storages/StorageAzureBlob.cpp +++ b/src/Storages/StorageAzureBlob.cpp @@ -314,7 +314,7 @@ void registerStorageAzureBlob(StorageFactory & factory) return std::make_shared( std::move(configuration), - std::make_unique("AzureBlobStorage", std::move(client), std::move(settings)), + std::make_unique("AzureBlobStorage", std::move(client), std::move(settings),configuration.container), args.getContext(), args.table_id, args.columns, diff --git a/src/TableFunctions/TableFunctionAzureBlobStorage.cpp b/src/TableFunctions/TableFunctionAzureBlobStorage.cpp index d394c836369..b098cac5144 100644 --- a/src/TableFunctions/TableFunctionAzureBlobStorage.cpp +++ b/src/TableFunctions/TableFunctionAzureBlobStorage.cpp @@ -262,7 +262,7 @@ ColumnsDescription TableFunctionAzureBlobStorage::getActualTableStructure(Contex auto client = StorageAzureBlob::createClient(configuration, !is_insert_query); auto settings = StorageAzureBlob::createSettings(context); - auto object_storage = std::make_unique("AzureBlobStorageTableFunction", std::move(client), std::move(settings)); + auto object_storage = std::make_unique("AzureBlobStorageTableFunction", std::move(client), std::move(settings), configuration.container); return StorageAzureBlob::getTableStructureFromData(object_storage.get(), configuration, std::nullopt, context, false); } @@ -293,7 +293,7 @@ StoragePtr TableFunctionAzureBlobStorage::executeImpl(const ASTPtr & /*ast_funct StoragePtr storage = std::make_shared( configuration, - std::make_unique(table_name, std::move(client), std::move(settings)), + std::make_unique(table_name, std::move(client), std::move(settings), configuration.container), context, StorageID(getDatabaseName(), table_name), columns, diff --git a/src/TableFunctions/TableFunctionAzureBlobStorageCluster.cpp b/src/TableFunctions/TableFunctionAzureBlobStorageCluster.cpp index eee585967c2..1c3b302a186 100644 --- a/src/TableFunctions/TableFunctionAzureBlobStorageCluster.cpp +++ b/src/TableFunctions/TableFunctionAzureBlobStorageCluster.cpp @@ -40,7 +40,7 @@ StoragePtr TableFunctionAzureBlobStorageCluster::executeImpl( /// On worker node this filename won't contains globs storage = std::make_shared( configuration, - std::make_unique(table_name, std::move(client), std::move(settings)), + std::make_unique(table_name, std::move(client), std::move(settings), configuration.container), context, StorageID(getDatabaseName(), table_name), columns, @@ -55,7 +55,7 @@ StoragePtr TableFunctionAzureBlobStorageCluster::executeImpl( storage = std::make_shared( cluster_name, configuration, - std::make_unique(table_name, std::move(client), std::move(settings)), + std::make_unique(table_name, std::move(client), std::move(settings), configuration.container), StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, From b250acff789620be57e21977d8f3d4a3468070d5 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 5 Jan 2024 11:26:32 +0100 Subject: [PATCH 020/103] Fixed style check --- src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 9db5ddb476a..3399f1705f4 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -359,7 +359,7 @@ void copyAzureBlobStorageFile( bool for_disk_azure_blob_storage) { - if (settings->use_native_copy ) + if (settings->use_native_copy) { ProfileEvents::increment(ProfileEvents::AzureCopyObject); if (for_disk_azure_blob_storage) From fd92c1961e5f09411d83b21c4fe9f00b78be22ba Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sun, 7 Jan 2024 16:33:48 +0100 Subject: [PATCH 021/103] Fix clang tidy build --- src/Backups/BackupIO_AzureBlobStorage.cpp | 12 ++++++------ src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp | 10 +++++----- src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 5ddbb42e2c0..8c6c1040eec 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -305,21 +305,21 @@ void BackupWriterAzureBlobStorage::removeFile(const String & file_name) object_storage->removeObjectIfExists(object); } -void BackupWriterAzureBlobStorage::removeFiles(const Strings & keys) +void BackupWriterAzureBlobStorage::removeFiles(const Strings & file_names) { StoredObjects objects; - for (const auto & key : keys) - objects.emplace_back(key); + for (const auto & file_name : file_names) + objects.emplace_back(file_name); object_storage->removeObjectsIfExist(objects); } -void BackupWriterAzureBlobStorage::removeFilesBatch(const Strings & keys) +void BackupWriterAzureBlobStorage::removeFilesBatch(const Strings & file_names) { StoredObjects objects; - for (const auto & key : keys) - objects.emplace_back(key); + for (const auto & file_name : file_names) + objects.emplace_back(file_name); object_storage->removeObjectsIfExist(objects); } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 3399f1705f4..272be914cc1 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -65,11 +65,11 @@ namespace , schedule(schedule_) , for_disk_azure_blob_storage(for_disk_azure_blob_storage_) , log(log_) - , max_single_part_upload_size(settings_.get()->max_single_part_upload_size) + , max_single_part_upload_size(settings_->max_single_part_upload_size) { } - ~UploadHelper() {} + virtual ~UploadHelper() = default; protected: std::function()> create_read_buffer; @@ -114,9 +114,9 @@ namespace if (!total_size) throw Exception(ErrorCodes::LOGICAL_ERROR, "Chosen multipart upload for an empty file. This must not happen"); - auto max_part_number = settings.get()->max_part_number; - auto min_upload_part_size = settings.get()->min_upload_part_size; - auto max_upload_part_size = settings.get()->max_upload_part_size; + auto max_part_number = settings->max_part_number; + auto min_upload_part_size = settings->min_upload_part_size; + auto max_upload_part_size = settings->max_upload_part_size; if (!max_part_number) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_part_number must not be 0"); diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h index a6502541db1..b022151d32d 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h @@ -24,11 +24,11 @@ void copyAzureBlobStorageFile( MultiVersion & src_client, MultiVersion & dest_client, const String & src_container, - const String & src_path, + const String & src_blob, size_t src_offset, size_t src_size, const String & dest_container, - const String & dest_path, + const String & dest_blob, std::shared_ptr settings, const ReadSettings & read_settings, const std::optional> & object_metadata = std::nullopt, @@ -47,7 +47,7 @@ void copyDataToAzureBlobStorageFile( size_t size, MultiVersion & client, const String & dest_container, - const String & dest_bucket, + const String & dest_blob, std::shared_ptr settings, const std::optional> & object_metadata = std::nullopt, ThreadPoolCallbackRunner schedule_ = {}, From f50f7f56949021d01ba692f6788e50d411ca8af9 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 8 Jan 2024 14:25:33 +0100 Subject: [PATCH 022/103] Removed unwanted includes --- .../registerBackupEngineAzureBlobStorage.cpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/Backups/registerBackupEngineAzureBlobStorage.cpp b/src/Backups/registerBackupEngineAzureBlobStorage.cpp index ef95206831f..810da5adb3f 100644 --- a/src/Backups/registerBackupEngineAzureBlobStorage.cpp +++ b/src/Backups/registerBackupEngineAzureBlobStorage.cpp @@ -10,13 +10,11 @@ #include #include #include -#include #endif namespace DB { -namespace fs = std::filesystem; namespace ErrorCodes { @@ -25,23 +23,6 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -#if USE_AZURE_BLOB_STORAGE -namespace -{ - String removeFileNameFromURL(String & url) - { - Poco::URI url2{url}; - String path = url2.getPath(); - size_t slash_pos = path.find_last_of('/'); - String file_name = path.substr(slash_pos + 1); - path.resize(slash_pos + 1); - url2.setPath(path); - url = url2.toString(); - return file_name; - } -} -#endif - void registerBackupEngineAzureBlobStorage(BackupFactory & factory) { From 2d914721e5101215c2c63c97151552cb7c8ff746 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 8 Jan 2024 15:10:37 +0100 Subject: [PATCH 023/103] Fix build --- .../registerBackupEngineAzureBlobStorage.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/Backups/registerBackupEngineAzureBlobStorage.cpp b/src/Backups/registerBackupEngineAzureBlobStorage.cpp index 810da5adb3f..3480ea75f1f 100644 --- a/src/Backups/registerBackupEngineAzureBlobStorage.cpp +++ b/src/Backups/registerBackupEngineAzureBlobStorage.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #endif @@ -23,6 +24,22 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } +#if USE_AZURE_BLOB_STORAGE +namespace +{ + String removeFileNameFromURL(String & url) + { + Poco::URI url2{url}; + String path = url2.getPath(); + size_t slash_pos = path.find_last_of('/'); + String file_name = path.substr(slash_pos + 1); + path.resize(slash_pos + 1); + url2.setPath(path); + url = url2.toString(); + return file_name; + } +} +#endif void registerBackupEngineAzureBlobStorage(BackupFactory & factory) { From ffde721f08359e0437c44026881e2514012a4966 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jan 2024 23:09:10 +0300 Subject: [PATCH 024/103] Update 02932_set_ttl_where.sql --- tests/queries/0_stateless/02932_set_ttl_where.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/02932_set_ttl_where.sql b/tests/queries/0_stateless/02932_set_ttl_where.sql index bf2b317c4bf..ee8473e1af2 100644 --- a/tests/queries/0_stateless/02932_set_ttl_where.sql +++ b/tests/queries/0_stateless/02932_set_ttl_where.sql @@ -1,3 +1,5 @@ +-- Tags: no-ordinary-database + create or replace table t_temp ( a UInt32, timestamp DateTime @@ -12,3 +14,5 @@ select sleep(1); insert into t_temp select rand(), now() from system.numbers limit 1_000_000; select sleep(1); optimize table t_temp final; + +DROP TABLE t_temp; From 12585ea0e4cae1771ee6b51dd85a309e5923f12c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jan 2024 23:10:27 +0300 Subject: [PATCH 025/103] Update TTLDescription.cpp --- src/Storages/TTLDescription.cpp | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index bfd3afc30d8..3db5269b617 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -117,11 +117,6 @@ TTLDescription::TTLDescription(const TTLDescription & other) , if_exists(other.if_exists) , recompression_codec(other.recompression_codec) { - // if (other.expression) - // expression = other.expression->clone(); - - // if (other.where_expression) - // where_expression = other.where_expression->clone(); } TTLDescription & TTLDescription::operator=(const TTLDescription & other) @@ -135,11 +130,6 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) else expression_ast.reset(); - // if (other.expression) - // expression = other.expression->clone(); - // else - // expression.reset(); - expression_columns = other.expression_columns; result_column = other.result_column; @@ -148,11 +138,6 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) else where_expression_ast.reset(); - // if (other.where_expression) - // where_expression = other.where_expression->clone(); - // else - // where_expression.reset(); - where_expression_columns = other.where_expression_columns; where_result_column = other.where_result_column; group_by_keys = other.group_by_keys; @@ -179,7 +164,6 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType auto dag = analyzer.getActionsDAG(false); const auto * col = &dag->findInOutputs(ast->getColumnName()); - // std::cerr << "buildExpressionAndSets " << ttl_string << std::endl; if (col->result_name != ttl_string) col = &dag->addAlias(*col, ttl_string); @@ -189,10 +173,6 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType result.expression = std::make_shared(dag, ExpressionActionsSettings::fromContext(context)); result.sets = analyzer.getPreparedSets(); - // std::cerr << "--------- buildExpressionAndSets\n"; - // std::cerr << result.expression->dumpActions() << std::endl; - // std::cerr << result.sets->getSubqueries().size() << std::endl; - return result; } @@ -232,8 +212,6 @@ TTLDescription TTLDescription::getTTLFromAST( auto expression = buildExpressionAndSets(ttl_ast, columns.getAllPhysical(), context).expression; result.expression_columns = expression->getRequiredColumnsWithTypes(); - // auto syntax_analyzer_result = TreeRewriter(context).analyze(ttl_ast, columns.getAllPhysical()); - // result.expression = ExpressionAnalyzer(ttl_ast, syntax_analyzer_result, context).getActions(false); result.result_column = expression->getSampleBlock().safeGetByPosition(0).name; ExpressionActionsPtr where_expression; @@ -256,9 +234,6 @@ TTLDescription TTLDescription::getTTLFromAST( { result.where_expression_ast = where_expr_ast->clone(); where_expression = buildExpressionAndSets(where_expr_ast, columns.getAllPhysical(), context).expression; - // auto where_syntax_result = TreeRewriter(context).analyze(where_expr_ast, columns.getAllPhysical()); - // result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, context).getActions(false); - result.where_expression_columns = where_expression->getRequiredColumnsWithTypes(); result.where_result_column = where_expression->getSampleBlock().safeGetByPosition(0).name; } From 776ea26ce71287735897b00c65b47d73e8d9811c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jan 2024 02:45:51 +0300 Subject: [PATCH 026/103] Update PreparedSets.h --- src/Interpreters/PreparedSets.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index 30bfda4700d..4f5ca337c5b 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -115,7 +115,6 @@ public: SetPtr buildSetInplace(const ContextPtr & context); std::unique_ptr build(const ContextPtr & context); - void buildSetInplace(const ContextPtr & context); QueryTreeNodePtr detachQueryTree() { return std::move(query_tree); } void setQueryPlan(std::unique_ptr source_); From d2c671c17eb4a85583b30d81033f7180ea93f627 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 19 Jan 2024 20:38:08 +0000 Subject: [PATCH 027/103] 1st attempt at vectorization 80 mio arrays with 150 elements each, runtimes in sec WITH (SELECT vec FROM vectors limit 1) AS const_vec SELECT sum(dist) FROM (SELECT (const_vec, vec) AS dist FROM vectors) auto-vectorized hand-vectorized L2 Float32 0.61 0.57 L2 Float64 1.15 0.99 cos Float32 0.78 0.65 cos Float64 1.35 1.05 --- src/Functions/array/arrayDistance.cpp | 145 +++++++++++++++++- .../02282_array_distance.reference | 4 + .../0_stateless/02282_array_distance.sql | 40 +++-- 3 files changed, 172 insertions(+), 17 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index c68c89ee0d5..670442c0c79 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -9,6 +10,10 @@ #include #include +#if USE_MULTITARGET_CODE +#include +#endif + namespace DB { namespace ErrorCodes @@ -75,6 +80,49 @@ struct L2Distance state.sum += other_state.sum; } +#if USE_MULTITARGET_CODE + template + AVX512_FUNCTION_SPECIFIC_ATTRIBUTE static void accumulateCombine( + const ResultType * __restrict data_x, + const ResultType * __restrict data_y, + size_t i_max, + size_t & i_x, + size_t & i_y, + State & state) + { + __m512 sums; + if constexpr (std::is_same_v) + sums = _mm512_setzero_ps(); + else + sums = _mm512_setzero_pd(); + + const size_t n = (std::is_same_v) ? 16 : 8; + + for (; i_x + n < i_max; i_x += n, i_y += n) + { + if constexpr (std::is_same_v) + { + __m512 x = _mm512_loadu_ps(data_x + i_x); + __m512 y = _mm512_loadu_ps(data_y + i_y); + __m512 differences = _mm512_sub_ps(x, y); + sums = _mm512_fmadd_ps(differences, differences, sums); + } + else + { + __m512 x = _mm512_loadu_pd(data_x + i_x); + __m512 y = _mm512_loadu_pd(data_y + i_y); + __m512 differences = _mm512_sub_pd(x, y); + sums = _mm512_fmadd_pd(differences, differences, sums); + } + } + + if constexpr (std::is_same_v) + state.sum = _mm512_reduce_add_ps(sums); + else + state.sum = _mm512_reduce_add_pd(sums); + } +#endif + template static ResultType finalize(const State & state, const ConstParams &) { @@ -189,6 +237,70 @@ struct CosineDistance state.y_squared += other_state.y_squared; } +#if USE_MULTITARGET_CODE + template + AVX512_FUNCTION_SPECIFIC_ATTRIBUTE static void accumulateCombine( + const ResultType * __restrict data_x, + const ResultType * __restrict data_y, + size_t i_max, + size_t & i_x, + size_t & i_y, + State & state) + { + __m512 dot_products; + __m512 x_squareds; + __m512 y_squareds; + + if constexpr (std::is_same_v) + { + dot_products = _mm512_setzero_ps(); + x_squareds = _mm512_setzero_ps(); + y_squareds = _mm512_setzero_ps(); + } + else + { + dot_products = _mm512_setzero_pd(); + x_squareds = _mm512_setzero_pd(); + y_squareds = _mm512_setzero_pd(); + } + + const size_t n = (std::is_same_v) ? 16 : 8; + + for (; i_x + n < i_max; i_x += n, i_y += n) + { + if constexpr (std::is_same_v) + { + __m512 x = _mm512_loadu_ps(data_x + i_x); + __m512 y = _mm512_loadu_ps(data_y + i_y); + dot_products = _mm512_fmadd_ps(x, y, dot_products); + x_squareds = _mm512_fmadd_ps(x, x, x_squareds); + y_squareds = _mm512_fmadd_ps(y, y, y_squareds); + } + else + { + __m512 x = _mm512_loadu_pd(data_x + i_x); + __m512 y = _mm512_loadu_pd(data_y + i_y); + dot_products = _mm512_fmadd_pd(x, y, dot_products); + x_squareds = _mm512_fmadd_pd(x, x, x_squareds); + y_squareds = _mm512_fmadd_pd(y, y, y_squareds); + } + } + + if constexpr (std::is_same_v) + { + state.dot_prod = _mm512_reduce_add_ps(dot_products); + state.x_squared = _mm512_reduce_add_ps(x_squareds); + state.y_squared = _mm512_reduce_add_ps(y_squareds); + } + else + { + state.dot_prod = _mm512_reduce_add_pd(dot_products); + state.x_squared = _mm512_reduce_add_pd(x_squareds); + state.y_squared = _mm512_reduce_add_pd(y_squareds); + } + } +#endif + template static ResultType finalize(const State & state, const ConstParams &) { @@ -352,7 +464,7 @@ private: /// Check that arrays in both columns are the sames size for (size_t row = 0; row < offsets_x.size(); ++row) { - if (unlikely(offsets_x[row] != offsets_y[row])) + if (offsets_x[row] != offsets_y[row]) [[unlikely]] { ColumnArray::Offset prev_offset = row > 0 ? offsets_x[row] : 0; throw Exception( @@ -420,7 +532,7 @@ private: ColumnArray::Offset prev_offset = 0; for (size_t row : collections::range(0, offsets_y.size())) { - if (unlikely(offsets_x[0] != offsets_y[row] - prev_offset)) + if (offsets_x[0] != offsets_y[row] - prev_offset) [[unlikely]] { throw Exception( ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, @@ -438,14 +550,35 @@ private: auto & result_data = result->getData(); /// Do the actual computation - ColumnArray::Offset prev = 0; + size_t prev = 0; size_t row = 0; + for (auto off : offsets_y) { + size_t i = 0; + typename Kernel::template State state; + + /// SIMD optimization: process multiple elements in both input arrays at once. + /// To avoid combinatorial explosion of SIMD kernels, focus on + /// - the two most common input/output types (Float32 x Float32) --> Float32 and (Float64 x Float64) --> Float64 instead of 10 x + /// 10 input types x 2 output types, + /// - const/non-const inputs instead of non-const/non-const inputs + /// - the two most common metrics L2 and cosine distance, + /// - the most powerful SIMD instruction set (AVX-512F). +#if USE_MULTITARGET_CODE + if constexpr (std::is_same_v && std::is_same_v) /// ResultType is Float32 or Float64 + { + if constexpr (std::is_same_v + || std::is_same_v) + { + if (isArchSupported(TargetArch::AVX512F)) + Kernel::template accumulateCombine(data_x.data(), data_y.data(), i + offsets_x[0], i, prev, state); + } + } +#else /// Process chunks in vectorized manner static constexpr size_t VEC_SIZE = 4; typename Kernel::template State states[VEC_SIZE]; - size_t i = 0; for (; prev + VEC_SIZE < off; i += VEC_SIZE, prev += VEC_SIZE) { for (size_t s = 0; s < VEC_SIZE; ++s) @@ -453,10 +586,9 @@ private: states[s], static_cast(data_x[i + s]), static_cast(data_y[prev + s]), kernel_params); } - typename Kernel::template State state; for (const auto & other_state : states) Kernel::template combine(state, other_state, kernel_params); - +#endif /// Process the tail for (; prev < off; ++i, ++prev) { @@ -466,6 +598,7 @@ private: result_data[row] = Kernel::finalize(state, kernel_params); row++; } + return result; } diff --git a/tests/queries/0_stateless/02282_array_distance.reference b/tests/queries/0_stateless/02282_array_distance.reference index 9758da9a833..c21e294cb62 100644 --- a/tests/queries/0_stateless/02282_array_distance.reference +++ b/tests/queries/0_stateless/02282_array_distance.reference @@ -80,3 +80,7 @@ nan 5 6 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 6 5 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 6 6 0 0 0 0 0 0 +5.8309517 +0.0003244877 +5.830951894845301 +0.0003245172890904424 diff --git a/tests/queries/0_stateless/02282_array_distance.sql b/tests/queries/0_stateless/02282_array_distance.sql index 9c16071dc1f..2cca853fd67 100644 --- a/tests/queries/0_stateless/02282_array_distance.sql +++ b/tests/queries/0_stateless/02282_array_distance.sql @@ -12,10 +12,10 @@ SELECT cosineDistance([1, 2, 3], [0, 0, 0]); -- Overflows WITH CAST([-547274980, 1790553898, 1981517754, 1908431500, 1352428565, -573412550, -552499284, 2096941042], 'Array(Int32)') AS a SELECT - L1Distance(a,a), - L2Distance(a,a), - L2SquaredDistance(a,a), - LinfDistance(a,a), + L1Distance(a, a), + L2Distance(a, a), + L2SquaredDistance(a, a), + LinfDistance(a, a), cosineDistance(a, a); DROP TABLE IF EXISTS vec1; @@ -88,15 +88,33 @@ SELECT FROM vec2f v1, vec2d v2 WHERE length(v1.v) == length(v2.v); -SELECT L1Distance([0, 0], [1]); -- { serverError 190 } -SELECT L2Distance([1, 2], (3,4)); -- { serverError 43 } -SELECT L2SquaredDistance([1, 2], (3,4)); -- { serverError 43 } -SELECT LpDistance([1, 2], [3,4]); -- { serverError 42 } -SELECT LpDistance([1, 2], [3,4], -1.); -- { serverError 69 } -SELECT LpDistance([1, 2], [3,4], 'aaa'); -- { serverError 43 } -SELECT LpDistance([1, 2], [3,4], materialize(2.7)); -- { serverError 44 } +SELECT L1Distance([0, 0], [1]); -- { serverError SIZES_OF_ARRAYS_DONT_MATCH } +SELECT L2Distance([1, 2], (3,4)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT L2SquaredDistance([1, 2], (3,4)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT LpDistance([1, 2], [3,4]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT LpDistance([1, 2], [3,4], -1.); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT LpDistance([1, 2], [3,4], 'aaa'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT LpDistance([1, 2], [3,4], materialize(2.7)); -- { serverError ILLEGAL_COLUMN } DROP TABLE vec1; DROP TABLE vec2; DROP TABLE vec2f; DROP TABLE vec2d; + +-- Queries which trigger manually vectorized implementation + +SELECT L2Distance( + [toFloat32(0.0), toFloat32(1.0), toFloat32(2.0), toFloat32(3.0), toFloat32(4.0), toFloat32(5.0), toFloat32(6.0), toFloat32(7.0), toFloat32(8.0), toFloat32(9.0), toFloat32(10.0), toFloat32(11.0), toFloat32(12.0), toFloat32(13.0), toFloat32(14.0), toFloat32(15.0), toFloat32(16.0), toFloat32(17.0), toFloat32(18.0), toFloat32(19.0), toFloat32(20.0), toFloat32(21.0), toFloat32(22.0), toFloat32(23.0), toFloat32(24.0), toFloat32(25.0), toFloat32(26.0), toFloat32(27.0), toFloat32(28.0), toFloat32(29.0), toFloat32(30.0), toFloat32(31.0), toFloat32(32.0), toFloat32(33.0)], + materialize([toFloat32(1.0), toFloat32(2.0), toFloat32(3.0), toFloat32(4.0), toFloat32(5.0), toFloat32(6.0), toFloat32(7.0), toFloat32(8.0), toFloat32(9.0), toFloat32(10.0), toFloat32(11.0), toFloat32(12.0), toFloat32(13.0), toFloat32(14.0), toFloat32(15.0), toFloat32(16.0), toFloat32(17.0), toFloat32(18.0), toFloat32(19.0), toFloat32(20.0), toFloat32(21.0), toFloat32(22.0), toFloat32(23.0), toFloat32(24.0), toFloat32(25.0), toFloat32(26.0), toFloat32(27.0), toFloat32(28.0), toFloat32(29.0), toFloat32(30.0), toFloat32(31.0), toFloat32(32.0), toFloat32(33.0), toFloat32(34.0)])); + +SELECT cosineDistance( + [toFloat32(0.0), toFloat32(1.0), toFloat32(2.0), toFloat32(3.0), toFloat32(4.0), toFloat32(5.0), toFloat32(6.0), toFloat32(7.0), toFloat32(8.0), toFloat32(9.0), toFloat32(10.0), toFloat32(11.0), toFloat32(12.0), toFloat32(13.0), toFloat32(14.0), toFloat32(15.0), toFloat32(16.0), toFloat32(17.0), toFloat32(18.0), toFloat32(19.0), toFloat32(20.0), toFloat32(21.0), toFloat32(22.0), toFloat32(23.0), toFloat32(24.0), toFloat32(25.0), toFloat32(26.0), toFloat32(27.0), toFloat32(28.0), toFloat32(29.0), toFloat32(30.0), toFloat32(31.0), toFloat32(32.0), toFloat32(33.0)], + materialize([toFloat32(1.0), toFloat32(2.0), toFloat32(3.0), toFloat32(4.0), toFloat32(5.0), toFloat32(6.0), toFloat32(7.0), toFloat32(8.0), toFloat32(9.0), toFloat32(10.0), toFloat32(11.0), toFloat32(12.0), toFloat32(13.0), toFloat32(14.0), toFloat32(15.0), toFloat32(16.0), toFloat32(17.0), toFloat32(18.0), toFloat32(19.0), toFloat32(20.0), toFloat32(21.0), toFloat32(22.0), toFloat32(23.0), toFloat32(24.0), toFloat32(25.0), toFloat32(26.0), toFloat32(27.0), toFloat32(28.0), toFloat32(29.0), toFloat32(30.0), toFloat32(31.0), toFloat32(32.0), toFloat32(33.0), toFloat32(34.0)])); + +SELECT L2Distance( + [toFloat64(0.0), toFloat64(1.0), toFloat64(2.0), toFloat64(3.0), toFloat64(4.0), toFloat64(5.0), toFloat64(6.0), toFloat64(7.0), toFloat64(8.0), toFloat64(9.0), toFloat64(10.0), toFloat64(11.0), toFloat64(12.0), toFloat64(13.0), toFloat64(14.0), toFloat64(15.0), toFloat64(16.0), toFloat64(17.0), toFloat64(18.0), toFloat64(19.0), toFloat64(20.0), toFloat64(21.0), toFloat64(22.0), toFloat64(23.0), toFloat64(24.0), toFloat64(25.0), toFloat64(26.0), toFloat64(27.0), toFloat64(28.0), toFloat64(29.0), toFloat64(30.0), toFloat64(31.0), toFloat64(32.0), toFloat64(33.0)], + materialize([toFloat64(1.0), toFloat64(2.0), toFloat64(3.0), toFloat64(4.0), toFloat64(5.0), toFloat64(6.0), toFloat64(7.0), toFloat64(8.0), toFloat64(9.0), toFloat64(10.0), toFloat64(11.0), toFloat64(12.0), toFloat64(13.0), toFloat64(14.0), toFloat64(15.0), toFloat64(16.0), toFloat64(17.0), toFloat64(18.0), toFloat64(19.0), toFloat64(20.0), toFloat64(21.0), toFloat64(22.0), toFloat64(23.0), toFloat64(24.0), toFloat64(25.0), toFloat64(26.0), toFloat64(27.0), toFloat64(28.0), toFloat64(29.0), toFloat64(30.0), toFloat64(31.0), toFloat64(32.0), toFloat64(33.0), toFloat64(34.0)])); + +SELECT cosineDistance( + [toFloat64(0.0), toFloat64(1.0), toFloat64(2.0), toFloat64(3.0), toFloat64(4.0), toFloat64(5.0), toFloat64(6.0), toFloat64(7.0), toFloat64(8.0), toFloat64(9.0), toFloat64(10.0), toFloat64(11.0), toFloat64(12.0), toFloat64(13.0), toFloat64(14.0), toFloat64(15.0), toFloat64(16.0), toFloat64(17.0), toFloat64(18.0), toFloat64(19.0), toFloat64(20.0), toFloat64(21.0), toFloat64(22.0), toFloat64(23.0), toFloat64(24.0), toFloat64(25.0), toFloat64(26.0), toFloat64(27.0), toFloat64(28.0), toFloat64(29.0), toFloat64(30.0), toFloat64(31.0), toFloat64(32.0), toFloat64(33.0)], + materialize([toFloat64(1.0), toFloat64(2.0), toFloat64(3.0), toFloat64(4.0), toFloat64(5.0), toFloat64(6.0), toFloat64(7.0), toFloat64(8.0), toFloat64(9.0), toFloat64(10.0), toFloat64(11.0), toFloat64(12.0), toFloat64(13.0), toFloat64(14.0), toFloat64(15.0), toFloat64(16.0), toFloat64(17.0), toFloat64(18.0), toFloat64(19.0), toFloat64(20.0), toFloat64(21.0), toFloat64(22.0), toFloat64(23.0), toFloat64(24.0), toFloat64(25.0), toFloat64(26.0), toFloat64(27.0), toFloat64(28.0), toFloat64(29.0), toFloat64(30.0), toFloat64(31.0), toFloat64(32.0), toFloat64(33.0), toFloat64(34.0)])); From 68d0f4e42161713f3b54de2069d894b1f84ed833 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 20 Jan 2024 21:36:25 +0000 Subject: [PATCH 028/103] (Futile) unrolling attempt at vectorization --- src/Functions/array/arrayDistance.cpp | 88 ++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 670442c0c79..aa13ee01d9a 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -90,36 +90,92 @@ struct L2Distance size_t & i_y, State & state) { - __m512 sums; - if constexpr (std::is_same_v) - sums = _mm512_setzero_ps(); - else - sums = _mm512_setzero_pd(); + __m512 sums1; + __m512 sums2; + __m512 sums3; + __m512 sums4; - const size_t n = (std::is_same_v) ? 16 : 8; + if constexpr (std::is_same_v) + { + sums1 = _mm512_setzero_ps(); + sums2 = _mm512_setzero_ps(); + sums3 = _mm512_setzero_ps(); + sums4 = _mm512_setzero_ps(); + } + else + { + sums1 = _mm512_setzero_pd(); + sums2 = _mm512_setzero_pd(); + sums3 = _mm512_setzero_pd(); + sums4 = _mm512_setzero_pd(); + } + + const size_t n = (std::is_same_v) ? 64 : 32; for (; i_x + n < i_max; i_x += n, i_y += n) { if constexpr (std::is_same_v) { - __m512 x = _mm512_loadu_ps(data_x + i_x); - __m512 y = _mm512_loadu_ps(data_y + i_y); - __m512 differences = _mm512_sub_ps(x, y); - sums = _mm512_fmadd_ps(differences, differences, sums); + __m512 x1 = _mm512_loadu_ps(data_x + i_x); + __m512 y1 = _mm512_loadu_ps(data_y + i_y); + __m512 diff1 = _mm512_sub_ps(x1, y1); + sums1 = _mm512_fmadd_ps(diff1, diff1, sums1); + + __m512 x2 = _mm512_loadu_ps(data_x + i_x + 16); + __m512 y2 = _mm512_loadu_ps(data_y + i_y + 16); + __m512 diff2 = _mm512_sub_ps(x2, y2); + sums2 = _mm512_fmadd_ps(diff2, diff2, sums2); + + __m512 x3 = _mm512_loadu_ps(data_x + i_x + 32); + __m512 y3 = _mm512_loadu_ps(data_y + i_y + 32); + __m512 diff3 = _mm512_sub_ps(x3, y3); + sums3 = _mm512_fmadd_ps(diff3, diff3, sums3); + + __m512 x4 = _mm512_loadu_ps(data_x + i_x + 48); + __m512 y4 = _mm512_loadu_ps(data_y + i_y + 48); + __m512 diff4 = _mm512_sub_ps(x4, y4); + sums4 = _mm512_fmadd_ps(diff4, diff4, sums4); } else { - __m512 x = _mm512_loadu_pd(data_x + i_x); - __m512 y = _mm512_loadu_pd(data_y + i_y); - __m512 differences = _mm512_sub_pd(x, y); - sums = _mm512_fmadd_pd(differences, differences, sums); + __m512 x1 = _mm512_loadu_pd(data_x + i_x); + __m512 y1 = _mm512_loadu_pd(data_y + i_y); + __m512 diff1 = _mm512_sub_pd(x1, y1); + sums1 = _mm512_fmadd_pd(diff1, diff1, sums1); + + __m512 x2 = _mm512_loadu_pd(data_x + i_x + 8); + __m512 y2 = _mm512_loadu_pd(data_y + i_y + 8); + __m512 diff2 = _mm512_sub_pd(x2, y2); + sums2 = _mm512_fmadd_pd(diff2, diff2, sums2); + + __m512 x3 = _mm512_loadu_pd(data_x + i_x + 16); + __m512 y3 = _mm512_loadu_pd(data_y + i_y + 16); + __m512 diff3 = _mm512_sub_pd(x3, y3); + sums3 = _mm512_fmadd_pd(diff3, diff3, sums3); + + __m512 x4 = _mm512_loadu_pd(data_x + i_x + 24); + __m512 y4 = _mm512_loadu_pd(data_y + i_y + 24); + __m512 diff4 = _mm512_sub_pd(x4, y4); + sums4 = _mm512_fmadd_pd(diff4, diff4, sums4); } } if constexpr (std::is_same_v) - state.sum = _mm512_reduce_add_ps(sums); + { + Float32 sum1 = _mm512_reduce_add_ps(sums1); + Float32 sum2 = _mm512_reduce_add_ps(sums2); + Float32 sum3 = _mm512_reduce_add_ps(sums3); + Float32 sum4 = _mm512_reduce_add_ps(sums4); + state.sum = sum1 + sum2 + sum3 + sum4; + } else - state.sum = _mm512_reduce_add_pd(sums); + { + Float64 sum1 = _mm512_reduce_add_pd(sums1); + Float64 sum2 = _mm512_reduce_add_pd(sums2); + Float64 sum3 = _mm512_reduce_add_pd(sums3); + Float64 sum4 = _mm512_reduce_add_pd(sums4); + state.sum = sum1 + sum2 + sum3 + sum4; + } } #endif From 68fc97089ec22d29b5d25df4e3865a22cf9701db Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 20 Jan 2024 21:50:13 +0000 Subject: [PATCH 029/103] Revert "(Futile) unrolling attempt at vectorization" This reverts commit df30a990545eafdf5e6a09034d81a97fb0188ba0. --- src/Functions/array/arrayDistance.cpp | 84 +++++---------------------- 1 file changed, 14 insertions(+), 70 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index aa13ee01d9a..670442c0c79 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -90,92 +90,36 @@ struct L2Distance size_t & i_y, State & state) { - __m512 sums1; - __m512 sums2; - __m512 sums3; - __m512 sums4; - + __m512 sums; if constexpr (std::is_same_v) - { - sums1 = _mm512_setzero_ps(); - sums2 = _mm512_setzero_ps(); - sums3 = _mm512_setzero_ps(); - sums4 = _mm512_setzero_ps(); - } + sums = _mm512_setzero_ps(); else - { - sums1 = _mm512_setzero_pd(); - sums2 = _mm512_setzero_pd(); - sums3 = _mm512_setzero_pd(); - sums4 = _mm512_setzero_pd(); - } + sums = _mm512_setzero_pd(); - const size_t n = (std::is_same_v) ? 64 : 32; + const size_t n = (std::is_same_v) ? 16 : 8; for (; i_x + n < i_max; i_x += n, i_y += n) { if constexpr (std::is_same_v) { - __m512 x1 = _mm512_loadu_ps(data_x + i_x); - __m512 y1 = _mm512_loadu_ps(data_y + i_y); - __m512 diff1 = _mm512_sub_ps(x1, y1); - sums1 = _mm512_fmadd_ps(diff1, diff1, sums1); - - __m512 x2 = _mm512_loadu_ps(data_x + i_x + 16); - __m512 y2 = _mm512_loadu_ps(data_y + i_y + 16); - __m512 diff2 = _mm512_sub_ps(x2, y2); - sums2 = _mm512_fmadd_ps(diff2, diff2, sums2); - - __m512 x3 = _mm512_loadu_ps(data_x + i_x + 32); - __m512 y3 = _mm512_loadu_ps(data_y + i_y + 32); - __m512 diff3 = _mm512_sub_ps(x3, y3); - sums3 = _mm512_fmadd_ps(diff3, diff3, sums3); - - __m512 x4 = _mm512_loadu_ps(data_x + i_x + 48); - __m512 y4 = _mm512_loadu_ps(data_y + i_y + 48); - __m512 diff4 = _mm512_sub_ps(x4, y4); - sums4 = _mm512_fmadd_ps(diff4, diff4, sums4); + __m512 x = _mm512_loadu_ps(data_x + i_x); + __m512 y = _mm512_loadu_ps(data_y + i_y); + __m512 differences = _mm512_sub_ps(x, y); + sums = _mm512_fmadd_ps(differences, differences, sums); } else { - __m512 x1 = _mm512_loadu_pd(data_x + i_x); - __m512 y1 = _mm512_loadu_pd(data_y + i_y); - __m512 diff1 = _mm512_sub_pd(x1, y1); - sums1 = _mm512_fmadd_pd(diff1, diff1, sums1); - - __m512 x2 = _mm512_loadu_pd(data_x + i_x + 8); - __m512 y2 = _mm512_loadu_pd(data_y + i_y + 8); - __m512 diff2 = _mm512_sub_pd(x2, y2); - sums2 = _mm512_fmadd_pd(diff2, diff2, sums2); - - __m512 x3 = _mm512_loadu_pd(data_x + i_x + 16); - __m512 y3 = _mm512_loadu_pd(data_y + i_y + 16); - __m512 diff3 = _mm512_sub_pd(x3, y3); - sums3 = _mm512_fmadd_pd(diff3, diff3, sums3); - - __m512 x4 = _mm512_loadu_pd(data_x + i_x + 24); - __m512 y4 = _mm512_loadu_pd(data_y + i_y + 24); - __m512 diff4 = _mm512_sub_pd(x4, y4); - sums4 = _mm512_fmadd_pd(diff4, diff4, sums4); + __m512 x = _mm512_loadu_pd(data_x + i_x); + __m512 y = _mm512_loadu_pd(data_y + i_y); + __m512 differences = _mm512_sub_pd(x, y); + sums = _mm512_fmadd_pd(differences, differences, sums); } } if constexpr (std::is_same_v) - { - Float32 sum1 = _mm512_reduce_add_ps(sums1); - Float32 sum2 = _mm512_reduce_add_ps(sums2); - Float32 sum3 = _mm512_reduce_add_ps(sums3); - Float32 sum4 = _mm512_reduce_add_ps(sums4); - state.sum = sum1 + sum2 + sum3 + sum4; - } + state.sum = _mm512_reduce_add_ps(sums); else - { - Float64 sum1 = _mm512_reduce_add_pd(sums1); - Float64 sum2 = _mm512_reduce_add_pd(sums2); - Float64 sum3 = _mm512_reduce_add_pd(sums3); - Float64 sum4 = _mm512_reduce_add_pd(sums4); - state.sum = sum1 + sum2 + sum3 + sum4; - } + state.sum = _mm512_reduce_add_pd(sums); } #endif From 2e7ce5b0e208c91874d44eb0c828a1e01544a387 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 22 Jan 2024 16:24:43 +0100 Subject: [PATCH 030/103] Updated settings ptr and fetching of client from Disk & ObjectStorage --- src/Backups/BackupIO_AzureBlobStorage.cpp | 32 ++++++++----------- src/Backups/BackupIO_AzureBlobStorage.h | 4 +-- .../AzureBlobStorage/AzureObjectStorage.h | 7 +++- .../Cached/CachedObjectStorage.h | 8 +++++ src/Disks/ObjectStorages/IObjectStorage.h | 13 ++++++++ .../copyAzureBlobStorageFile.cpp | 22 ++++++------- .../copyAzureBlobStorageFile.h | 4 +-- src/Storages/StorageAzureBlob.cpp | 2 +- 8 files changed, 57 insertions(+), 35 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 8c6c1040eec..fca324869ae 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -37,13 +37,12 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( , configuration(configuration_) { auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); - settings = StorageAzureBlob::createSettingsAsSharedPtr(context_); - auto settings_as_unique_ptr = StorageAzureBlob::createSettings(context_); object_storage = std::make_unique("BackupReaderAzureBlobStorage", std::move(client_ptr), - std::move(settings_as_unique_ptr), + StorageAzureBlob::createSettings(context_), configuration_.container); - client = object_storage->getClient(); + client = object_storage->getAzureBlobStorageClient(); + settings = object_storage->getSettings(); } BackupReaderAzureBlobStorage::~BackupReaderAzureBlobStorage() = default; @@ -89,8 +88,8 @@ std::unique_ptr BackupReaderAzureBlobStorage::readFile(const key = file_name; } return std::make_unique( - client.get(), key, read_settings, settings->max_single_read_retries, - settings->max_single_download_retries); + client.get(), key, read_settings, settings.get()->max_single_read_retries, + settings.get()->max_single_download_retries); } void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup, @@ -98,10 +97,8 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, { LOG_INFO(&Poco::Logger::get("BackupReaderAzureBlobStorage"), "Enter copyFileToDisk"); - /// Use the native copy as a more optimal way to copy a file from AzureBlobStorage to AzureBlobStorage if it's possible. - /// We don't check for `has_throttling` here because the native copy almost doesn't use network. auto destination_data_source_description = destination_disk->getDataSourceDescription(); - if (destination_data_source_description.sameKind(data_source_description) + if ((destination_data_source_description.type == DataSourceType::AzureBlobStorage) && (destination_data_source_description.is_encrypted == encrypted_in_backup)) { LOG_TRACE(log, "Copying {} from AzureBlobStorage to disk {}", path_in_backup, destination_disk->getName()); @@ -115,7 +112,7 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, copyAzureBlobStorageFile( client, - reinterpret_cast(destination_disk->getObjectStorage().get())->getClient(), + destination_disk->getObjectStorage()->getAzureBlobStorageClient(), configuration.container, fs::path(configuration.blob_path) / path_in_backup, 0, @@ -150,13 +147,12 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( , configuration(configuration_) { auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); - settings = StorageAzureBlob::createSettingsAsSharedPtr(context_); - auto settings_as_unique_ptr = StorageAzureBlob::createSettings(context_); object_storage = std::make_unique("BackupWriterAzureBlobStorage", std::move(client_ptr), - std::move(settings_as_unique_ptr), + StorageAzureBlob::createSettings(context_), configuration_.container); - client = object_storage->getClient(); + client = object_storage->getAzureBlobStorageClient(); + settings = object_storage->getSettings(); } void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, @@ -172,7 +168,7 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu { LOG_TRACE(log, "Copying file {} from disk {} to AzureBlobStorag", src_path, src_disk->getName()); copyAzureBlobStorageFile( - reinterpret_cast(src_disk->getObjectStorage().get())->getClient(), + src_disk->getObjectStorage()->getAzureBlobStorageClient(), client, /* src_container */ blob_path[1], /* src_path */ blob_path[0], @@ -267,8 +263,8 @@ std::unique_ptr BackupWriterAzureBlobStorage::readFile(const String } return std::make_unique( - client.get(), key, read_settings, settings->max_single_read_retries, - settings->max_single_download_retries); + client.get(), key, read_settings, settings.get()->max_single_read_retries, + settings.get()->max_single_download_retries); } std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const String & file_name) @@ -285,7 +281,7 @@ std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const Strin return std::make_unique( client.get(), key, - settings->max_single_part_upload_size, + settings.get()->max_single_part_upload_size, DBMS_DEFAULT_BUFFER_SIZE, write_settings); } diff --git a/src/Backups/BackupIO_AzureBlobStorage.h b/src/Backups/BackupIO_AzureBlobStorage.h index 12bf073cd08..87dc470cdb3 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.h +++ b/src/Backups/BackupIO_AzureBlobStorage.h @@ -31,7 +31,7 @@ private: MultiVersion client; StorageAzureBlob::Configuration configuration; std::unique_ptr object_storage; - std::shared_ptr settings; + MultiVersion settings; }; class BackupWriterAzureBlobStorage : public BackupWriterDefault @@ -60,7 +60,7 @@ private: MultiVersion client; StorageAzureBlob::Configuration configuration; std::unique_ptr object_storage; - std::shared_ptr settings; + MultiVersion settings; }; } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 52d535054ff..a9d082539e6 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -139,7 +139,12 @@ public: bool isRemote() const override { return true; } - MultiVersion & getClient() { return client; } + MultiVersion & getSettings() { return settings; } + + MultiVersion & getAzureBlobStorageClient() override + { + return client; + } private: const String name; diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h index 4c185db051d..6b0ff8be58a 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h @@ -3,6 +3,7 @@ #include #include #include +#include "config.h" namespace Poco { @@ -118,6 +119,13 @@ public: static bool canUseReadThroughCache(const ReadSettings & settings); +#if USE_AZURE_BLOB_STORAGE + MultiVersion & getAzureBlobStorageClient() override + { + return object_storage->getAzureBlobStorageClient(); + } +#endif + private: FileCache::Key getCacheKey(const std::string & path) const; diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index f405be72287..cf113586ddf 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -23,7 +23,12 @@ #include #include #include +#include "config.h" +#if USE_AZURE_BLOB_STORAGE +#include +#include +#endif namespace DB { @@ -212,6 +217,14 @@ public: virtual WriteSettings patchSettings(const WriteSettings & write_settings) const; +#if USE_AZURE_BLOB_STORAGE + virtual MultiVersion & getAzureBlobStorageClient() + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "This function is only implemented for AzureBlobStorage"); + } +#endif + + private: mutable std::mutex throttlers_mutex; ThrottlerPtr remote_read_throttler; diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 272be914cc1..bb8702e9b41 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -49,7 +49,7 @@ namespace size_t total_size_, const String & dest_container_, const String & dest_blob_, - std::shared_ptr settings_, + MultiVersion settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_azure_blob_storage_, @@ -65,7 +65,7 @@ namespace , schedule(schedule_) , for_disk_azure_blob_storage(for_disk_azure_blob_storage_) , log(log_) - , max_single_part_upload_size(settings_->max_single_part_upload_size) + , max_single_part_upload_size(settings_.get()->max_single_part_upload_size) { } @@ -78,7 +78,7 @@ namespace size_t total_size; const String & dest_container; const String & dest_blob; - std::shared_ptr settings; + MultiVersion settings; const std::optional> & object_metadata; ThreadPoolCallbackRunner schedule; bool for_disk_azure_blob_storage; @@ -114,9 +114,9 @@ namespace if (!total_size) throw Exception(ErrorCodes::LOGICAL_ERROR, "Chosen multipart upload for an empty file. This must not happen"); - auto max_part_number = settings->max_part_number; - auto min_upload_part_size = settings->min_upload_part_size; - auto max_upload_part_size = settings->max_upload_part_size; + auto max_part_number = settings.get()->max_part_number; + auto min_upload_part_size = settings.get()->min_upload_part_size; + auto max_upload_part_size = settings.get()->max_upload_part_size; if (!max_part_number) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_part_number must not be 0"); @@ -333,7 +333,7 @@ void copyDataToAzureBlobStorageFile( MultiVersion & dest_client, const String & dest_container, const String & dest_blob, - std::shared_ptr settings, + MultiVersion settings, const std::optional> & object_metadata, ThreadPoolCallbackRunner schedule, bool for_disk_azure_blob_storage) @@ -352,14 +352,14 @@ void copyAzureBlobStorageFile( size_t size, const String & dest_container, const String & dest_blob, - std::shared_ptr settings, + MultiVersion settings, const ReadSettings & read_settings, const std::optional> & object_metadata, ThreadPoolCallbackRunner schedule, bool for_disk_azure_blob_storage) { - if (settings->use_native_copy) + if (settings.get()->use_native_copy) { ProfileEvents::increment(ProfileEvents::AzureCopyObject); if (for_disk_azure_blob_storage) @@ -393,8 +393,8 @@ void copyAzureBlobStorageFile( LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Container: {}, Blob: {}", src_container, src_blob); auto create_read_buffer = [&] { - return std::make_unique(src_client.get(), src_blob, read_settings, settings->max_single_read_retries, - settings->max_single_download_retries); + return std::make_unique(src_client.get(), src_blob, read_settings, settings.get()->max_single_read_retries, + settings.get()->max_single_download_retries); }; UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container, dest_blob, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyAzureBlobStorageFile")}; diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h index b022151d32d..491f7cd7176 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h @@ -29,7 +29,7 @@ void copyAzureBlobStorageFile( size_t src_size, const String & dest_container, const String & dest_blob, - std::shared_ptr settings, + MultiVersion settings, const ReadSettings & read_settings, const std::optional> & object_metadata = std::nullopt, ThreadPoolCallbackRunner schedule_ = {}, @@ -48,7 +48,7 @@ void copyDataToAzureBlobStorageFile( MultiVersion & client, const String & dest_container, const String & dest_blob, - std::shared_ptr settings, + MultiVersion settings, const std::optional> & object_metadata = std::nullopt, ThreadPoolCallbackRunner schedule_ = {}, bool for_disk_azure_blob_storage = false); diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp index 7a40d2dcb73..e54838c7a61 100644 --- a/src/Storages/StorageAzureBlob.cpp +++ b/src/Storages/StorageAzureBlob.cpp @@ -1214,7 +1214,7 @@ StorageAzureBlobSource::ReaderHolder StorageAzureBlobSource::createReader() QueryPipelineBuilder builder; std::shared_ptr source; std::unique_ptr read_buf; - std::optional num_rows_from_cache = need_only_count && getContext()->getSettingsRef().use_cache_for_count_from_files + std::optional num_rows_from_cache = need_only_count && getContext()->getSettingsRef().use_cache_for_count_from_files ? tryGetNumRowsFromCache(path_with_metadata) : std::nullopt; if (num_rows_from_cache) { From 992d859e726895dadc9fbab1ebf99acd4b29881c Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 23 Jan 2024 14:16:14 +0100 Subject: [PATCH 031/103] Fix style check --- src/Disks/ObjectStorages/IObjectStorage.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index cf113586ddf..b7db353fb6a 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "config.h" #if USE_AZURE_BLOB_STORAGE @@ -33,6 +34,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + class ReadBufferFromFileBase; class WriteBufferFromFileBase; From 849858017237d9752f3efb801bcc2267288cb8c8 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 24 Jan 2024 10:01:06 +0100 Subject: [PATCH 032/103] Fixing build --- src/Backups/BackupIO_AzureBlobStorage.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index fca324869ae..34be110cd42 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -143,7 +144,7 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( const WriteSettings & write_settings_, const ContextPtr & context_) : BackupWriterDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupWriterAzureBlobStorage")) - , data_source_description{DataSourceType::AzureBlobStorage,configuration_.container, false, false} + , data_source_description{DataSourceType::AzureBlobStorage, configuration_.container, false, false} , configuration(configuration_) { auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); From 788eb487075fe770097759edfd46544134e11116 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 24 Jan 2024 11:51:02 +0100 Subject: [PATCH 033/103] Fix build after merging master --- src/Backups/BackupIO_AzureBlobStorage.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 34be110cd42..2c2396e9c0a 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -34,7 +34,7 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( const WriteSettings & write_settings_, const ContextPtr & context_) : BackupReaderDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupReaderAzureBlobStorage")) - , data_source_description{DataSourceType::AzureBlobStorage, configuration_.container, false, false} + , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.container, false, false} , configuration(configuration_) { auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); @@ -99,7 +99,8 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, LOG_INFO(&Poco::Logger::get("BackupReaderAzureBlobStorage"), "Enter copyFileToDisk"); auto destination_data_source_description = destination_disk->getDataSourceDescription(); - if ((destination_data_source_description.type == DataSourceType::AzureBlobStorage) + if ((destination_data_source_description.type == DataSourceType::ObjectStorage) + && (destination_data_source_description.object_storage_type == ObjectStorageType::Azure) && (destination_data_source_description.is_encrypted == encrypted_in_backup)) { LOG_TRACE(log, "Copying {} from AzureBlobStorage to disk {}", path_in_backup, destination_disk->getName()); @@ -144,7 +145,7 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( const WriteSettings & write_settings_, const ContextPtr & context_) : BackupWriterDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupWriterAzureBlobStorage")) - , data_source_description{DataSourceType::AzureBlobStorage, configuration_.container, false, false} + , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.container, false, false} , configuration(configuration_) { auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false); From f551081dd4c38ac014f554c7ee4efc4e18777f9a Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 24 Jan 2024 21:10:50 +0100 Subject: [PATCH 034/103] Addressed review comments --- src/Backups/BackupIO_AzureBlobStorage.cpp | 7 ++--- .../copyAzureBlobStorageFile.cpp | 27 ++++++++----------- .../copyAzureBlobStorageFile.h | 8 +++--- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 2c2396e9c0a..1b4c10ad0cb 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -104,7 +104,7 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, && (destination_data_source_description.is_encrypted == encrypted_in_backup)) { LOG_TRACE(log, "Copying {} from AzureBlobStorage to disk {}", path_in_backup, destination_disk->getName()); - auto write_blob_function = [&](const Strings & blob_path, WriteMode mode, const std::optional & object_attributes) -> size_t + auto write_blob_function = [&](const Strings & blob_path, WriteMode mode, const std::optional &) -> size_t { /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files. if (blob_path.size() != 2 || mode != WriteMode::Rewrite) @@ -123,7 +123,6 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, /* dest_path */ blob_path[0], settings, read_settings, - object_attributes, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupRDAzure"), /* for_disk_azure_blob_storage= */ true); @@ -180,7 +179,6 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(const String & path_in_backu fs::path(configuration.blob_path) / path_in_backup, settings, read_settings, - {}, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWRAzure")); return; /// copied! } @@ -204,14 +202,13 @@ void BackupWriterAzureBlobStorage::copyFile(const String & destination, const St /* dest_path */ destination, settings, read_settings, - {}, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWRAzure"), /* for_disk_azure_blob_storage= */ true); } void BackupWriterAzureBlobStorage::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToAzureBlobStorageFile(create_read_buffer, start_pos, length, client, configuration.container, path_in_backup, settings, {}, + copyDataToAzureBlobStorageFile(create_read_buffer, start_pos, length, client, configuration.container, path_in_backup, settings, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWRAzure")); } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index bb8702e9b41..350d2d1d34e 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -47,10 +47,9 @@ namespace MultiVersion & client_, size_t offset_, size_t total_size_, - const String & dest_container_, + const String & dest_container_for_logging_, const String & dest_blob_, MultiVersion settings_, - const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_azure_blob_storage_, const Poco::Logger * log_) @@ -58,10 +57,9 @@ namespace , client(client_) , offset (offset_) , total_size (total_size_) - , dest_container(dest_container_) + , dest_container_for_logging(dest_container_for_logging_) , dest_blob(dest_blob_) , settings(settings_) - , object_metadata(object_metadata_) , schedule(schedule_) , for_disk_azure_blob_storage(for_disk_azure_blob_storage_) , log(log_) @@ -76,10 +74,9 @@ namespace MultiVersion & client; size_t offset; size_t total_size; - const String & dest_container; + const String & dest_container_for_logging; const String & dest_blob; MultiVersion settings; - const std::optional> & object_metadata; ThreadPoolCallbackRunner schedule; bool for_disk_azure_blob_storage; const Poco::Logger * log; @@ -208,7 +205,7 @@ namespace void uploadPart(size_t part_offset, size_t part_size) { - LOG_TRACE(log, "Writing part. Container: {}, Blob: {}, Size: {}", dest_container, dest_blob, part_size); + LOG_TRACE(log, "Writing part. Container: {}, Blob: {}, Size: {}", dest_container_for_logging, dest_blob, part_size); if (!part_size) { @@ -287,7 +284,7 @@ namespace std::lock_guard lock(bg_tasks_mutex); /// Protect bg_tasks from race task.block_id = block_id; - LOG_TRACE(log, "Writing part finished. Container: {}, Blob: {}, block_id: {}, Parts: {}", dest_container, dest_blob, block_id, bg_tasks.size()); + LOG_TRACE(log, "Writing part finished. Container: {}, Blob: {}, block_id: {}, Parts: {}", dest_container_for_logging, dest_blob, block_id, bg_tasks.size()); } String processUploadPartRequest(UploadPartTask & task) @@ -331,14 +328,13 @@ void copyDataToAzureBlobStorageFile( size_t offset, size_t size, MultiVersion & dest_client, - const String & dest_container, + const String & dest_container_for_logging, const String & dest_blob, MultiVersion settings, - const std::optional> & object_metadata, ThreadPoolCallbackRunner schedule, bool for_disk_azure_blob_storage) { - UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container, dest_blob, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyDataToAzureBlobStorageFile")}; + UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container_for_logging, dest_blob, settings, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyDataToAzureBlobStorageFile")}; helper.performCopy(); } @@ -346,15 +342,14 @@ void copyDataToAzureBlobStorageFile( void copyAzureBlobStorageFile( MultiVersion & src_client, MultiVersion & dest_client, - const String & src_container, + const String & src_container_for_logging, const String & src_blob, size_t offset, size_t size, - const String & dest_container, + const String & dest_container_for_logging, const String & dest_blob, MultiVersion settings, const ReadSettings & read_settings, - const std::optional> & object_metadata, ThreadPoolCallbackRunner schedule, bool for_disk_azure_blob_storage) { @@ -390,14 +385,14 @@ void copyAzureBlobStorageFile( } else { - LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Container: {}, Blob: {}", src_container, src_blob); + LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Container: {}, Blob: {}", src_container_for_logging, src_blob); auto create_read_buffer = [&] { return std::make_unique(src_client.get(), src_blob, read_settings, settings.get()->max_single_read_retries, settings.get()->max_single_download_retries); }; - UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container, dest_blob, settings, object_metadata, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyAzureBlobStorageFile")}; + UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container_for_logging, dest_blob, settings, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyAzureBlobStorageFile")}; helper.performCopy(); } } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h index 491f7cd7176..15a31031f63 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h @@ -23,15 +23,14 @@ using CreateReadBuffer = std::function()>; void copyAzureBlobStorageFile( MultiVersion & src_client, MultiVersion & dest_client, - const String & src_container, + const String & src_container_for_logging, const String & src_blob, size_t src_offset, size_t src_size, - const String & dest_container, + const String & dest_container_for_logging, const String & dest_blob, MultiVersion settings, const ReadSettings & read_settings, - const std::optional> & object_metadata = std::nullopt, ThreadPoolCallbackRunner schedule_ = {}, bool for_disk_azure_blob_storage = false); @@ -46,10 +45,9 @@ void copyDataToAzureBlobStorageFile( size_t offset, size_t size, MultiVersion & client, - const String & dest_container, + const String & dest_container_for_logging, const String & dest_blob, MultiVersion settings, - const std::optional> & object_metadata = std::nullopt, ThreadPoolCallbackRunner schedule_ = {}, bool for_disk_azure_blob_storage = false); From 09f1e2840c2859a517c9f76183a7abd51c488b6f Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 25 Jan 2024 10:06:05 +0100 Subject: [PATCH 035/103] Simplified calculatePartSize and upload task --- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 2 - .../AzureBlobStorage/AzureObjectStorage.h | 4 - .../copyAzureBlobStorageFile.cpp | 114 +++++------------- 3 files changed, 27 insertions(+), 93 deletions(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index cbc2996f5c1..02b0d5bb599 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -165,9 +165,7 @@ std::unique_ptr getAzureBlobStorageSettings(const Po config.getInt(config_prefix + ".max_single_read_retries", 3), config.getInt(config_prefix + ".max_single_download_retries", 3), config.getInt(config_prefix + ".list_object_keys_size", 1000), - config.getUInt64(config_prefix + ".min_upload_part_size", 16 * 1024 * 1024), config.getUInt64(config_prefix + ".max_upload_part_size", 5ULL * 1024 * 1024 * 1024), - config.getUInt64(config_prefix + ".max_part_number", 10000), config.getBool(config_prefix + ".use_native_copy", false) ); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 3be4989d4f2..30fedb601dc 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -24,18 +24,14 @@ struct AzureObjectStorageSettings int max_single_read_retries_, int max_single_download_retries_, int list_object_keys_size_, - size_t min_upload_part_size_, size_t max_upload_part_size_, - size_t max_part_number_, bool use_native_copy_) : max_single_part_upload_size(max_single_part_upload_size_) , min_bytes_for_seek(min_bytes_for_seek_) , max_single_read_retries(max_single_read_retries_) , max_single_download_retries(max_single_download_retries_) , list_object_keys_size(list_object_keys_size_) - , min_upload_part_size(min_upload_part_size_) , max_upload_part_size(max_upload_part_size_) - , max_part_number(max_part_number_) , use_native_copy(use_native_copy_) { } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 350d2d1d34e..e5517a1a021 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -84,17 +84,10 @@ namespace struct UploadPartTask { - char *data = nullptr; - size_t size = 0; - std::string block_id; + std::unique_ptr read_buffer = nullptr; + std::vector block_ids; bool is_finished = false; std::exception_ptr exception; - - ~UploadPartTask() - { - if (data != nullptr) - free(data); - } }; size_t normal_part_size; @@ -108,56 +101,11 @@ namespace void calculatePartSize() { - if (!total_size) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Chosen multipart upload for an empty file. This must not happen"); - - auto max_part_number = settings.get()->max_part_number; - auto min_upload_part_size = settings.get()->min_upload_part_size; auto max_upload_part_size = settings.get()->max_upload_part_size; - - if (!max_part_number) - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_part_number must not be 0"); - else if (!min_upload_part_size) - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "min_upload_part_size must not be 0"); - else if (max_upload_part_size < min_upload_part_size) - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_upload_part_size must not be less than min_upload_part_size"); - - size_t part_size = min_upload_part_size; - size_t num_parts = (total_size + part_size - 1) / part_size; - - if (num_parts > max_part_number) - { - part_size = (total_size + max_part_number - 1) / max_part_number; - num_parts = (total_size + part_size - 1) / part_size; - } - - if (part_size > max_upload_part_size) - { - part_size = max_upload_part_size; - num_parts = (total_size + part_size - 1) / part_size; - } - - if (num_parts < 1 || num_parts > max_part_number || part_size < min_upload_part_size || part_size > max_upload_part_size) - { - String msg; - if (num_parts < 1) - msg = "Number of parts is zero"; - else if (num_parts > max_part_number) - msg = fmt::format("Number of parts exceeds {}", num_parts, max_part_number); - else if (part_size < min_upload_part_size) - msg = fmt::format("Size of a part is less than {}", part_size, min_upload_part_size); - else - msg = fmt::format("Size of a part exceeds {}", part_size, max_upload_part_size); - - throw Exception( - ErrorCodes::INVALID_CONFIG_PARAMETER, - "{} while writing {} bytes to AzureBlobStorage. Check max_part_number = {}, " - "min_upload_part_size = {}, max_upload_part_size = {}", - msg, total_size, max_part_number, min_upload_part_size, max_upload_part_size); - } - + if (!max_upload_part_size) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_upload_part_size must not be 0"); /// We've calculated the size of a normal part (the final part can be smaller). - normal_part_size = part_size; + normal_part_size = max_upload_part_size; } public: @@ -238,18 +186,13 @@ namespace try { - auto read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); - task->data = new char[part_size]; - task->size = part_size; - size_t n = read_buffer->read(task->data,part_size); - if (n != part_size) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected size"); + task->read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); schedule([this, task, task_finish_notify]() { try { - processUploadTask(*task); + processUploadPartRequest(*task); } catch (...) { @@ -267,38 +210,35 @@ namespace else { UploadPartTask task; - auto read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); - task.data = new char[part_size]; - size_t n = read_buffer->read(task.data,part_size); - if (n != part_size) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected size"); - task.size = part_size; - processUploadTask(task); - block_ids.emplace_back(task.block_id); + task.read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); + processUploadPartRequest(task); + block_ids.insert(block_ids.end(),task.block_ids.begin(), task.block_ids.end()); } } - void processUploadTask(UploadPartTask & task) - { - auto block_id = processUploadPartRequest(task); - - std::lock_guard lock(bg_tasks_mutex); /// Protect bg_tasks from race - task.block_id = block_id; - LOG_TRACE(log, "Writing part finished. Container: {}, Blob: {}, block_id: {}, Parts: {}", dest_container_for_logging, dest_blob, block_id, bg_tasks.size()); - } - - String processUploadPartRequest(UploadPartTask & task) + void processUploadPartRequest(UploadPartTask & task) { ProfileEvents::increment(ProfileEvents::AzureUploadPart); if (for_disk_azure_blob_storage) ProfileEvents::increment(ProfileEvents::DiskAzureUploadPart); auto block_blob_client = client.get()->GetBlockBlobClient(dest_blob); - task.block_id = getRandomASCIIString(64); - Azure::Core::IO::MemoryBodyStream memory(reinterpret_cast(task.data), task.size); - block_blob_client.StageBlock(task.block_id, memory); - return task.block_id; + while (!task.read_buffer->eof()) + { + auto size = task.read_buffer->available(); + if (size > 0) + { + auto block_id = getRandomASCIIString(64); + Azure::Core::IO::MemoryBodyStream memory(reinterpret_cast(task.read_buffer->position()), size); + block_blob_client.StageBlock(block_id, memory); + task.block_ids.emplace_back(block_id); + task.read_buffer->ignore(size); + LOG_TRACE(log, "Writing part. Container: {}, Blob: {}, block_id: {}", dest_container_for_logging, dest_blob, block_id); + } + } + std::lock_guard lock(bg_tasks_mutex); /// Protect bg_tasks from race + LOG_TRACE(log, "Writing part finished. Container: {}, Blob: {}, Parts: {}", dest_container_for_logging, dest_blob, bg_tasks.size()); } @@ -316,7 +256,7 @@ namespace { if (task.exception) std::rethrow_exception(task.exception); - block_ids.emplace_back(task.block_id); + block_ids.insert(block_ids.end(),task.block_ids.begin(), task.block_ids.end()); } } }; From d264a5a148c577ab046dc4bbef50b5a4e0c32db9 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sun, 28 Jan 2024 12:06:52 +0100 Subject: [PATCH 036/103] Update client & settings to std::shared_ptr --- src/Backups/BackupIO_AzureBlobStorage.cpp | 6 ++-- src/Backups/BackupIO_AzureBlobStorage.h | 8 +++--- .../AzureBlobStorage/AzureObjectStorage.h | 6 ++-- .../Cached/CachedObjectStorage.h | 2 +- src/Disks/ObjectStorages/IObjectStorage.h | 2 +- .../copyAzureBlobStorageFile.cpp | 28 +++++++++---------- .../copyAzureBlobStorageFile.h | 10 +++---- 7 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 1b4c10ad0cb..d99f296cca1 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -89,7 +89,7 @@ std::unique_ptr BackupReaderAzureBlobStorage::readFile(const key = file_name; } return std::make_unique( - client.get(), key, read_settings, settings.get()->max_single_read_retries, + client, key, read_settings, settings.get()->max_single_read_retries, settings.get()->max_single_download_retries); } @@ -262,7 +262,7 @@ std::unique_ptr BackupWriterAzureBlobStorage::readFile(const String } return std::make_unique( - client.get(), key, read_settings, settings.get()->max_single_read_retries, + client, key, read_settings, settings.get()->max_single_read_retries, settings.get()->max_single_download_retries); } @@ -278,7 +278,7 @@ std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const Strin key = file_name; } return std::make_unique( - client.get(), + client, key, settings.get()->max_single_part_upload_size, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Backups/BackupIO_AzureBlobStorage.h b/src/Backups/BackupIO_AzureBlobStorage.h index 87dc470cdb3..95325044a62 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.h +++ b/src/Backups/BackupIO_AzureBlobStorage.h @@ -28,10 +28,10 @@ public: private: const DataSourceDescription data_source_description; - MultiVersion client; + std::shared_ptr client; StorageAzureBlob::Configuration configuration; std::unique_ptr object_storage; - MultiVersion settings; + std::shared_ptr settings; }; class BackupWriterAzureBlobStorage : public BackupWriterDefault @@ -57,10 +57,10 @@ private: std::unique_ptr readFile(const String & file_name, size_t expected_file_size) override; void removeFilesBatch(const Strings & file_names); const DataSourceDescription data_source_description; - MultiVersion client; + std::shared_ptr client; StorageAzureBlob::Configuration configuration; std::unique_ptr object_storage; - MultiVersion settings; + std::shared_ptr settings; }; } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 30fedb601dc..0ae12fb205f 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -137,11 +137,11 @@ public: bool isRemote() const override { return true; } - MultiVersion & getSettings() { return settings; } + std::shared_ptr getSettings() { return settings.get(); } - MultiVersion & getAzureBlobStorageClient() override + std::shared_ptr getAzureBlobStorageClient() override { - return client; + return client.get(); } private: diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h index 2ed8990515f..1f293e5857e 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h @@ -122,7 +122,7 @@ public: static bool canUseReadThroughCache(const ReadSettings & settings); #if USE_AZURE_BLOB_STORAGE - MultiVersion & getAzureBlobStorageClient() override + std::shared_ptr getAzureBlobStorageClient() override { return object_storage->getAzureBlobStorageClient(); } diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index e066beaefcc..049935ad60c 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -226,7 +226,7 @@ public: virtual WriteSettings patchSettings(const WriteSettings & write_settings) const; #if USE_AZURE_BLOB_STORAGE - virtual MultiVersion & getAzureBlobStorageClient() + virtual std::shared_ptr getAzureBlobStorageClient() { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "This function is only implemented for AzureBlobStorage"); } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index e5517a1a021..537a5a191e7 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -44,12 +44,12 @@ namespace public: UploadHelper( const CreateReadBuffer & create_read_buffer_, - MultiVersion & client_, + std::shared_ptr client_, size_t offset_, size_t total_size_, const String & dest_container_for_logging_, const String & dest_blob_, - MultiVersion settings_, + std::shared_ptr settings_, ThreadPoolCallbackRunner schedule_, bool for_disk_azure_blob_storage_, const Poco::Logger * log_) @@ -71,12 +71,12 @@ namespace protected: std::function()> create_read_buffer; - MultiVersion & client; + std::shared_ptr client; size_t offset; size_t total_size; const String & dest_container_for_logging; const String & dest_blob; - MultiVersion settings; + std::shared_ptr settings; ThreadPoolCallbackRunner schedule; bool for_disk_azure_blob_storage; const Poco::Logger * log; @@ -116,7 +116,7 @@ namespace void completeMultipartUpload() { - auto block_blob_client = client.get()->GetBlockBlobClient(dest_blob); + auto block_blob_client = client->GetBlockBlobClient(dest_blob); block_blob_client.CommitBlockList(block_ids); } @@ -222,7 +222,7 @@ namespace if (for_disk_azure_blob_storage) ProfileEvents::increment(ProfileEvents::DiskAzureUploadPart); - auto block_blob_client = client.get()->GetBlockBlobClient(dest_blob); + auto block_blob_client = client->GetBlockBlobClient(dest_blob); while (!task.read_buffer->eof()) { @@ -267,10 +267,10 @@ void copyDataToAzureBlobStorageFile( const std::function()> & create_read_buffer, size_t offset, size_t size, - MultiVersion & dest_client, + std::shared_ptr dest_client, const String & dest_container_for_logging, const String & dest_blob, - MultiVersion settings, + std::shared_ptr settings, ThreadPoolCallbackRunner schedule, bool for_disk_azure_blob_storage) { @@ -280,15 +280,15 @@ void copyDataToAzureBlobStorageFile( void copyAzureBlobStorageFile( - MultiVersion & src_client, - MultiVersion & dest_client, + std::shared_ptr src_client, + std::shared_ptr dest_client, const String & src_container_for_logging, const String & src_blob, size_t offset, size_t size, const String & dest_container_for_logging, const String & dest_blob, - MultiVersion settings, + std::shared_ptr settings, const ReadSettings & read_settings, ThreadPoolCallbackRunner schedule, bool for_disk_azure_blob_storage) @@ -300,8 +300,8 @@ void copyAzureBlobStorageFile( if (for_disk_azure_blob_storage) ProfileEvents::increment(ProfileEvents::DiskAzureCopyObject); - auto block_blob_client_src = src_client.get()->GetBlockBlobClient(src_blob); - auto block_blob_client_dest = dest_client.get()->GetBlockBlobClient(dest_blob); + auto block_blob_client_src = src_client->GetBlockBlobClient(src_blob); + auto block_blob_client_dest = dest_client->GetBlockBlobClient(dest_blob); auto source_uri = block_blob_client_src.GetUrl(); if (size < max_single_operation_copy_size) @@ -328,7 +328,7 @@ void copyAzureBlobStorageFile( LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Container: {}, Blob: {}", src_container_for_logging, src_blob); auto create_read_buffer = [&] { - return std::make_unique(src_client.get(), src_blob, read_settings, settings.get()->max_single_read_retries, + return std::make_unique(src_client, src_blob, read_settings, settings.get()->max_single_read_retries, settings.get()->max_single_download_retries); }; diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h index 15a31031f63..83814f42693 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h @@ -21,15 +21,15 @@ using CreateReadBuffer = std::function()>; /// Copies a file from AzureBlobStorage to AzureBlobStorage. /// The parameters `src_offset` and `src_size` specify a part in the source to copy. void copyAzureBlobStorageFile( - MultiVersion & src_client, - MultiVersion & dest_client, + std::shared_ptr src_client, + std::shared_ptr dest_client, const String & src_container_for_logging, const String & src_blob, size_t src_offset, size_t src_size, const String & dest_container_for_logging, const String & dest_blob, - MultiVersion settings, + std::shared_ptr settings, const ReadSettings & read_settings, ThreadPoolCallbackRunner schedule_ = {}, bool for_disk_azure_blob_storage = false); @@ -44,10 +44,10 @@ void copyDataToAzureBlobStorageFile( const std::function()> & create_read_buffer, size_t offset, size_t size, - MultiVersion & client, + std::shared_ptr client, const String & dest_container_for_logging, const String & dest_blob, - MultiVersion settings, + std::shared_ptr settings, ThreadPoolCallbackRunner schedule_ = {}, bool for_disk_azure_blob_storage = false); From a22b68f46fec54f98fc3c3cb9a9c1f597bae7ffd Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 29 Jan 2024 10:49:36 +0100 Subject: [PATCH 037/103] Added setting azure_max_single_part_copy_size --- src/Core/Settings.h | 3 ++- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 4 +++- .../ObjectStorages/AzureBlobStorage/AzureObjectStorage.h | 3 +++ src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp | 6 +----- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 305d6466658..4ae5d1585f3 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -82,7 +82,8 @@ class IColumn; M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, 500, "Each time this number of parts was uploaded to S3, s3_min_upload_part_size is multiplied by s3_upload_part_size_multiply_factor.", 0) \ M(UInt64, s3_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited. You ", 0) \ M(UInt64, s3_max_single_part_upload_size, 32*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \ - M(UInt64, azure_max_single_part_upload_size, 100*1024*1024, "The maximum size of object to upload using singlepart upload to Azure blob storage.", 0) \ + M(UInt64, azure_max_single_part_upload_size, 100*1024*1024, "The maximum size of object to upload using singlepart upload to Azure blob storage.", 0) \ + M(UInt64, azure_max_single_part_copy_size, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage.", 0) \ M(UInt64, s3_max_single_read_retries, 4, "The maximum number of retries during single S3 read.", 0) \ M(UInt64, azure_max_single_read_retries, 4, "The maximum number of retries during single Azure blob storage read.", 0) \ M(UInt64, s3_max_unexpected_write_error_retries, 4, "The maximum number of retries in case of unexpected errors during S3 write.", 0) \ diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 02b0d5bb599..9da84d430e4 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -7,6 +7,7 @@ #include #include #include +#include using namespace Azure::Storage::Blobs; @@ -157,7 +158,7 @@ std::unique_ptr getAzureBlobContainerClient( } } -std::unique_ptr getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr /*context*/) +std::unique_ptr getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) { return std::make_unique( config.getUInt64(config_prefix + ".max_single_part_upload_size", 100 * 1024 * 1024), @@ -166,6 +167,7 @@ std::unique_ptr getAzureBlobStorageSettings(const Po config.getInt(config_prefix + ".max_single_download_retries", 3), config.getInt(config_prefix + ".list_object_keys_size", 1000), config.getUInt64(config_prefix + ".max_upload_part_size", 5ULL * 1024 * 1024 * 1024), + config.getUInt64(config_prefix + ".max_single_part_copy_size", context->getSettings().azure_max_single_part_copy_size), config.getBool(config_prefix + ".use_native_copy", false) ); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 0ae12fb205f..18b1a70defe 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -25,6 +25,7 @@ struct AzureObjectStorageSettings int max_single_download_retries_, int list_object_keys_size_, size_t max_upload_part_size_, + size_t max_single_part_copy_size_, bool use_native_copy_) : max_single_part_upload_size(max_single_part_upload_size_) , min_bytes_for_seek(min_bytes_for_seek_) @@ -32,6 +33,7 @@ struct AzureObjectStorageSettings , max_single_download_retries(max_single_download_retries_) , list_object_keys_size(list_object_keys_size_) , max_upload_part_size(max_upload_part_size_) + , max_single_part_copy_size(max_single_part_copy_size_) , use_native_copy(use_native_copy_) { } @@ -46,6 +48,7 @@ struct AzureObjectStorageSettings size_t min_upload_part_size = 16 * 1024 * 1024; size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; size_t max_part_number = 10000; + size_t max_single_part_copy_size = 256 * 1024 * 1024; bool use_native_copy = false; }; diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 537a5a191e7..ff4cfe62feb 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -33,10 +33,6 @@ namespace ErrorCodes extern const int AZURE_BLOB_STORAGE_ERROR; } - -size_t max_single_operation_copy_size = 256 * 1024 * 1024; - - namespace { class UploadHelper @@ -304,7 +300,7 @@ void copyAzureBlobStorageFile( auto block_blob_client_dest = dest_client->GetBlockBlobClient(dest_blob); auto source_uri = block_blob_client_src.GetUrl(); - if (size < max_single_operation_copy_size) + if (size < settings.get()->max_single_part_copy_size) { block_blob_client_dest.CopyFromUri(source_uri); } From 99a1b269d71054a1d4d1e59a55b229469652435c Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 29 Jan 2024 11:00:59 +0100 Subject: [PATCH 038/103] Removed unwanted setting --- src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 18b1a70defe..7d5c8f07a75 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -47,7 +47,6 @@ struct AzureObjectStorageSettings int list_object_keys_size = 1000; size_t min_upload_part_size = 16 * 1024 * 1024; size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; - size_t max_part_number = 10000; size_t max_single_part_copy_size = 256 * 1024 * 1024; bool use_native_copy = false; }; From ce0ebd964519d0961d92318e8a171d5338365213 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 29 Jan 2024 11:14:19 +0100 Subject: [PATCH 039/103] Removed unwanted log lines --- src/Backups/BackupImpl.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 0fb0d8cbda9..28a7d60b52c 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -535,7 +535,6 @@ void BackupImpl::checkBackupDoesntExist() const else file_name_to_check_existence = ".backup"; - LOG_INFO(&Poco::Logger::get("BackupImpl"), "checkBackupDoesntExist 1"); if (writer->fileExists(file_name_to_check_existence)) throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} already exists", backup_name_for_logging); @@ -567,8 +566,6 @@ bool BackupImpl::checkLockFile(bool throw_if_failed) const if (throw_if_failed) { - LOG_INFO(&Poco::Logger::get("BackupImpl"), "checkLockFile"); - if (!writer->fileExists(lock_file_name)) { throw Exception( From 6bfa910d9ea403e91fb9be04573c73bfae77b4c4 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 29 Jan 2024 16:47:02 +0100 Subject: [PATCH 040/103] Fix merge --- src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 78a67f3e59a..8556f0237e3 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -47,12 +47,10 @@ struct AzureObjectStorageSettings size_t max_single_read_retries = 3; size_t max_single_download_retries = 3; int list_object_keys_size = 1000; -<<<<<<< HEAD size_t min_upload_part_size = 16 * 1024 * 1024; size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024; size_t max_single_part_copy_size = 256 * 1024 * 1024; bool use_native_copy = false; -======= size_t max_unexpected_write_error_retries = 4; >>>>>>> master }; From 82d7b2214407c68b96334d002f01ed68937e07f4 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 1 Feb 2024 21:06:29 +0000 Subject: [PATCH 041/103] Parallel replicas: better replicas failover --- .../ClusterProxy/executeQuery.cpp | 15 +++--- src/Processors/QueryPlan/ReadFromRemote.cpp | 52 ++++++------------- src/Processors/QueryPlan/ReadFromRemote.h | 5 +- src/QueryPipeline/RemoteQueryExecutor.cpp | 44 ++++++++++++++++ src/QueryPipeline/RemoteQueryExecutor.h | 13 ++++- 5 files changed, 82 insertions(+), 47 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 35451e1d774..023ed6c7b61 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -374,12 +374,12 @@ void executeQueryWithParallelReplicas( shard_num = column->getUInt(0); } - ClusterPtr new_cluster; + const auto shard_count = not_optimized_cluster->getShardCount(); + ClusterPtr new_cluster = not_optimized_cluster; /// if got valid shard_num from query initiator, then parallel replicas scope is the specified shard /// shards are numbered in order of appearance in the cluster config if (shard_num > 0) { - const auto shard_count = not_optimized_cluster->getShardCount(); if (shard_num > shard_count) throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -395,17 +395,16 @@ void executeQueryWithParallelReplicas( // get cluster for shard specified by shard_num // shard_num is 1-based, but getClusterWithSingleShard expects 0-based index - auto single_shard_cluster = not_optimized_cluster->getClusterWithSingleShard(shard_num - 1); - // convert cluster to representation expected by parallel replicas - new_cluster = single_shard_cluster->getClusterWithReplicasAsShards(settings, settings.max_parallel_replicas); + new_cluster = not_optimized_cluster->getClusterWithSingleShard(shard_num - 1); } else { - new_cluster = not_optimized_cluster->getClusterWithReplicasAsShards(settings, settings.max_parallel_replicas); + // todo: add error and exception for this case + chassert(not_optimized_cluster->getShardCount() == 1); } - auto coordinator - = std::make_shared(new_cluster->getShardCount(), settings.parallel_replicas_mark_segment_size); + auto coordinator = std::make_shared( + new_cluster->getShardsInfo().begin()->getAllNodeCount(), settings.parallel_replicas_mark_segment_size); auto external_tables = new_context->getExternalTables(); auto read_from_remote = std::make_unique( query_ast, diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 4dd79903965..fcdb7cd4a70 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -399,51 +399,33 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder const Settings & current_settings = context->getSettingsRef(); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings); + const auto & shard = cluster->getShardsInfo().at(0); size_t all_replicas_count = current_settings.max_parallel_replicas; - if (all_replicas_count > cluster->getShardsInfo().size()) + if (all_replicas_count > shard.getAllNodeCount()) { - LOG_INFO(getLogger("ReadFromParallelRemoteReplicasStep"), - "The number of replicas requested ({}) is bigger than the real number available in the cluster ({}). "\ - "Will use the latter number to execute the query.", current_settings.max_parallel_replicas, cluster->getShardsInfo().size()); - all_replicas_count = cluster->getShardsInfo().size(); + LOG_INFO( + getLogger("ReadFromParallelRemoteReplicasStep"), + "The number of replicas requested ({}) is bigger than the real number available in the cluster ({}). " + "Will use the latter number to execute the query.", + current_settings.max_parallel_replicas, + shard.getAllNodeCount()); + all_replicas_count = shard.getAllNodeCount(); } - /// Find local shard. It might happen that there is no local shard, but that's fine - for (const auto & shard: cluster->getShardsInfo()) + chassert(cluster->getShardCount() == 1); + auto shuffled_pool = shard.pool->getShuffledPools(current_settings); + shuffled_pool.resize(all_replicas_count); + + for (size_t i=0; i < all_replicas_count; ++i) { - if (shard.isLocal()) - { - IConnections::ReplicaInfo replica_info - { - .all_replicas_count = all_replicas_count, - /// `shard_num` will be equal to the number of the given replica in the cluster (set by `Cluster::getClusterWithReplicasAsShards`). - /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it. - .number_of_current_replica = shard.shard_num - 1, - }; - - addPipeForSingeReplica(pipes, shard.pool, replica_info); - } - } - - auto current_shard = cluster->getShardsInfo().begin(); - while (pipes.size() != all_replicas_count) - { - if (current_shard->isLocal()) - { - ++current_shard; - continue; - } - IConnections::ReplicaInfo replica_info { .all_replicas_count = all_replicas_count, - /// `shard_num` will be equal to the number of the given replica in the cluster (set by `Cluster::getClusterWithReplicasAsShards`). /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it. - .number_of_current_replica = current_shard->shard_num - 1, + .number_of_current_replica = i, }; - addPipeForSingeReplica(pipes, current_shard->pool, replica_info); - ++current_shard; + addPipeForSingeReplica(pipes, shuffled_pool[i].pool, replica_info); } auto pipe = Pipe::unitePipes(std::move(pipes)); @@ -456,7 +438,7 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder } -void ReadFromParallelRemoteReplicasStep::addPipeForSingeReplica(Pipes & pipes, std::shared_ptr pool, IConnections::ReplicaInfo replica_info) +void ReadFromParallelRemoteReplicasStep::addPipeForSingeReplica(Pipes & pipes, IConnectionPool* pool, IConnections::ReplicaInfo replica_info) { bool add_agg_info = stage == QueryProcessingStage::WithMergeableState; bool add_totals = false; diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index f853a12910b..07443220c8d 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -10,8 +10,7 @@ namespace DB { -class ConnectionPoolWithFailover; -using ConnectionPoolWithFailoverPtr = std::shared_ptr; +class IConnectionPool; class Throttler; using ThrottlerPtr = std::shared_ptr; @@ -92,7 +91,7 @@ public: private: - void addPipeForSingeReplica(Pipes & pipes, std::shared_ptr pool, IConnections::ReplicaInfo replica_info); + void addPipeForSingeReplica(Pipes & pipes, IConnectionPool* pool, IConnections::ReplicaInfo replica_info); ClusterPtr cluster; ASTPtr query_ast; diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 136a3bb09c6..7f25c2331c3 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -62,6 +62,50 @@ RemoteQueryExecutor::RemoteQueryExecutor( { } +RemoteQueryExecutor::RemoteQueryExecutor( + IConnectionPool * pool, + const String & query_, + const Block & header_, + ContextPtr context_, + ThrottlerPtr throttler, + const Scalars & scalars_, + const Tables & external_tables_, + QueryProcessingStage::Enum stage_, + std::optional extension_) + : RemoteQueryExecutor(query_, header_, context_, scalars_, external_tables_, stage_, extension_) +{ + create_connections = [this, pool, throttler, extension_](AsyncCallback) + { + const Settings & current_settings = context->getSettingsRef(); + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings); + + ConnectionPoolWithFailover::TryResult result; + std::string fail_message; + if (main_table) + { + auto table_name = main_table.getQualifiedName(); + + ConnectionEstablisher connection_establisher(pool, &timeouts, current_settings, log, &table_name); + connection_establisher.run(result, fail_message); + } + else + { + ConnectionEstablisher connection_establisher(pool, &timeouts, current_settings, log, nullptr); + connection_establisher.run(result, fail_message); + } + + std::vector connection_entries; + if (!result.entry.isNull() && result.is_usable) + connection_entries.emplace_back(std::move(result.entry)); + + auto res = std::make_unique(std::move(connection_entries), context->getSettingsRef(), throttler); + if (extension_ && extension_->replica_info) + res->setReplicaInfo(*extension_->replica_info); + + return res; + }; +} + RemoteQueryExecutor::RemoteQueryExecutor( Connection & connection, const String & query_, const Block & header_, ContextPtr context_, diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index 444f1258f3e..cc3291313a8 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -50,9 +50,20 @@ public: std::shared_ptr task_iterator = nullptr; std::shared_ptr parallel_reading_coordinator = nullptr; std::optional replica_info = {}; - GetPriorityForLoadBalancing::Func priority_func; }; + /// Takes a connection pool to a node (not cluster) + RemoteQueryExecutor( + IConnectionPool * pool, + const String & query_, + const Block & header_, + ContextPtr context_, + ThrottlerPtr throttler = nullptr, + const Scalars & scalars_ = Scalars(), + const Tables & external_tables_ = Tables(), + QueryProcessingStage::Enum stage_ = QueryProcessingStage::Complete, + std::optional extension_ = std::nullopt); + /// Takes already set connection. RemoteQueryExecutor( Connection & connection, From 4b77258341cfc35bfeb2ad63a6747f34cfb0c54b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 2 Feb 2024 11:28:35 +0000 Subject: [PATCH 042/103] Fix test --- src/QueryPipeline/RemoteQueryExecutor.h | 2 +- .../02769_parallel_replicas_unavailable_shards.sql | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index cc3291313a8..1eee0ac664f 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -52,7 +52,7 @@ public: std::optional replica_info = {}; }; - /// Takes a connection pool to a node (not cluster) + /// Takes a connection pool for a node (not cluster) RemoteQueryExecutor( IConnectionPool * pool, const String & query_, diff --git a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql index 38d592201e3..be200353f06 100644 --- a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql +++ b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql @@ -6,10 +6,11 @@ SYSTEM FLUSH LOGS; SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1; SET send_logs_level='error'; -SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*); +SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79'; SYSTEM FLUSH LOGS; -SELECT count() > 0 FROM system.text_log WHERE yesterday() <= event_date AND message LIKE '%Replica number 10 is unavailable%'; +SET allow_experimental_parallel_reading_from_replicas=0; +SELECT count() FROM system.text_log WHERE yesterday() <= event_date AND query_id in (select query_id from system.query_log where log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79' and current_database = currentDatabase()) and message LIKE '%Replica number % is unavailable%'; DROP TABLE test_parallel_replicas_unavailable_shards; From 8ebd7a7952dcace13bf71cf29431f992869ad909 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 2 Feb 2024 13:39:06 +0000 Subject: [PATCH 043/103] Formatting --- src/Processors/QueryPlan/ReadFromRemote.cpp | 12 ++++++++++-- src/Processors/QueryPlan/ReadFromRemote.h | 3 +-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index fcdb7cd4a70..cab3ae72678 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -438,7 +438,8 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder } -void ReadFromParallelRemoteReplicasStep::addPipeForSingeReplica(Pipes & pipes, IConnectionPool* pool, IConnections::ReplicaInfo replica_info) +void ReadFromParallelRemoteReplicasStep::addPipeForSingeReplica( + Pipes & pipes, IConnectionPool * pool, IConnections::ReplicaInfo replica_info) { bool add_agg_info = stage == QueryProcessingStage::WithMergeableState; bool add_totals = false; @@ -458,7 +459,14 @@ void ReadFromParallelRemoteReplicasStep::addPipeForSingeReplica(Pipes & pipes, I assert(output_stream); auto remote_query_executor = std::make_shared( - pool, query_string, output_stream->header, context, throttler, scalars, external_tables, stage, + pool, + query_string, + output_stream->header, + context, + throttler, + scalars, + external_tables, + stage, RemoteQueryExecutor::Extension{.parallel_reading_coordinator = coordinator, .replica_info = std::move(replica_info)}); remote_query_executor->setLogger(log); diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index 07443220c8d..eb3bcd12cc3 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -90,8 +90,7 @@ public: void enforceAggregationInOrder(); private: - - void addPipeForSingeReplica(Pipes & pipes, IConnectionPool* pool, IConnections::ReplicaInfo replica_info); + void addPipeForSingeReplica(Pipes & pipes, IConnectionPool * pool, IConnections::ReplicaInfo replica_info); ClusterPtr cluster; ASTPtr query_ast; From ea2dad181d5c91bddd63fc1ca64636123053fb8a Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 2 Feb 2024 15:05:14 +0000 Subject: [PATCH 044/103] Fix non linux builds --- src/QueryPipeline/RemoteQueryExecutor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 7f25c2331c3..90bd4bfdfdf 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -4,7 +4,7 @@ #include #include -#include "Core/Protocol.h" +#include #include #include #include @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include From b0994c5fa79ec6ba8dd14e028a6e2731d4e5f031 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sun, 4 Feb 2024 11:28:20 +0100 Subject: [PATCH 045/103] Addressed comments, added test for named collection --- src/Backups/BackupIO_AzureBlobStorage.cpp | 10 +- src/Backups/BackupImpl.cpp | 1 - .../registerBackupEngineAzureBlobStorage.cpp | 75 +++++++---- .../AzureBlobStorage/AzureObjectStorage.cpp | 1 + .../AzureBlobStorage/AzureObjectStorage.h | 1 - .../copyAzureBlobStorageFile.cpp | 19 +-- src/Storages/StorageAzureBlob.cpp | 11 -- src/Storages/StorageAzureBlob.h | 1 - .../test.py | 123 ++++++++++++++++-- 9 files changed, 179 insertions(+), 63 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index d99f296cca1..27928e871ce 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -33,7 +33,7 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_) - : BackupReaderDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupReaderAzureBlobStorage")) + : BackupReaderDefault(read_settings_, write_settings_, getLogger("BackupReaderAzureBlobStorage")) , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.container, false, false} , configuration(configuration_) { @@ -96,8 +96,6 @@ std::unique_ptr BackupReaderAzureBlobStorage::readFile(const void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) { - LOG_INFO(&Poco::Logger::get("BackupReaderAzureBlobStorage"), "Enter copyFileToDisk"); - auto destination_data_source_description = destination_disk->getDataSourceDescription(); if ((destination_data_source_description.type == DataSourceType::ObjectStorage) && (destination_data_source_description.object_storage_type == ObjectStorageType::Azure) @@ -143,7 +141,7 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_) - : BackupWriterDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupWriterAzureBlobStorage")) + : BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterAzureBlobStorage")) , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.container, false, false} , configuration(configuration_) { @@ -225,14 +223,11 @@ bool BackupWriterAzureBlobStorage::fileExists(const String & file_name) { key = file_name; } - LOG_INFO(&Poco::Logger::get("BackupWriterAzureBlobStorage"), "Result fileExists {} ", object_storage->exists(StoredObject(key))); - return object_storage->exists(StoredObject(key)); } UInt64 BackupWriterAzureBlobStorage::getFileSize(const String & file_name) { - LOG_INFO(&Poco::Logger::get("BackupWriterAzureBlobStorage"), "Enter getFileSize"); String key; if (startsWith(file_name, ".")) { @@ -281,6 +276,7 @@ std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const Strin client, key, settings.get()->max_single_part_upload_size, + settings.get()->max_unexpected_write_error_retries, DBMS_DEFAULT_BUFFER_SIZE, write_settings); } diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 28a7d60b52c..8a4ed31bfd7 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -542,7 +542,6 @@ void BackupImpl::checkBackupDoesntExist() const if (!is_internal_backup) { assert(!lock_file_name.empty()); - LOG_INFO(&Poco::Logger::get("BackupImpl"), "checkBackupDoesntExist 2"); if (writer->fileExists(lock_file_name)) throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} is being written already", backup_name_for_logging); } diff --git a/src/Backups/registerBackupEngineAzureBlobStorage.cpp b/src/Backups/registerBackupEngineAzureBlobStorage.cpp index 3480ea75f1f..48f66569304 100644 --- a/src/Backups/registerBackupEngineAzureBlobStorage.cpp +++ b/src/Backups/registerBackupEngineAzureBlobStorage.cpp @@ -49,40 +49,65 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory) const String & id_arg = params.backup_info.id_arg; const auto & args = params.backup_info.args; - LOG_INFO(&Poco::Logger::get("registerBackupEngineAzureBlobStorage"), "Begin id_arg={} args.size={}", id_arg, args.size()); - StorageAzureBlob::Configuration configuration; - if (args.size() == 3) + if (!id_arg.empty()) { - configuration.connection_url = args[0].safeGet(); - configuration.is_connection_string = true; + const auto & config = params.context->getConfigRef(); + auto config_prefix = "named_collections." + id_arg; - configuration.container = args[1].safeGet(); - configuration.blob_path = args[2].safeGet(); + if (!config.has(config_prefix)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no collection named `{}` in config", id_arg); - LOG_TRACE(&Poco::Logger::get("registerBackupEngineAzureBlobStorage"), "configuration.connection_url = {}" - "configuration.container = {}" - "configuration.blob_path = {}", - configuration.connection_url, configuration.container, configuration.blob_path); - } - else if (args.size() == 5) - { - configuration.connection_url = args[0].safeGet(); - configuration.is_connection_string = false; + if (config.has(config_prefix + ".connection_string")) + { + configuration.connection_url = config.getString(config_prefix + ".connection_string"); + configuration.is_connection_string = true; + configuration.container = config.getString(config_prefix + ".container"); + } + else + { + configuration.connection_url = config.getString(config_prefix + ".storage_account_url"); + configuration.is_connection_string = false; + configuration.container = config.getString(config_prefix + ".container"); + configuration.account_name = config.getString(config_prefix + ".account_name"); + configuration.account_key = config.getString(config_prefix + ".account_key"); + } - configuration.container = args[1].safeGet(); - configuration.blob_path = args[2].safeGet(); - configuration.account_name = args[3].safeGet(); - configuration.account_key = args[4].safeGet(); + if (args.size() > 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Backup AzureBlobStorage requires 1 or 2 arguments: named_collection, [filename]"); + + if (args.size() == 1) + configuration.blob_path = args[0].safeGet(); } else { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Backup AzureBlobStorage requires 3 or 5 arguments: connection string>/(); + configuration.is_connection_string = true; + configuration.container = args[1].safeGet(); + configuration.blob_path = args[2].safeGet(); + } + else if (args.size() == 5) + { + configuration.connection_url = args[0].safeGet(); + configuration.is_connection_string = false; + + configuration.container = args[1].safeGet(); + configuration.blob_path = args[2].safeGet(); + configuration.account_name = args[3].safeGet(); + configuration.account_key = args[4].safeGet(); + + } + else + { + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Backup AzureBlobStorage requires 3 or 5 arguments: connection string>/>>>>>> master }; using AzureClient = Azure::Storage::Blobs::BlobContainerClient; diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index ff4cfe62feb..114a970384b 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -80,7 +80,8 @@ namespace struct UploadPartTask { - std::unique_ptr read_buffer = nullptr; + size_t part_offset; + size_t part_size; std::vector block_ids; bool is_finished = false; std::exception_ptr exception; @@ -182,7 +183,8 @@ namespace try { - task->read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); + task->part_offset = part_offset; + task->part_size = part_size; schedule([this, task, task_finish_notify]() { @@ -206,7 +208,8 @@ namespace else { UploadPartTask task; - task.read_buffer = std::make_unique(create_read_buffer(), part_offset, part_size); + task.part_offset = part_offset; + task.part_size = part_size; processUploadPartRequest(task); block_ids.insert(block_ids.end(),task.block_ids.begin(), task.block_ids.end()); } @@ -219,17 +222,17 @@ namespace ProfileEvents::increment(ProfileEvents::DiskAzureUploadPart); auto block_blob_client = client->GetBlockBlobClient(dest_blob); - - while (!task.read_buffer->eof()) + auto read_buffer = std::make_unique(create_read_buffer(), task.part_offset, task.part_size); + while (!read_buffer->eof()) { - auto size = task.read_buffer->available(); + auto size = read_buffer->available(); if (size > 0) { auto block_id = getRandomASCIIString(64); - Azure::Core::IO::MemoryBodyStream memory(reinterpret_cast(task.read_buffer->position()), size); + Azure::Core::IO::MemoryBodyStream memory(reinterpret_cast(read_buffer->position()), size); block_blob_client.StageBlock(block_id, memory); task.block_ids.emplace_back(block_id); - task.read_buffer->ignore(size); + read_buffer->ignore(size); LOG_TRACE(log, "Writing part. Container: {}, Blob: {}, block_id: {}", dest_container_for_logging, dest_blob, block_id); } } diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp index 67d67ea3fae..c09db0bfb7b 100644 --- a/src/Storages/StorageAzureBlob.cpp +++ b/src/Storages/StorageAzureBlob.cpp @@ -253,17 +253,6 @@ AzureObjectStorage::SettingsPtr StorageAzureBlob::createSettings(ContextPtr loca return settings_ptr; } -std::shared_ptr StorageAzureBlob::createSettingsAsSharedPtr(ContextPtr local_context) -{ - const auto & context_settings = local_context->getSettingsRef(); - auto settings_ptr = std::make_shared(); - settings_ptr->max_single_part_upload_size = context_settings.azure_max_single_part_upload_size; - settings_ptr->max_single_read_retries = context_settings.azure_max_single_read_retries; - settings_ptr->list_object_keys_size = static_cast(context_settings.azure_list_object_keys_size); - - return settings_ptr; -} - void registerStorageAzureBlob(StorageFactory & factory) { factory.registerStorage("AzureBlobStorage", [](const StorageFactory::Arguments & args) diff --git a/src/Storages/StorageAzureBlob.h b/src/Storages/StorageAzureBlob.h index 196983522bf..6fc3c5ce592 100644 --- a/src/Storages/StorageAzureBlob.h +++ b/src/Storages/StorageAzureBlob.h @@ -72,7 +72,6 @@ public: static AzureClientPtr createClient(StorageAzureBlob::Configuration configuration, bool is_read_only); static AzureObjectStorage::SettingsPtr createSettings(ContextPtr local_context); - static std::shared_ptr createSettingsAsSharedPtr(ContextPtr local_context); static void processNamedCollectionResult(StorageAzureBlob::Configuration & configuration, const NamedCollection & collection); diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index 06c18d7468f..22aff39ce87 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -18,12 +18,45 @@ from helpers.mock_servers import start_mock_servers from helpers.test_tools import exec_query_with_retry +def generate_cluster_def(port): + path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "./_gen/named_collections.xml", + ) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write( + f""" + + + DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:{port}/devstoreaccount1; + cont + CSV + + + http://azurite1:{port}/devstoreaccount1 + cont + CSV + devstoreaccount1 + Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + + + +""" + ) + return path + + + @pytest.fixture(scope="module") def cluster(): try: cluster = ClickHouseCluster(__file__) + port = cluster.azurite_port + path = generate_cluster_def(port) cluster.add_instance( "node", + main_configs=[path], with_azurite=True, ) cluster.start() @@ -123,15 +156,6 @@ def delete_all_files(cluster): yield - -def test_create_table_connection_string(cluster): - node = cluster.instances["node"] - azure_query( - node, - f"CREATE TABLE test_create_table_conn_string (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_create_connection_string', 'CSV')", - ) - - def test_backup_restore(cluster): node = cluster.instances["node"] port = cluster.env_variables["AZURITE_PORT"] @@ -159,3 +183,84 @@ def test_backup_restore(cluster): azure_query(node, f"SELECT * from test_simple_write_connection_string_restored") == "1\ta\n" ) + +def test_backup_restore_diff_container(cluster): + node = cluster.instances["node"] + port = cluster.env_variables["AZURITE_PORT"] + azure_query( + node, + f"CREATE TABLE test_simple_write_connection_string_cont1 (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write_c_cont1.csv', 'CSV')", + ) + azure_query( + node, f"INSERT INTO test_simple_write_connection_string_cont1 VALUES (1, 'a')" + ) + backup_destination = f"AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont1', 'test_simple_write_c_backup_cont1.csv')" + azure_query( + node, + f"BACKUP TABLE test_simple_write_connection_string_cont1 TO {backup_destination}", + ) + azure_query( + node, + f"RESTORE TABLE test_simple_write_connection_string_cont1 AS test_simple_write_connection_string_restored_cont1 FROM {backup_destination};", + ) + assert ( + azure_query(node, f"SELECT * from test_simple_write_connection_string_restored_cont1") + == "1\ta\n" + ) + + +def test_backup_restore_with_named_collection_azure_conf1(cluster): + node = cluster.instances["node"] + port = cluster.env_variables["AZURITE_PORT"] + azure_query( + node, + f"CREATE TABLE test_write_connection_string (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write.csv', 'CSV')", + ) + azure_query( + node, f"INSERT INTO test_write_connection_string VALUES (1, 'a')" + ) + print(get_azure_file_content("test_simple_write.csv", port)) + assert get_azure_file_content("test_simple_write.csv", port) == '1,"a"\n' + + backup_destination = f"AzureBlobStorage(azure_conf1, 'test_simple_write_nc_backup.csv')" + azure_query( + node, + f"BACKUP TABLE test_write_connection_string TO {backup_destination}", + ) + print(get_azure_file_content("test_simple_write_nc_backup.csv.backup", port)) + azure_query( + node, + f"RESTORE TABLE test_write_connection_string AS test_write_connection_string_restored FROM {backup_destination};", + ) + assert ( + azure_query(node, f"SELECT * from test_write_connection_string_restored") + == "1\ta\n" + ) + +def test_backup_restore_with_named_collection_azure_conf2(cluster): + node = cluster.instances["node"] + port = cluster.env_variables["AZURITE_PORT"] + azure_query( + node, + f"CREATE TABLE test_write_connection_string_2 (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write_2.csv', 'CSV')", + ) + azure_query( + node, f"INSERT INTO test_write_connection_string_2 VALUES (1, 'a')" + ) + print(get_azure_file_content("test_simple_write_2.csv", port)) + assert get_azure_file_content("test_simple_write_2.csv", port) == '1,"a"\n' + + backup_destination = f"AzureBlobStorage(azure_conf2, 'test_simple_write_nc_backup_2.csv')" + azure_query( + node, + f"BACKUP TABLE test_write_connection_string_2 TO {backup_destination}", + ) + print(get_azure_file_content("test_simple_write_nc_backup_2.csv.backup", port)) + azure_query( + node, + f"RESTORE TABLE test_write_connection_string_2 AS test_write_connection_string_restored_2 FROM {backup_destination};", + ) + assert ( + azure_query(node, f"SELECT * from test_write_connection_string_restored_2") + == "1\ta\n" + ) From ce6df0fb137a16c574f5be561205bd171e9ce3a5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 4 Feb 2024 10:37:05 +0000 Subject: [PATCH 046/103] Automatic style fix --- .../test.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index 22aff39ce87..a7c7b439560 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -47,7 +47,6 @@ def generate_cluster_def(port): return path - @pytest.fixture(scope="module") def cluster(): try: @@ -156,6 +155,7 @@ def delete_all_files(cluster): yield + def test_backup_restore(cluster): node = cluster.instances["node"] port = cluster.env_variables["AZURITE_PORT"] @@ -184,6 +184,7 @@ def test_backup_restore(cluster): == "1\ta\n" ) + def test_backup_restore_diff_container(cluster): node = cluster.instances["node"] port = cluster.env_variables["AZURITE_PORT"] @@ -204,8 +205,10 @@ def test_backup_restore_diff_container(cluster): f"RESTORE TABLE test_simple_write_connection_string_cont1 AS test_simple_write_connection_string_restored_cont1 FROM {backup_destination};", ) assert ( - azure_query(node, f"SELECT * from test_simple_write_connection_string_restored_cont1") - == "1\ta\n" + azure_query( + node, f"SELECT * from test_simple_write_connection_string_restored_cont1" + ) + == "1\ta\n" ) @@ -216,13 +219,13 @@ def test_backup_restore_with_named_collection_azure_conf1(cluster): node, f"CREATE TABLE test_write_connection_string (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write.csv', 'CSV')", ) - azure_query( - node, f"INSERT INTO test_write_connection_string VALUES (1, 'a')" - ) + azure_query(node, f"INSERT INTO test_write_connection_string VALUES (1, 'a')") print(get_azure_file_content("test_simple_write.csv", port)) assert get_azure_file_content("test_simple_write.csv", port) == '1,"a"\n' - backup_destination = f"AzureBlobStorage(azure_conf1, 'test_simple_write_nc_backup.csv')" + backup_destination = ( + f"AzureBlobStorage(azure_conf1, 'test_simple_write_nc_backup.csv')" + ) azure_query( node, f"BACKUP TABLE test_write_connection_string TO {backup_destination}", @@ -233,10 +236,11 @@ def test_backup_restore_with_named_collection_azure_conf1(cluster): f"RESTORE TABLE test_write_connection_string AS test_write_connection_string_restored FROM {backup_destination};", ) assert ( - azure_query(node, f"SELECT * from test_write_connection_string_restored") - == "1\ta\n" + azure_query(node, f"SELECT * from test_write_connection_string_restored") + == "1\ta\n" ) + def test_backup_restore_with_named_collection_azure_conf2(cluster): node = cluster.instances["node"] port = cluster.env_variables["AZURITE_PORT"] @@ -244,13 +248,13 @@ def test_backup_restore_with_named_collection_azure_conf2(cluster): node, f"CREATE TABLE test_write_connection_string_2 (key UInt64, data String) Engine = AzureBlobStorage('{cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', 'test_simple_write_2.csv', 'CSV')", ) - azure_query( - node, f"INSERT INTO test_write_connection_string_2 VALUES (1, 'a')" - ) + azure_query(node, f"INSERT INTO test_write_connection_string_2 VALUES (1, 'a')") print(get_azure_file_content("test_simple_write_2.csv", port)) assert get_azure_file_content("test_simple_write_2.csv", port) == '1,"a"\n' - backup_destination = f"AzureBlobStorage(azure_conf2, 'test_simple_write_nc_backup_2.csv')" + backup_destination = ( + f"AzureBlobStorage(azure_conf2, 'test_simple_write_nc_backup_2.csv')" + ) azure_query( node, f"BACKUP TABLE test_write_connection_string_2 TO {backup_destination}", @@ -261,6 +265,6 @@ def test_backup_restore_with_named_collection_azure_conf2(cluster): f"RESTORE TABLE test_write_connection_string_2 AS test_write_connection_string_restored_2 FROM {backup_destination};", ) assert ( - azure_query(node, f"SELECT * from test_write_connection_string_restored_2") - == "1\ta\n" + azure_query(node, f"SELECT * from test_write_connection_string_restored_2") + == "1\ta\n" ) From 23b9f43d4f56a99e6c6b324910e9c69a665ce92d Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sun, 4 Feb 2024 16:18:17 +0100 Subject: [PATCH 047/103] Fix style --- src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 114a970384b..2f4c9374def 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -28,7 +28,6 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int INVALID_CONFIG_PARAMETER; extern const int AZURE_BLOB_STORAGE_ERROR; } From f036948f91882f6a9b594fe1393f82f2122f7da4 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 5 Feb 2024 10:09:48 +0100 Subject: [PATCH 048/103] Fix clang tidy build --- src/Backups/BackupIO_AzureBlobStorage.cpp | 12 ++++++------ src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 27928e871ce..52ce20d5108 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -89,8 +89,8 @@ std::unique_ptr BackupReaderAzureBlobStorage::readFile(const key = file_name; } return std::make_unique( - client, key, read_settings, settings.get()->max_single_read_retries, - settings.get()->max_single_download_retries); + client, key, read_settings, settings->max_single_read_retries, + settings->max_single_download_retries); } void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup, @@ -257,8 +257,8 @@ std::unique_ptr BackupWriterAzureBlobStorage::readFile(const String } return std::make_unique( - client, key, read_settings, settings.get()->max_single_read_retries, - settings.get()->max_single_download_retries); + client, key, read_settings, settings->max_single_read_retries, + settings->max_single_download_retries); } std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const String & file_name) @@ -275,8 +275,8 @@ std::unique_ptr BackupWriterAzureBlobStorage::writeFile(const Strin return std::make_unique( client, key, - settings.get()->max_single_part_upload_size, - settings.get()->max_unexpected_write_error_retries, + settings->max_single_part_upload_size, + settings->max_unexpected_write_error_retries, DBMS_DEFAULT_BUFFER_SIZE, write_settings); } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 2f4c9374def..9162f371b5b 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -97,7 +97,7 @@ namespace void calculatePartSize() { - auto max_upload_part_size = settings.get()->max_upload_part_size; + auto max_upload_part_size = settings->max_upload_part_size; if (!max_upload_part_size) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_upload_part_size must not be 0"); /// We've calculated the size of a normal part (the final part can be smaller). @@ -292,7 +292,7 @@ void copyAzureBlobStorageFile( bool for_disk_azure_blob_storage) { - if (settings.get()->use_native_copy) + if (settings->use_native_copy) { ProfileEvents::increment(ProfileEvents::AzureCopyObject); if (for_disk_azure_blob_storage) @@ -302,7 +302,7 @@ void copyAzureBlobStorageFile( auto block_blob_client_dest = dest_client->GetBlockBlobClient(dest_blob); auto source_uri = block_blob_client_src.GetUrl(); - if (size < settings.get()->max_single_part_copy_size) + if (size < settings->max_single_part_copy_size) { block_blob_client_dest.CopyFromUri(source_uri); } @@ -326,8 +326,8 @@ void copyAzureBlobStorageFile( LOG_TRACE(&Poco::Logger::get("copyAzureBlobStorageFile"), "Reading from Container: {}, Blob: {}", src_container_for_logging, src_blob); auto create_read_buffer = [&] { - return std::make_unique(src_client, src_blob, read_settings, settings.get()->max_single_read_retries, - settings.get()->max_single_download_retries); + return std::make_unique(src_client, src_blob, read_settings, settings->max_single_read_retries, + settings->max_single_download_retries); }; UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container_for_logging, dest_blob, settings, schedule, for_disk_azure_blob_storage, &Poco::Logger::get("copyAzureBlobStorageFile")}; From fd2fdcdb4b33fe91227d7d623aea24a0356f29f0 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 5 Feb 2024 11:23:37 +0100 Subject: [PATCH 049/103] Fixed unwanted dereferencing --- src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index 9162f371b5b..4714c795927 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -58,7 +58,7 @@ namespace , schedule(schedule_) , for_disk_azure_blob_storage(for_disk_azure_blob_storage_) , log(log_) - , max_single_part_upload_size(settings_.get()->max_single_part_upload_size) + , max_single_part_upload_size(settings_->max_single_part_upload_size) { } From 45bcc04d98ebaaf511ccf2920c1eaca98ee3a10c Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 5 Feb 2024 12:56:03 +0000 Subject: [PATCH 050/103] Use ConnectionPoolPtr instead of raw pointer --- src/Processors/QueryPlan/ReadFromRemote.cpp | 2 +- src/Processors/QueryPlan/ReadFromRemote.h | 2 +- src/QueryPipeline/RemoteQueryExecutor.cpp | 2 +- src/QueryPipeline/RemoteQueryExecutor.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 379ea950081..1edb672aa38 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -439,7 +439,7 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder void ReadFromParallelRemoteReplicasStep::addPipeForSingeReplica( - Pipes & pipes, IConnectionPool * pool, IConnections::ReplicaInfo replica_info) + Pipes & pipes, const ConnectionPoolPtr & pool, IConnections::ReplicaInfo replica_info) { bool add_agg_info = stage == QueryProcessingStage::WithMergeableState; bool add_totals = false; diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index eb3bcd12cc3..67fd9d24261 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -90,7 +90,7 @@ public: void enforceAggregationInOrder(); private: - void addPipeForSingeReplica(Pipes & pipes, IConnectionPool * pool, IConnections::ReplicaInfo replica_info); + void addPipeForSingeReplica(Pipes & pipes, const ConnectionPoolPtr & pool, IConnections::ReplicaInfo replica_info); ClusterPtr cluster; ASTPtr query_ast; diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 8a4bee1d8af..1a68c9d4471 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -64,7 +64,7 @@ RemoteQueryExecutor::RemoteQueryExecutor( } RemoteQueryExecutor::RemoteQueryExecutor( - IConnectionPool * pool, + ConnectionPoolPtr pool, const String & query_, const Block & header_, ContextPtr context_, diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index e617deef7e8..6b1539bd08e 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -54,7 +54,7 @@ public: /// Takes a connection pool for a node (not cluster) RemoteQueryExecutor( - IConnectionPool * pool, + ConnectionPoolPtr pool, const String & query_, const Block & header_, ContextPtr context_, From 576cfdbf5cb4428868a03f1b78da760f776010fd Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 5 Feb 2024 14:27:07 +0000 Subject: [PATCH 051/103] Includes cleanup --- src/Client/Connection.h | 1 - src/Client/IConnections.h | 2 -- src/Processors/QueryPlan/ReadFromRemote.cpp | 2 +- src/Processors/QueryPlan/ReadFromRemote.h | 3 --- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Client/Connection.h b/src/Client/Connection.h index e93a7539d15..5d0411027a1 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -19,7 +19,6 @@ #include -#include #include #include "config.h" diff --git a/src/Client/IConnections.h b/src/Client/IConnections.h index ee17d198fc3..ebc71511834 100644 --- a/src/Client/IConnections.h +++ b/src/Client/IConnections.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 1edb672aa38..6764e095088 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -12,7 +12,7 @@ #include #include #include -#include "Common/logger_useful.h" +#include #include #include #include diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index 67fd9d24261..498d584e85a 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -9,9 +9,6 @@ namespace DB { - -class IConnectionPool; - class Throttler; using ThrottlerPtr = std::shared_ptr; From 8748d8c537c96dccb31df76099425b3321ff9e02 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 5 Feb 2024 15:10:51 +0000 Subject: [PATCH 052/103] UNEXPECTED_CLUSTER error for cluster with more than 1 shard --- src/Common/ErrorCodes.cpp | 1 + src/Interpreters/ClusterProxy/executeQuery.cpp | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 2e154ddb32d..8e81a626b41 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -593,6 +593,7 @@ M(711, FILECACHE_ACCESS_DENIED) \ M(712, TOO_MANY_MATERIALIZED_VIEWS) \ M(713, BROKEN_PROJECTION) \ + M(714, UNEXPECTED_CLUSTER) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 023ed6c7b61..33b86854ba9 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -32,6 +32,7 @@ namespace ErrorCodes extern const int TOO_LARGE_DISTRIBUTED_DEPTH; extern const int LOGICAL_ERROR; extern const int CLUSTER_DOESNT_EXIST; + extern const int UNEXPECTED_CLUSTER; } namespace ClusterProxy @@ -399,8 +400,10 @@ void executeQueryWithParallelReplicas( } else { - // todo: add error and exception for this case - chassert(not_optimized_cluster->getShardCount() == 1); + if (not_optimized_cluster->getShardCount() > 1) + throw DB::Exception( + ErrorCodes::UNEXPECTED_CLUSTER, + "`cluster_for_parallel_replicas` setting refers to cluster with several shards. Expected a cluster with one shard"); } auto coordinator = std::make_shared( From a50a14062621733b938cb9fb986c18d698ee3c36 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 5 Feb 2024 15:29:08 +0000 Subject: [PATCH 053/103] Test for UNEXPECTED_CLUSTER --- .../02982_parallel_replicas_unexpected_cluster.reference | 0 .../02982_parallel_replicas_unexpected_cluster.sql | 8 ++++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.reference create mode 100644 tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql diff --git a/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.reference b/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql b/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql new file mode 100644 index 00000000000..210b7d2a18a --- /dev/null +++ b/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql @@ -0,0 +1,8 @@ +DROP TABLE IF EXISTS test_unexpected_cluster; +CREATE TABLE test_unexpected_cluster (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); +INSERT INTO test_unexpected_cluster SELECT * FROM numbers(10); + +SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=2, cluster_for_parallel_replicas='test_cluster_two_shards', parallel_replicas_for_non_replicated_merge_tree=1; +SELECT count() FROM test_unexpected_cluster WHERE NOT ignore(*); -- { serverError UNEXPECTED_CLUSTER } + +DROP TABLE test_unexpected_cluster; From dab078f7d24bca61a1cf05dc0294fdee48103202 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 5 Feb 2024 20:21:48 +0000 Subject: [PATCH 054/103] Profile events --- src/Common/ProfileEvents.cpp | 2 ++ src/QueryPipeline/RemoteQueryExecutor.cpp | 6 ++++++ .../MergeTree/ParallelReplicasReadingCoordinator.cpp | 8 ++++---- .../02769_parallel_replicas_unavailable_shards.sql | 4 +--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 8782f895f3f..0218545c3a4 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -616,6 +616,8 @@ The server successfully detected this situation and will download merged part fr M(InterfacePostgreSQLReceiveBytes, "Number of bytes received through PostgreSQL interfaces") \ \ M(ParallelReplicasUsedCount, "Number of replicas used to execute a query with task-based parallel replicas") \ + M(ParallelReplicasAvailableCount, "Number of replicas available to execute a query with task-based parallel replicas") \ + M(ParallelReplicasUnavailableCount, "Number of replicas which was chosen, but unavailable, to execute a query with task-based parallel replicas") \ #ifdef APPLY_FOR_EXTERNAL_EVENTS #define APPLY_FOR_EVENTS(M) APPLY_FOR_BUILTIN_EVENTS(M) APPLY_FOR_EXTERNAL_EVENTS(M) diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 1a68c9d4471..e44749dfb97 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -30,6 +30,7 @@ namespace ProfileEvents extern const Event SuspendSendingQueryToShard; extern const Event ReadTaskRequestsReceived; extern const Event MergeTreeReadTaskRequestsReceived; + extern const Event ParallelReplicasAvailableCount; } namespace DB @@ -97,7 +98,12 @@ RemoteQueryExecutor::RemoteQueryExecutor( std::vector connection_entries; if (!result.entry.isNull() && result.is_usable) + { + if (extension_ && extension_->parallel_reading_coordinator) + ProfileEvents::increment(ProfileEvents::ParallelReplicasAvailableCount); + connection_entries.emplace_back(std::move(result.entry)); + } auto res = std::make_unique(std::move(connection_entries), context->getSettingsRef(), throttler); if (extension_ && extension_->replica_info) diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index abc51bde3fb..2fe237efdc7 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -97,11 +97,9 @@ extern const Event ParallelReplicasCollectingOwnedSegmentsMicroseconds; extern const Event ParallelReplicasReadAssignedMarks; extern const Event ParallelReplicasReadUnassignedMarks; extern const Event ParallelReplicasReadAssignedForStealingMarks; -} -namespace ProfileEvents -{ - extern const Event ParallelReplicasUsedCount; +extern const Event ParallelReplicasUsedCount; +extern const Event ParallelReplicasUnavailableCount; } namespace DB @@ -1025,6 +1023,8 @@ ParallelReadResponse ParallelReplicasReadingCoordinator::handleRequest(ParallelR void ParallelReplicasReadingCoordinator::markReplicaAsUnavailable(size_t replica_number) { + ProfileEvents::increment(ProfileEvents::ParallelReplicasUnavailableCount); + std::lock_guard lock(mutex); if (!pimpl) diff --git a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql index be200353f06..1a75e000349 100644 --- a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql +++ b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql @@ -2,8 +2,6 @@ DROP TABLE IF EXISTS test_parallel_replicas_unavailable_shards; CREATE TABLE test_parallel_replicas_unavailable_shards (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO test_parallel_replicas_unavailable_shards SELECT * FROM numbers(10); -SYSTEM FLUSH LOGS; - SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1; SET send_logs_level='error'; SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79'; @@ -11,6 +9,6 @@ SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(* SYSTEM FLUSH LOGS; SET allow_experimental_parallel_reading_from_replicas=0; -SELECT count() FROM system.text_log WHERE yesterday() <= event_date AND query_id in (select query_id from system.query_log where log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79' and current_database = currentDatabase()) and message LIKE '%Replica number % is unavailable%'; +SELECT ProfileEvents['ParallelReplicasUnavailableCount'] FROM system.query_log WHERE yesterday() <= event_date AND query_id in (select query_id from system.query_log where log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79' and current_database = currentDatabase()) and type = 'QueryFinish' and query_id == initial_query_id; DROP TABLE test_parallel_replicas_unavailable_shards; From b43f90fce209351972fafd420e1b62d58159ba81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 6 Feb 2024 19:55:01 +0100 Subject: [PATCH 055/103] Fix query start time on non initial queries --- src/Interpreters/executeQuery.cpp | 18 ++++-------- .../02985_shard_query_start_time.reference | 2 ++ .../02985_shard_query_start_time.sql | 29 +++++++++++++++++++ 3 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 tests/queries/0_stateless/02985_shard_query_start_time.reference create mode 100644 tests/queries/0_stateless/02985_shard_query_start_time.sql diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 1787f627c2e..5c16a5d800f 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -663,15 +663,17 @@ static std::tuple executeQueryImpl( if (query_span && query_span->trace_id != UUID{}) LOG_TRACE(getLogger("executeQuery"), "Query span trace_id for opentelemetry log: {}", query_span->trace_id); + /// Used for logging query start time in system.query_log auto query_start_time = std::chrono::system_clock::now(); - /// Used to set the watch in QueryStatus and the output formats. It is not based on query_start_time as that might be based on - /// the value passed by the client + /// Used for: + /// * Setting the watch in QueryStatus (controls timeouts and progress) and the output formats + /// * Logging query duration (system.query_log) Stopwatch start_watch{CLOCK_MONOTONIC}; const auto & client_info = context->getClientInfo(); - if (!internal) + if (!internal && client_info.initial_query_start_time == 0) { // If it's not an internal query and we don't see an initial_query_start_time yet, initialize it // to current time. Internal queries are those executed without an independent client context, @@ -679,15 +681,7 @@ static std::tuple executeQueryImpl( // possible to have unset initial_query_start_time for non-internal and non-initial queries. For // example, the query is from an initiator that is running an old version of clickhouse. // On the other hand, if it's initialized then take it as the start of the query - if (client_info.initial_query_start_time == 0) - { - context->setInitialQueryStartTime(query_start_time); - } - else - { - query_start_time = std::chrono::time_point( - std::chrono::microseconds{client_info.initial_query_start_time_microseconds}); - } + context->setInitialQueryStartTime(query_start_time); } assert(internal || CurrentThread::get().getQueryContext()); diff --git a/tests/queries/0_stateless/02985_shard_query_start_time.reference b/tests/queries/0_stateless/02985_shard_query_start_time.reference new file mode 100644 index 00000000000..1957f3a9604 --- /dev/null +++ b/tests/queries/0_stateless/02985_shard_query_start_time.reference @@ -0,0 +1,2 @@ +1 1 +1 1 diff --git a/tests/queries/0_stateless/02985_shard_query_start_time.sql b/tests/queries/0_stateless/02985_shard_query_start_time.sql new file mode 100644 index 00000000000..b0d8d2b6e53 --- /dev/null +++ b/tests/queries/0_stateless/02985_shard_query_start_time.sql @@ -0,0 +1,29 @@ +DROP TABLE IF EXISTS sharded_table; +CREATE TABLE sharded_table (dummy UInt8) ENGINE = Distributed('test_cluster_two_shards', 'system', 'one'); + +SELECT * FROM sharded_table FORMAT Null SETTINGS log_comment='02985_shard_query_start_time_query_1'; + +SYSTEM FLUSH LOGS; + +-- We do not test for query_start_time because that would conflict pretty easily +WITH +( + SELECT + (query_id, query_start_time_microseconds) + FROM + system.query_log + WHERE + event_date >= yesterday() + AND current_database = currentDatabase() + AND log_comment = '02985_shard_query_start_time_query_1' + AND type = 'QueryFinish' + ORDER BY query_start_time_microseconds DESC + LIMIT 1 +) AS id_and_start_tuple +SELECT + query_start_time_microseconds > initial_query_start_time_microseconds, + initial_query_start_time_microseconds = id_and_start_tuple.2 +FROM + system.query_log +WHERE + NOT is_initial_query AND initial_query_id = id_and_start_tuple.1; From 10082399d540a1ab5cee3ff381856193afc9ad22 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 7 Feb 2024 00:52:25 +0000 Subject: [PATCH 056/103] Minor review fixes --- src/Client/ConnectionPool.h | 22 ++++++++++++--------- src/Common/ProfileEvents.cpp | 2 +- src/Processors/QueryPlan/ReadFromRemote.cpp | 8 ++++---- src/QueryPipeline/RemoteQueryExecutor.cpp | 2 +- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h index 1886a0431a5..8e707e8190f 100644 --- a/src/Client/ConnectionPool.h +++ b/src/Client/ConnectionPool.h @@ -27,6 +27,9 @@ class IConnectionPool : private boost::noncopyable public: using Entry = PoolBase::Entry; + IConnectionPool() = default; + IConnectionPool(String host_, UInt16 port_) : host(host_), port(port_), address(host + ":" + toString(port_)) {} + virtual ~IConnectionPool() = default; /// Selects the connection to work. @@ -36,7 +39,15 @@ public: const Settings & settings, bool force_connected = true) = 0; + const std::string & getHost() const { return host; } + UInt16 getPort() const { return port; } + const String & getAddress() const { return address; } virtual Priority getPriority() const { return Priority{1}; } + +protected: + const String host; + const UInt16 port = 0; + const String address; }; using ConnectionPoolPtr = std::shared_ptr; @@ -63,10 +74,9 @@ public: Protocol::Compression compression_, Protocol::Secure secure_, Priority priority_ = Priority{1}) - : Base(max_connections_, + : IConnectionPool(host_, port_), + Base(max_connections_, getLogger("ConnectionPool (" + host_ + ":" + toString(port_) + ")")), - host(host_), - port(port_), default_database(default_database_), user(user_), password(password_), @@ -99,10 +109,6 @@ public: return entry; } - const std::string & getHost() const - { - return host; - } std::string getDescription() const { return host + ":" + toString(port); @@ -125,8 +131,6 @@ protected: } private: - String host; - UInt16 port; String default_database; String user; String password; diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 0218545c3a4..fc30a4e0794 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -617,7 +617,7 @@ The server successfully detected this situation and will download merged part fr \ M(ParallelReplicasUsedCount, "Number of replicas used to execute a query with task-based parallel replicas") \ M(ParallelReplicasAvailableCount, "Number of replicas available to execute a query with task-based parallel replicas") \ - M(ParallelReplicasUnavailableCount, "Number of replicas which was chosen, but unavailable, to execute a query with task-based parallel replicas") \ + M(ParallelReplicasUnavailableCount, "Number of replicas which was chosen, but found to be unavailable during query execution with task-based parallel replicas") \ #ifdef APPLY_FOR_EXTERNAL_EVENTS #define APPLY_FOR_EVENTS(M) APPLY_FOR_BUILTIN_EVENTS(M) APPLY_FOR_EXTERNAL_EVENTS(M) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 6764e095088..5707eb2e9c6 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -375,10 +375,11 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( , storage_limits(std::move(storage_limits_)) , log(log_) { - std::vector description; + chassert(cluster->getShardCount() == 1); - for (const auto & address : cluster->getShardsAddresses()) - description.push_back(fmt::format("Replica: {}", address[0].host_name)); + std::vector description; + for (const auto & pool : cluster->getShardsInfo().front().per_replica_pools) + description.push_back(fmt::format("Replica: {}", pool->getHost())); setStepDescription(boost::algorithm::join(description, ", ")); } @@ -412,7 +413,6 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder all_replicas_count = shard.getAllNodeCount(); } - chassert(cluster->getShardCount() == 1); auto shuffled_pool = shard.pool->getShuffledPools(current_settings); shuffled_pool.resize(all_replicas_count); diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index e44749dfb97..46616905bcb 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -105,7 +105,7 @@ RemoteQueryExecutor::RemoteQueryExecutor( connection_entries.emplace_back(std::move(result.entry)); } - auto res = std::make_unique(std::move(connection_entries), context->getSettingsRef(), throttler); + auto res = std::make_unique(std::move(connection_entries), current_settings, throttler); if (extension_ && extension_->replica_info) res->setReplicaInfo(*extension_->replica_info); From 755298838fba555fa5dba644277e151841d1cf67 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 7 Feb 2024 01:28:27 +0000 Subject: [PATCH 057/103] Preserve replicas order for data locality --- src/Processors/QueryPlan/ReadFromRemote.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 5707eb2e9c6..91c81d619a7 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -413,7 +413,17 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder all_replicas_count = shard.getAllNodeCount(); } - auto shuffled_pool = shard.pool->getShuffledPools(current_settings); + + std::vector shuffled_pool; + if (all_replicas_count < shard.getAllNodeCount()) + shuffled_pool = shard.pool->getShuffledPools(current_settings); + else + { + /// try to preserve replicas order if all replicas in cluster are used for query execution + /// it's important for data locality during query execution + auto priority_func = [](size_t i) { return Priority{static_cast(i)}; }; + shuffled_pool = shard.pool->getShuffledPools(current_settings, priority_func); + } shuffled_pool.resize(all_replicas_count); for (size_t i=0; i < all_replicas_count; ++i) From 4f153b59c06afa2c75e9f43b1279900d594359cd Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 7 Feb 2024 01:30:48 +0000 Subject: [PATCH 058/103] Minor fix --- src/Processors/QueryPlan/ReadFromRemote.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 91c81d619a7..93c73a66b78 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -416,7 +416,10 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder std::vector shuffled_pool; if (all_replicas_count < shard.getAllNodeCount()) + { shuffled_pool = shard.pool->getShuffledPools(current_settings); + shuffled_pool.resize(all_replicas_count); + } else { /// try to preserve replicas order if all replicas in cluster are used for query execution @@ -424,7 +427,6 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder auto priority_func = [](size_t i) { return Priority{static_cast(i)}; }; shuffled_pool = shard.pool->getShuffledPools(current_settings, priority_func); } - shuffled_pool.resize(all_replicas_count); for (size_t i=0; i < all_replicas_count; ++i) { From 41d624317064bc2a2784b6afaf3b3f22548c2f2b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 7 Feb 2024 12:43:26 +0000 Subject: [PATCH 059/103] Fix PreparedSets --- src/Interpreters/PreparedSets.cpp | 9 +-------- src/Interpreters/PreparedSets.h | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index 946eef08ce3..76f75cde1dc 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -205,18 +205,11 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) } } - if (!set_and_key->set->hasSetElements()) - set_and_key->set->fillSetElements(); - - return buildSetInplace(context); -} - -SetPtr FutureSetFromSubquery::buildSetInplace(const ContextPtr & context) -{ auto plan = build(context); if (!plan) return nullptr; + set_and_key->set->fillSetElements(); auto builder = plan->buildQueryPipeline(QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context)); auto pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); pipeline.complete(std::make_shared(Block())); diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index a65f30351db..3419d3b6839 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -111,9 +111,9 @@ public: SetPtr get() const override; DataTypes getTypes() const override; SetPtr buildOrderedSetInplace(const ContextPtr & context) override; - SetPtr buildSetInplace(const ContextPtr & context); std::unique_ptr build(const ContextPtr & context); + void buildSetInplace(const ContextPtr & context); QueryTreeNodePtr detachQueryTree() { return std::move(query_tree); } void setQueryPlan(std::unique_ptr source_); From 9a4dbc843ac26165a876395ae61983b3dcc32ae0 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 7 Feb 2024 17:28:09 +0000 Subject: [PATCH 060/103] validate type of arguments for minmax secondary index --- .../MergeTree/MergeTreeIndexMinMax.cpp | 16 ++++++++- ..._minmax_index_aggregate_function.reference | 6 ++++ .../02985_minmax_index_aggregate_function.sql | 36 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02985_minmax_index_aggregate_function.reference create mode 100644 tests/queries/0_stateless/02985_minmax_index_aggregate_function.sql diff --git a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp index b1f8e09be9f..20dfed8cf8f 100644 --- a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp @@ -15,6 +15,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } @@ -217,7 +218,20 @@ MergeTreeIndexPtr minmaxIndexCreator( return std::make_shared(index); } -void minmaxIndexValidator(const IndexDescription & /* index */, bool /* attach */) +void minmaxIndexValidator(const IndexDescription & index, bool attach) { + if (attach) + return; + + for (const auto & column : index.sample_block) + { + if (!column.type->isComparable()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Data type of argument for minmax index must be comparable, got {} type for column {} instead", + column.type->getName(), column.name); + } + } } + } diff --git a/tests/queries/0_stateless/02985_minmax_index_aggregate_function.reference b/tests/queries/0_stateless/02985_minmax_index_aggregate_function.reference new file mode 100644 index 00000000000..e71eb4f0d57 --- /dev/null +++ b/tests/queries/0_stateless/02985_minmax_index_aggregate_function.reference @@ -0,0 +1,6 @@ +1 +5 10 +6 11 +7 12 +8 13 +9 14 diff --git a/tests/queries/0_stateless/02985_minmax_index_aggregate_function.sql b/tests/queries/0_stateless/02985_minmax_index_aggregate_function.sql new file mode 100644 index 00000000000..7d35c1b310b --- /dev/null +++ b/tests/queries/0_stateless/02985_minmax_index_aggregate_function.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS t_index_agg_func; + +CREATE TABLE t_index_agg_func +( + id UInt64, + v AggregateFunction(avg, UInt64), + INDEX idx_v v TYPE minmax GRANULARITY 1 +) +ENGINE = AggregatingMergeTree ORDER BY id +SETTINGS index_granularity = 4; -- { serverError BAD_ARGUMENTS } + +CREATE TABLE t_index_agg_func +( + id UInt64, + v AggregateFunction(avg, UInt64), +) +ENGINE = AggregatingMergeTree ORDER BY id +SETTINGS index_granularity = 4; + +ALTER TABLE t_index_agg_func ADD INDEX idx_v v TYPE minmax GRANULARITY 1; -- { serverError BAD_ARGUMENTS } + +ALTER TABLE t_index_agg_func ADD INDEX idx_v finalizeAggregation(v) TYPE minmax GRANULARITY 1; + +INSERT INTO t_index_agg_func SELECT number % 10, initializeAggregation('avgState', toUInt64(number % 20)) FROM numbers(1000); +INSERT INTO t_index_agg_func SELECT number % 10, initializeAggregation('avgState', toUInt64(number % 20)) FROM numbers(1000, 1000); + +OPTIMIZE TABLE t_index_agg_func FINAL; + +SELECT count() FROM system.parts WHERE table = 't_index_agg_func' AND database = currentDatabase() AND active; + +SET force_data_skipping_indices = 'idx_v'; +SET use_skip_indexes_if_final = 1; + +SELECT id, finalizeAggregation(v) AS vv FROM t_index_agg_func FINAL WHERE vv >= 10 ORDER BY id; + +DROP TABLE t_index_agg_func; From 12a71375b8a482ef9f64bcc970b5e1ae03758daa Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 8 Feb 2024 00:44:21 +0100 Subject: [PATCH 061/103] Fix skipping unused shards with analyzer. --- src/Interpreters/ActionsDAG.cpp | 19 +++++++++++++------ src/Interpreters/ActionsDAG.h | 1 + .../evaluateConstantExpression.cpp | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 6512def9202..a1858916ca7 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -605,7 +605,7 @@ ActionsDAGPtr ActionsDAG::cloneSubDAG(const NodeRawConstPtrs & outputs, bool rem return actions; } -static ColumnWithTypeAndName executeActionForHeader(const ActionsDAG::Node * node, ColumnsWithTypeAndName arguments) +static ColumnWithTypeAndName executeActionForPartialResult(const ActionsDAG::Node * node, ColumnsWithTypeAndName arguments, size_t input_rows_count) { ColumnWithTypeAndName res_column; res_column.type = node->result_type; @@ -615,7 +615,7 @@ static ColumnWithTypeAndName executeActionForHeader(const ActionsDAG::Node * nod { case ActionsDAG::ActionType::FUNCTION: { - res_column.column = node->function->execute(arguments, res_column.type, 0, true); + res_column.column = node->function->execute(arguments, res_column.type, input_rows_count, true); break; } @@ -628,13 +628,17 @@ static ColumnWithTypeAndName executeActionForHeader(const ActionsDAG::Node * nod if (!array) throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN of not array nor map: {}", node->result_name); - res_column.column = array->getDataPtr()->cloneEmpty(); + res_column.column = array->getDataPtr(); + if (input_rows_count < array->size()) + res_column.column = res_column.column->cloneResized(array->getOffsets()[input_rows_count - 1]); break; } case ActionsDAG::ActionType::COLUMN: { - res_column.column = node->column->cloneResized(0); + res_column.column = node->column; + if (input_rows_count < res_column.column->size()) + res_column.column = res_column.column->cloneResized(input_rows_count); break; } @@ -681,7 +685,7 @@ Block ActionsDAG::updateHeader(Block header) const ColumnsWithTypeAndName result_columns; try { - result_columns = evaluatePartialResult(node_to_column, outputs, true); + result_columns = evaluatePartialResult(node_to_column, outputs, /* input_rows_count= */ 0, /* throw_on_error= */ true); } catch (Exception & e) { @@ -710,8 +714,11 @@ Block ActionsDAG::updateHeader(Block header) const ColumnsWithTypeAndName ActionsDAG::evaluatePartialResult( IntermediateExecutionResult & node_to_column, const NodeRawConstPtrs & outputs, + size_t input_rows_count, bool throw_on_error) { + chassert(input_rows_count <= 1); /// evaluatePartialResult() should be used only to evaluate headers or constants + ColumnsWithTypeAndName result_columns; result_columns.reserve(outputs.size()); @@ -768,7 +775,7 @@ ColumnsWithTypeAndName ActionsDAG::evaluatePartialResult( node->result_name); if (node->type != ActionsDAG::ActionType::INPUT && has_all_arguments) - node_to_column[node] = executeActionForHeader(node, std::move(arguments)); + node_to_column[node] = executeActionForPartialResult(node, std::move(arguments), input_rows_count); } } diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 45f6e5cc717..e6272ac9f60 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -278,6 +278,7 @@ public: static ColumnsWithTypeAndName evaluatePartialResult( IntermediateExecutionResult & node_to_column, const NodeRawConstPtrs & outputs, + size_t input_rows_count, bool throw_on_error); /// For apply materialize() function for every output. diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index af8bd19370b..00d36750cc1 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -661,7 +661,7 @@ namespace const ActionsDAG::NodeRawConstPtrs & target_expr, ConjunctionMap && conjunction) { - auto columns = ActionsDAG::evaluatePartialResult(conjunction, target_expr, false); + auto columns = ActionsDAG::evaluatePartialResult(conjunction, target_expr, /* input_rows_count= */ 1, /* throw_on_error= */ false); for (const auto & column : columns) if (!column.column) return {}; From ece4febe3265e542f979d57bd435127a8243bf0f Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Wed, 7 Feb 2024 01:35:48 +0000 Subject: [PATCH 062/103] Insert synchronously if dependent MV deduplication is enabled --- src/Interpreters/executeQuery.cpp | 2 + src/Server/TCPHandler.cpp | 2 +- ...c_inserts_for_dependent_mv_dedup.reference | 1 + ...e_async_inserts_for_dependent_mv_dedup.sql | 46 +++++++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02985_disable_async_inserts_for_dependent_mv_dedup.reference create mode 100644 tests/queries/0_stateless/02985_disable_async_inserts_for_dependent_mv_dedup.sql diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 1787f627c2e..ce6d1de4af4 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -933,6 +933,8 @@ static std::tuple executeQueryImpl( reason = "asynchronous insert queue is not configured"; else if (insert_query->select) reason = "insert query has select"; + else if (settings.deduplicate_blocks_in_dependent_materialized_views) + reason = "dependent materialized views block deduplication is enabled"; else if (insert_query->hasInlinedData()) async_insert = true; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 2c4e9c1e3b2..e1086ac5833 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -933,7 +933,7 @@ void TCPHandler::processInsertQuery() if (auto table = DatabaseCatalog::instance().tryGetTable(insert_query.table_id, query_context)) async_insert_enabled |= table->areAsynchronousInsertsEnabled(); - if (insert_queue && async_insert_enabled && !insert_query.select) + if (insert_queue && async_insert_enabled && !insert_query.select && !settings.deduplicate_blocks_in_dependent_materialized_views) { auto result = processAsyncInsertQuery(*insert_queue); if (result.status == AsynchronousInsertQueue::PushResult::OK) diff --git a/tests/queries/0_stateless/02985_disable_async_inserts_for_dependent_mv_dedup.reference b/tests/queries/0_stateless/02985_disable_async_inserts_for_dependent_mv_dedup.reference new file mode 100644 index 00000000000..4ff73b99975 --- /dev/null +++ b/tests/queries/0_stateless/02985_disable_async_inserts_for_dependent_mv_dedup.reference @@ -0,0 +1 @@ +Values Ok 4 Parsed diff --git a/tests/queries/0_stateless/02985_disable_async_inserts_for_dependent_mv_dedup.sql b/tests/queries/0_stateless/02985_disable_async_inserts_for_dependent_mv_dedup.sql new file mode 100644 index 00000000000..41b23374bfc --- /dev/null +++ b/tests/queries/0_stateless/02985_disable_async_inserts_for_dependent_mv_dedup.sql @@ -0,0 +1,46 @@ +-- Tags: no-parallel + +SET async_insert = 1; +SET insert_deduplicate = 1; +SET deduplicate_blocks_in_dependent_materialized_views = 1; + +DROP TABLE IF EXISTS 02985_test; +CREATE TABLE 02985_test +( + d Date, + value UInt64 +) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 1000; + +DROP VIEW IF EXISTS 02985_mv; +CREATE MATERIALIZED VIEW 02985_mv +ENGINE = SummingMergeTree ORDER BY d AS +SELECT + d, sum(value) s +FROM 02985_test GROUP BY d; + +-- Inserts are synchronous. +INSERT INTO 02985_test (*) +VALUES ('2024-01-01', 1), ('2024-01-01', 2), ('2024-01-02', 1); + +SYSTEM FLUSH LOGS; + +SELECT format, status, rows, data_kind FROM system.asynchronous_insert_log +WHERE database = currentDatabase() AND table = '02985_test'; + +SET deduplicate_blocks_in_dependent_materialized_views = 0; + +-- Set a large value for async_insert_busy_timeout_max_ms to avoid flushing the entry synchronously. +INSERT INTO 02985_test (*) +SETTINGS + async_insert_busy_timeout_min_ms=200, + async_insert_busy_timeout_max_ms=100000 +VALUES ('2024-01-01', 1), ('2024-01-01', 2), ('2024-01-02', 1), ('2024-01-02', 4); + +SYSTEM FLUSH LOGS; + +SELECT format, status, rows, data_kind +FROM system.asynchronous_insert_log +WHERE database = currentDatabase() AND table = '02985_test'; + +DROP VIEW IF EXISTS 02985_mv; +DROP TABLE IF EXISTS 02985_test; From c2019b3b1eb3b09803ecbbce8bd9bfe0560faf36 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 8 Feb 2024 09:08:44 +0000 Subject: [PATCH 063/103] Fix test --- tests/queries/0_stateless/02972_parallel_replicas_cte.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02972_parallel_replicas_cte.sql b/tests/queries/0_stateless/02972_parallel_replicas_cte.sql index c39ad172a27..3702184e336 100644 --- a/tests/queries/0_stateless/02972_parallel_replicas_cte.sql +++ b/tests/queries/0_stateless/02972_parallel_replicas_cte.sql @@ -12,16 +12,16 @@ SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups. WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 10000) SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_two_shards', max_parallel_replicas = 3; +SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- Testing that it is disabled for allow_experimental_analyzer=0. With analyzer it will be supported (with correct result) WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 10000) SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a -SETTINGS allow_experimental_analyzer = 0, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_two_shards', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED } +SETTINGS allow_experimental_analyzer = 0, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED } -- Sanitizer SELECT count() FROM pr_2 JOIN numbers(10) as pr_1 ON pr_2.a = pr_1.number -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_two_shards', max_parallel_replicas = 3; +SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; DROP TABLE IF EXISTS pr_1; DROP TABLE IF EXISTS pr_2; From 0130d525f40521cae5c79e4c002ea9c268f826d7 Mon Sep 17 00:00:00 2001 From: serxa Date: Thu, 8 Feb 2024 12:02:28 +0000 Subject: [PATCH 064/103] fix dashboard params default values --- programs/server/dashboard.html | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html index ae916f2527e..3c2916b6a16 100644 --- a/programs/server/dashboard.html +++ b/programs/server/dashboard.html @@ -527,10 +527,11 @@ let queries = []; /// Query parameters with predefined default values. /// All other parameters will be automatically found in the queries. -let params = { +let default_params = { 'rounding': '60', 'seconds': '86400' }; +let params = default_params; /// Palette generation for charts function generatePalette(baseColor, numColors) { @@ -594,13 +595,19 @@ let plots = []; let charts = document.getElementById('charts'); /// This is not quite correct (we cannot really parse SQL with regexp) but tolerable. -const query_param_regexp = /\{(\w+):[^}]+\}/g; +const query_param_regexp = /\{(\w+):([^}]+)\}/g; /// Automatically parse more parameters from the queries. function findParamsInQuery(query, new_params) { + const typeDefault = (type) => type.includes('Int') ? '0' + : (type.includes('Float') ? '0.0' + : (type.includes('Bool') ? 'false' + : (type.includes('Date') ? new Date().toISOString().slice(0, 10) + : (type.includes('UUID') ? '00000000-0000-0000-0000-000000000000' + : '')))); for (let match of query.matchAll(query_param_regexp)) { const name = match[1]; - new_params[name] = params[name] || ''; + new_params[name] = params[name] || default_params[name] || typeDefault(match[2]); } } From b8dcc233fd9d8774ad125412d44294ffac7c3de4 Mon Sep 17 00:00:00 2001 From: AlexeyGrezz <38166396+AlexeyGrezz@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:32:50 +0300 Subject: [PATCH 065/103] Update grants.md --- docs/ru/operations/system-tables/grants.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/operations/system-tables/grants.md b/docs/ru/operations/system-tables/grants.md index b3ef789e95b..4485b684218 100644 --- a/docs/ru/operations/system-tables/grants.md +++ b/docs/ru/operations/system-tables/grants.md @@ -19,7 +19,7 @@ slug: /ru/operations/system-tables/grants - `column` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Имя столбца, к которому предоставляется доступ. - `is_partial_revoke` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Логическое значение. Показывает, были ли отменены некоторые привилегии. Возможные значения: -- `0` — Строка описывает частичный отзыв. -- `1` — Строка описывает грант. +- `0` — Строка описывает грант. +- `1` — Строка описывает частичный отзыв. - `grant_option` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Разрешение предоставлено с опцией `WITH GRANT OPTION`, подробнее см. [GRANT](../../sql-reference/statements/grant.md#grant-privigele-syntax). From 73d2ff3933ca60133c33ed61af277d616e10bf9c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 8 Feb 2024 14:55:35 +0100 Subject: [PATCH 066/103] Update MergeTask.cpp --- src/Storages/MergeTree/MergeTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index f7cd721b8dd..9cbcdbaaaaa 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1066,7 +1066,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() #ifndef NDEBUG if (!sort_description.empty()) { - res_pipe.addSimpleTransform([&](const Block & header_) + builder->addSimpleTransform([&](const Block & header_) { auto transform = std::make_shared(header_, sort_description); return transform; From 160f1b7fd85add76128c978f0d7ae93729b53a5e Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 8 Feb 2024 15:01:56 +0100 Subject: [PATCH 067/103] Fix logical optimizer with LowCardinality --- src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp | 8 ++++---- .../02987_logical_optimizer_pass_lowcardinality.reference | 0 .../02987_logical_optimizer_pass_lowcardinality.sql | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.reference create mode 100644 tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.sql diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index a8cdd27c9bf..5f08bb9035e 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -404,12 +404,12 @@ private: auto operand_type = and_operands[0]->getResultType(); auto function_type = function_node.getResultType(); - assert(!function_type->isNullable()); + chassert(!function_type->isNullable()); if (!function_type->equals(*operand_type)) { /// Result of equality operator can be low cardinality, while AND always returns UInt8. /// In that case we replace `(lc = 1) AND (lc = 1)` with `(lc = 1) AS UInt8` - assert(function_type->equals(*removeLowCardinality(operand_type))); + chassert(function_type->equals(*removeLowCardinality(operand_type))); node = createCastFunction(std::move(and_operands[0]), function_type, getContext()); } else @@ -427,7 +427,7 @@ private: void tryReplaceOrEqualsChainWithIn(QueryTreeNodePtr & node) { auto & function_node = node->as(); - assert(function_node.getFunctionName() == "or"); + chassert(function_node.getFunctionName() == "or"); QueryTreeNodes or_operands; @@ -486,7 +486,7 @@ private: /// first we create tuple from RHS of equals functions for (const auto & equals : equals_functions) { - is_any_nullable |= equals->getResultType()->isNullable(); + is_any_nullable |= removeLowCardinality(equals->getResultType())->isNullable(); const auto * equals_function = equals->as(); assert(equals_function && equals_function->getFunctionName() == "equals"); diff --git a/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.reference b/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.sql b/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.sql new file mode 100644 index 00000000000..be7689025b2 --- /dev/null +++ b/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.sql @@ -0,0 +1,5 @@ +CREATE TABLE 02987_logical_optimizer_table (key Int, value Int) ENGINE=Memory(); +CREATE VIEW v1 AS SELECT * FROM 02987_logical_optimizer_table; +CREATE TABLE 02987_logical_optimizer_merge AS v1 ENGINE=Merge(currentDatabase(), 'v1'); + +SELECT _table, key FROM 02987_logical_optimizer_merge WHERE (_table = toFixedString(toFixedString(toFixedString('v1', toNullable(2)), 2), 2)) OR ((value = toLowCardinality(toNullable(10))) AND (_table = toFixedString(toNullable('v3'), 2))) OR ((value = 20) AND (_table = toFixedString(toFixedString(toFixedString('v1', 2), 2), 2)) AND (_table = toFixedString(toLowCardinality(toFixedString('v3', 2)), 2))) SETTINGS allow_experimental_analyzer = true, join_use_nulls = true, convert_query_to_cnf = true; From 97d1eb109190b0bff8fdf3c61cf44d5c1fbafd7d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 8 Feb 2024 14:31:24 +0000 Subject: [PATCH 068/103] Fixing test. --- tests/integration/test_recompression_ttl/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_recompression_ttl/test.py b/tests/integration/test_recompression_ttl/test.py index 851e3bb4eb8..9d7b09eacdf 100644 --- a/tests/integration/test_recompression_ttl/test.py +++ b/tests/integration/test_recompression_ttl/test.py @@ -155,7 +155,7 @@ def test_recompression_multiple_ttls(started_cluster): node2.query( "SELECT recompression_ttl_info.expression FROM system.parts where name = 'all_1_1_4'" ) - == "['plus(d, toIntervalSecond(10))','plus(d, toIntervalSecond(15))','plus(d, toIntervalSecond(5))']\n" + == "['d + toIntervalSecond(10)','d + toIntervalSecond(15)','d + toIntervalSecond(5)']\n" ) From 1b9620001b31256ffb6e8fee7b521efa8688fbee Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 8 Feb 2024 15:24:01 +0100 Subject: [PATCH 069/103] Move ColumnArray-specific code out of the ActionsDAG. --- src/Columns/ColumnArray.cpp | 15 +++++++++++++++ src/Columns/ColumnArray.h | 4 ++++ src/Interpreters/ActionsDAG.cpp | 17 ++++++++++++----- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index fddfa2ac6b2..6f60ec0e642 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -554,6 +554,21 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng } +MutableColumnPtr ColumnArray::getDataInRange(size_t start, size_t length) const +{ + if (start + length > getOffsets().size()) + throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Parameter out of bound in ColumnArray::getDataPtrForRange method. " + "[start({}) + length({}) > offsets.size({})]", start, length, getOffsets().size()); + + size_t start_offset = offsetAt(start); + size_t end_offset = offsetAt(start + length); + + auto res = getData().cloneEmpty(); + res->insertRangeFrom(getData(), start_offset, end_offset - start_offset); + return res; +} + + ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const { if (typeid_cast(data.get())) diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 407f44a6f3c..64c8801b0c3 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -143,6 +143,10 @@ public: const ColumnPtr & getOffsetsPtr() const { return offsets; } ColumnPtr & getOffsetsPtr() { return offsets; } + /// Returns a copy of the data column's part corresponding to a specified range of rows. + /// For example, `getDataInRange(0, size())` is the same as `getDataPtr()->clone()`. + MutableColumnPtr getDataInRange(size_t start, size_t length) const; + MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override { return scatterImpl(num_columns, selector); diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index a1858916ca7..b3f3f8da26d 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -628,17 +628,24 @@ static ColumnWithTypeAndName executeActionForPartialResult(const ActionsDAG::Nod if (!array) throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN of not array nor map: {}", node->result_name); - res_column.column = array->getDataPtr(); + + ColumnPtr data; if (input_rows_count < array->size()) - res_column.column = res_column.column->cloneResized(array->getOffsets()[input_rows_count - 1]); + data = array->getDataInRange(0, input_rows_count); + else + data = array->getDataPtr(); + + res_column.column = data; break; } case ActionsDAG::ActionType::COLUMN: { - res_column.column = node->column; - if (input_rows_count < res_column.column->size()) - res_column.column = res_column.column->cloneResized(input_rows_count); + auto column = node->column; + if (input_rows_count < column->size()) + column = column->cloneResized(input_rows_count); + + res_column.column = column; break; } From 5f750871786772e1135084c34b5a20aa8c108c74 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 8 Feb 2024 15:48:20 +0000 Subject: [PATCH 070/103] Add comments --- src/Storages/StorageMerge.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index d3b8f30b1c5..09c38996b22 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -756,16 +756,23 @@ QueryTreeNodePtr replaceTableExpressionAndRemoveJoin( auto join_tree_type = query_node->getJoinTree()->getNodeType(); auto modified_query = query_node->cloneAndReplace(original_table_expression, replacement_table_expression); + // For the case when join tree is just a table or a table function we don't need to do anything more. if (join_tree_type == QueryTreeNodeType::TABLE || join_tree_type == QueryTreeNodeType::TABLE_FUNCTION) return modified_query; + // JOIN needs to be removed because StorageMerge should produce not joined data. + // GROUP BY should be removed as well. + auto * modified_query_node = modified_query->as(); + // Remove the JOIN statement. As a result query will have a form like: SELECT * FROM ... modified_query = modified_query->cloneAndReplace(modified_query_node->getJoinTree(), replacement_table_expression); modified_query_node = modified_query->as(); query_node = modified_query->as(); + // For backward compatibility we need to leave all filters related to this table. + // It may lead to some incorrect result. if (query_node->hasPrewhere()) replaceFilterExpression(query_node->getPrewhere(), replacement_table_expression, context); if (query_node->hasWhere()) @@ -779,6 +786,9 @@ QueryTreeNodePtr replaceTableExpressionAndRemoveJoin( projection.clear(); NamesAndTypes projection_columns; + // Select only required columns from the table, because prjection list may contain: + // 1. aggregate functions + // 2. expressions referencing other tables of JOIN for (auto const & column_name : required_column_names) { QueryTreeNodePtr fake_node = std::make_shared(Identifier{column_name}); @@ -791,6 +801,8 @@ QueryTreeNodePtr replaceTableExpressionAndRemoveJoin( throw Exception(ErrorCodes::LOGICAL_ERROR, "Required column '{}' is not resolved", column_name); auto fake_column = resolved_column->getColumn(); + // Identifier is resolved to ColumnNode, but we need to get rid of ALIAS columns + // and also fix references to source expression (now column is referencing original table expression). ApplyAliasColumnExpressionsVisitor visitor(replacement_table_expression); visitor.visit(fake_node); @@ -860,7 +872,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextPtr & modified_ QueryTreeNodePtr column_node; - + // Replace all references to ALIAS columns in the query by expressions. if (is_alias) { QueryTreeNodePtr fake_node = std::make_shared(Identifier{column}); From 927e00f8f1a7ee602c72f40f9e47657e04669222 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 8 Feb 2024 18:51:50 +0100 Subject: [PATCH 071/103] Update libuv --- contrib/libuv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libuv b/contrib/libuv index 3a85b2eb3d8..4482964660c 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit 3a85b2eb3d83f369b8a8cafd329d7e9dc28f60cf +Subproject commit 4482964660c77eec1166cd7d14fb915e3dbd774a From 641c7b547d898696fe6719cf2f5662b138f4e44c Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Sun, 4 Feb 2024 19:12:37 +0000 Subject: [PATCH 072/103] CI: enable await #no_merge_commit --- .github/workflows/master.yml | 97 ++++++++ .github/workflows/pull_request.yml | 2 +- .github/workflows/reusable_build.yml | 1 + .github/workflows/reusable_test.yml | 1 + tests/ci/ci.py | 360 ++++++++++++++++++--------- tests/ci/ci_config.py | 28 +-- tests/ci/ci_utils.py | 17 +- tests/ci/commit_status_helper.py | 3 + tests/ci/report.py | 3 +- tests/ci/test_ci_cache.py | 35 ++- 10 files changed, 409 insertions(+), 138 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2471e4f9194..dac1332adc6 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -385,6 +385,22 @@ jobs: test_name: Stateless tests (release, s3 storage) runner_type: func-tester data: ${{ needs.RunConfig.outputs.data }} + FunctionalStatelessTestS3Debug: + needs: [RunConfig, BuilderDebDebug] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Stateless tests (debug, s3 storage) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} + FunctionalStatelessTestS3Tsan: + needs: [RunConfig, BuilderDebTsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Stateless tests (tsan, s3 storage) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestAarch64: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} @@ -493,6 +509,55 @@ jobs: test_name: Stateful tests (debug) runner_type: func-tester data: ${{ needs.RunConfig.outputs.data }} + # Parallel replicas + FunctionalStatefulTestDebugParallelReplicas: + needs: [RunConfig, BuilderDebDebug] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Stateful tests (debug, ParallelReplicas) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} + FunctionalStatefulTestUBsanParallelReplicas: + needs: [RunConfig, BuilderDebUBsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Stateful tests (ubsan, ParallelReplicas) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} + FunctionalStatefulTestMsanParallelReplicas: + needs: [RunConfig, BuilderDebMsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Stateful tests (msan, ParallelReplicas) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} + FunctionalStatefulTestTsanParallelReplicas: + needs: [RunConfig, BuilderDebTsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Stateful tests (tsan, ParallelReplicas) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} + FunctionalStatefulTestAsanParallelReplicas: + needs: [RunConfig, BuilderDebAsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Stateful tests (asan, ParallelReplicas) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} + FunctionalStatefulTestReleaseParallelReplicas: + needs: [RunConfig, BuilderDebRelease] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Stateful tests (release, ParallelReplicas) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} ############################################################################################## ########################### ClickBench ####################################################### ############################################################################################## @@ -700,6 +765,28 @@ jobs: runner_type: func-tester-aarch64 data: ${{ needs.RunConfig.outputs.data }} ############################################################################################## +############################ SQLLOGIC TEST ################################################### +############################################################################################## + SQLLogicTestRelease: + needs: [RunConfig, BuilderDebRelease] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Sqllogic test (release) + runner_type: func-tester + data: ${{ needs.RunConfig.outputs.data }} +############################################################################################## +##################################### SQL TEST ############################################### +############################################################################################## + SQLTest: + needs: [RunConfig, BuilderDebRelease] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: SQLTest + runner_type: fuzzer-unit-tester + data: ${{ needs.RunConfig.outputs.data }} +############################################################################################## ###################################### SQLANCER FUZZERS ###################################### ############################################################################################## SQLancerTestRelease: @@ -732,6 +819,8 @@ jobs: - FunctionalStatelessTestTsan - FunctionalStatelessTestMsan - FunctionalStatelessTestUBsan + - FunctionalStatelessTestS3Debug + - FunctionalStatelessTestS3Tsan - FunctionalStatefulTestDebug - FunctionalStatefulTestRelease - FunctionalStatefulTestAarch64 @@ -739,6 +828,12 @@ jobs: - FunctionalStatefulTestTsan - FunctionalStatefulTestMsan - FunctionalStatefulTestUBsan + - FunctionalStatefulTestDebugParallelReplicas + - FunctionalStatefulTestUBsanParallelReplicas + - FunctionalStatefulTestMsanParallelReplicas + - FunctionalStatefulTestTsanParallelReplicas + - FunctionalStatefulTestAsanParallelReplicas + - FunctionalStatefulTestReleaseParallelReplicas - StressTestDebug - StressTestAsan - StressTestTsan @@ -764,6 +859,8 @@ jobs: - UnitTestsReleaseClang - SQLancerTestRelease - SQLancerTestDebug + - SQLLogicTestRelease + - SQLTest runs-on: [self-hosted, style-checker] steps: - name: Check out repository code diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 09e2d6dbb97..cf31738643b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1002,7 +1002,7 @@ jobs: ####################################### libFuzzer ########################################### ############################################################################################# libFuzzer: - if: ${{ !failure() && !cancelled() && contains(github.event.pull_request.labels.*.name, 'libFuzzer') }} + if: ${{ !failure() && !cancelled() }} needs: [RunConfig, StyleCheck] uses: ./.github/workflows/libfuzzer.yml with: diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index 2371579692f..6be9d30175e 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -85,6 +85,7 @@ jobs: run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.build_name}}' - name: Mark as done + if: ${{ !cancelled() }} run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.build_name}}' - name: Clean diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 749f64d434e..e30ef863a86 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -107,6 +107,7 @@ jobs: run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.test_name}}' - name: Mark as done + if: ${{ !cancelled() }} run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.test_name}}' --batch ${{matrix.batch}} - name: Clean diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 622b7bb005a..5c33d4e02a8 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1,5 +1,8 @@ import argparse import concurrent.futures +from copy import deepcopy +from dataclasses import asdict, dataclass +from enum import Enum import json import logging import os @@ -7,16 +10,14 @@ import re import subprocess import sys import time -from dataclasses import asdict, dataclass -from enum import Enum from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Union import docker_images_helper import upload_result_helper from build_check import get_release_or_pr -from ci_config import CI_CONFIG, Build, JobNames, Labels -from ci_utils import GHActions, is_hex +from ci_config import CI_CONFIG, Build, Labels, JobNames +from ci_utils import GHActions, is_hex, normalize_string from clickhouse_helper import ( CiLogsCredentials, ClickHouseHelper, @@ -48,7 +49,7 @@ from git_helper import GIT_PREFIX, Git from git_helper import Runner as GitRunner from github import Github from pr_info import PRInfo -from report import SUCCESS, BuildResult, JobReport +from report import ERROR, SUCCESS, BuildResult, JobReport from s3_helper import S3Helper from version_helper import get_version_from_repo @@ -88,6 +89,7 @@ class CiCache: class RecordType(Enum): SUCCESSFUL = "successful" PENDING = "pending" + FAILED = "failed" @dataclass class Record: @@ -249,6 +251,13 @@ class CiCache: ) return record + def print_status(self): + for record_type in self.RecordType: + GHActions.print_in_group( + f"Cache records: [{record_type}]", list(self.records[record_type]) + ) + return self + def update(self): """ Pulls cache records from s3. Only records name w/o content. @@ -260,9 +269,6 @@ class CiCache: path = self.cache_s3_paths[job_type] records = self.s3.list_prefix(f"{path}{prefix}", S3_BUILDS_BUCKET) records = [record.split("/")[-1] for record in records] - GHActions.print_in_group( - f"Cache records: [{record_type}] in [{job_type.value}]", records - ) for file in records: record = self._parse_record_file_name( record_type=record_type, file_name=file @@ -384,6 +390,9 @@ class CiCache: if record_type == self.RecordType.SUCCESSFUL: assert isinstance(status, CommitStatusData) status.dump_to_file(record_file) + elif record_type == self.RecordType.FAILED: + assert isinstance(status, CommitStatusData) + status.dump_to_file(record_file) elif record_type == self.RecordType.PENDING: assert isinstance(status, PendingState) with open(record_file, "w") as json_file: @@ -488,6 +497,16 @@ class CiCache: self.RecordType.SUCCESSFUL, job, batch, num_batches, release_branch ) + def is_failed( + self, job: str, batch: int, num_batches: int, release_branch: bool + ) -> bool: + """ + checks if a given job have already been done with failure + """ + return self.exist( + self.RecordType.FAILED, job, batch, num_batches, release_branch + ) + def is_pending( self, job: str, batch: int, num_batches: int, release_branch: bool ) -> bool: @@ -495,8 +514,9 @@ class CiCache: check pending record in the cache for a given job @release_branch - checks that "release" attribute is set for a record """ - if self.is_successful(job, batch, num_batches, release_branch): - # successful record is present - not pending + if self.is_successful( + job, batch, num_batches, release_branch + ) or self.is_failed(job, batch, num_batches, release_branch): return False return self.exist( @@ -524,6 +544,27 @@ class CiCache: release_branch, ) + def push_failed( + self, + job: str, + batch: int, + num_batches: int, + job_status: CommitStatusData, + release_branch: bool = False, + ) -> None: + """ + Pushes a cache record of type Failed (CommitStatusData) + @release_branch adds "release" attribute to a record + """ + self.push( + self.RecordType.FAILED, + job, + [batch], + num_batches, + job_status, + release_branch, + ) + def push_pending( self, job: str, batches: List[int], num_batches: int, release_branch: bool ) -> None: @@ -591,46 +632,87 @@ class CiCache: bucket=S3_BUILDS_BUCKET, file_path=result_json_path, s3_path=s3_path ) - # def await_jobs(self, jobs_with_params: Dict[str, Dict[str, Any]]) -> List[str]: - # if not jobs_with_params: - # return [] - # print(f"Start awaiting jobs [{list(jobs_with_params)}]") - # poll_interval_sec = 180 - # start_at = int(time.time()) - # TIMEOUT = 3000 - # expired_sec = 0 - # done_jobs = [] # type: List[str] - # while expired_sec < TIMEOUT and jobs_with_params: - # time.sleep(poll_interval_sec) - # self.update() - # pending_finished: List[str] = [] - # for job_name in jobs_with_params: - # num_batches = jobs_with_params[job_name]["num_batches"] - # for batch in jobs_with_params[job_name]["batches"]: - # if self.is_pending(job_name, batch, num_batches): - # continue - # print( - # f"Job [{job_name}_[{batch}/{num_batches}]] is not pending anymore" - # ) - # pending_finished.append(job_name) - # if pending_finished: - # # restart timer - # start_at = int(time.time()) - # expired_sec = 0 - # # remove finished jobs from awaiting list - # for job in pending_finished: - # del jobs_with_params[job] - # done_jobs.append(job) - # else: - # expired_sec = int(time.time()) - start_at - # print(f" ...awaiting continues... time left [{TIMEOUT - expired_sec}]") - # if done_jobs: - # print( - # f"Awaiting OK. Left jobs: [{list(jobs_with_params)}], finished jobs: [{done_jobs}]" - # ) - # else: - # print("Awaiting FAILED. No job has finished.") - # return done_jobs + def await_jobs( + self, jobs_with_params: Dict[str, Dict[str, Any]], is_release_branch: bool + ) -> Dict[str, List[int]]: + """ + await pending jobs to be finished + @jobs_with_params - jobs to await. {JOB_NAME: {"batches": [BATCHES...], "num_batches": NUM_BATCHES}} + returns successfully finished jobs: {JOB_NAME: [BATCHES...]} + """ + if not jobs_with_params: + return {} + poll_interval_sec = 300 + TIMEOUT = 3600 + await_finished: Dict[str, List[int]] = {} + round_cnt = 0 + while len(jobs_with_params) > 5 and round_cnt < 3: + round_cnt += 1 + GHActions.print_in_group( + f"Wait pending jobs, round [{round_cnt}]:", list(jobs_with_params) + ) + # this is initial approach to wait pending jobs: + # start waiting for the next TIMEOUT seconds if there are more than X(=5) jobs to wait + # wait TIMEOUT seconds in rounds. Y(=3) is the max number of rounds + expired_sec = 0 + start_at = int(time.time()) + while expired_sec < TIMEOUT and jobs_with_params: + time.sleep(poll_interval_sec) + self.update() + jobs_with_params_copy = deepcopy(jobs_with_params) + for job_name in jobs_with_params: + num_batches = jobs_with_params[job_name]["num_batches"] + job_config = CI_CONFIG.get_job_config(job_name) + for batch in jobs_with_params[job_name]["batches"]: + if self.is_pending( + job_name, + batch, + num_batches, + release_branch=is_release_branch + and job_config.required_on_release_branch, + ): + continue + print( + f"Job [{job_name}_[{batch}/{num_batches}]] is not pending anymore" + ) + + # some_job_ready = True + jobs_with_params_copy[job_name]["batches"].remove(batch) + if not jobs_with_params_copy[job_name]["batches"]: + del jobs_with_params_copy[job_name] + + if not self.is_successful( + job_name, + batch, + num_batches, + release_branch=is_release_branch + and job_config.required_on_release_branch, + ): + print( + f"NOTE: Job [{job_name}:{batch}] finished but no success - remove from awaiting list, do not add to ready" + ) + continue + if job_name in await_finished: + await_finished[job_name].append(batch) + else: + await_finished[job_name] = [batch] + jobs_with_params = jobs_with_params_copy + expired_sec = int(time.time()) - start_at + print( + f"...awaiting continues... seconds left [{TIMEOUT - expired_sec}]" + ) + if await_finished: + GHActions.print_in_group( + "Finished jobs:", + [f"{job}:{batches}" for job, batches in await_finished.items()], + ) + else: + print("Awaiting FAILED. No job has finished successfully.") + GHActions.print_in_group( + "Remaining jobs:", + [f"{job}:{params['batches']}" for job, params in jobs_with_params.items()], + ) + return await_finished def get_check_name(check_name: str, batch: int, num_batches: int) -> str: @@ -832,7 +914,10 @@ def _pre_action(s3, indata, pr_info): ci_cache = CiCache(s3, indata["jobs_data"]["digests"]) # for release/master branches reports must be from the same branches - report_prefix = pr_info.head_ref if pr_info.number == 0 else "" + report_prefix = normalize_string(pr_info.head_ref) if pr_info.number == 0 else "" + print( + f"Use report prefix [{report_prefix}], pr_num [{pr_info.number}], head_ref [{pr_info.head_ref}]" + ) reports_files = ci_cache.download_build_reports(file_prefix=report_prefix) print(f"Pre action done. Report files [{reports_files}] have been downloaded") @@ -883,8 +968,19 @@ def _mark_success_action( job, batch, num_batches, job_status, pr_info.is_release_branch() ) print(f"Job [{job}] is ok") - elif job_status: - print(f"Job [{job}] is not ok, status [{job_status.status}]") + elif job_status and not job_status.is_ok(): + ci_cache.push_failed( + job, batch, num_batches, job_status, pr_info.is_release_branch() + ) + print(f"Job [{job}] is failed with status [{job_status.status}]") + else: + job_status = CommitStatusData( + description="dummy description", status=ERROR, report_url="dummy url" + ) + ci_cache.push_failed( + job, batch, num_batches, job_status, pr_info.is_release_branch() + ) + print(f"No CommitStatusData for [{job}], push dummy failure to ci_cache") def _print_results(result: Any, outfile: Optional[str], pretty: bool = False) -> None: @@ -992,8 +1088,8 @@ def _configure_jobs( jobs_to_do: List[str] = [] jobs_to_skip: List[str] = [] digests: Dict[str, str] = {} - print("::group::Job Digests") + print("::group::Job Digests") for job in CI_CONFIG.job_generator(): digest = job_digester.get_job_digest(CI_CONFIG.get_digest_config(job)) digests[job] = digest @@ -1003,7 +1099,8 @@ def _configure_jobs( ## b. check what we need to run ci_cache = None if not ci_cache_disabled: - ci_cache = CiCache(s3, digests) + ci_cache = CiCache(s3, digests).update() + ci_cache.print_status() jobs_to_wait: Dict[str, Dict[str, Any]] = {} @@ -1012,10 +1109,13 @@ def _configure_jobs( job_config = CI_CONFIG.get_job_config(job) num_batches: int = job_config.num_batches batches_to_do: List[int] = [] + add_to_skip = False for batch in range(num_batches): # type: ignore if job_config.pr_only and pr_info.is_release_branch(): continue + if job_config.release_only and not pr_info.is_release_branch(): + continue if job_config.run_by_label: # this job controlled by label, add to todo if its label is set in pr if job_config.run_by_label in pr_info.labels: @@ -1036,7 +1136,13 @@ def _configure_jobs( batches_to_do.append(batch) # check if it's pending in the cache - if ci_cache.is_pending(job, batch, num_batches, release_branch=False): + if ci_cache.is_pending( + job, + batch, + num_batches, + release_branch=pr_info.is_release_branch() + and job_config.required_on_release_branch, + ): if job in jobs_to_wait: jobs_to_wait[job]["batches"].append(batch) else: @@ -1044,10 +1150,12 @@ def _configure_jobs( "batches": [batch], "num_batches": num_batches, } + else: + add_to_skip = True if batches_to_do: jobs_to_do.append(job) - elif not job_config.run_by_label: + elif add_to_skip: # treat job as being skipped only if it's controlled by digest jobs_to_skip.append(job) jobs_params[job] = { @@ -1119,49 +1227,64 @@ def _configure_jobs( "digests": digests, "jobs_to_do": jobs_to_do, "jobs_to_skip": jobs_to_skip, - "jobs_to_wait": jobs_to_wait, + "jobs_to_wait": { + job: params for job, params in jobs_to_wait.items() if job in jobs_to_do + }, "jobs_params": { job: params for job, params in jobs_params.items() if job in jobs_to_do }, } +def _create_gh_status( + commit: Any, job: str, batch: int, num_batches: int, job_status: CommitStatusData +) -> None: + print(f"Going to re-create GH status for job [{job}]") + assert job_status.status == SUCCESS, "BUG!" + commit.create_status( + state=job_status.status, + target_url=job_status.report_url, + description=format_description( + f"Reused from [{job_status.pr_num}-{job_status.sha[0:8]}]: " + f"{job_status.description}" + ), + context=get_check_name(job, batch=batch, num_batches=num_batches), + ) + + def _update_gh_statuses_action(indata: Dict, s3: S3Helper) -> None: if indata["ci_flags"][Labels.NO_CI_CACHE]: print("CI cache is disabled - skip restoring commit statuses from CI cache") return job_digests = indata["jobs_data"]["digests"] - ci_cache = CiCache(s3, job_digests).update().fetch_records_data() + jobs_to_skip = indata["jobs_data"]["jobs_to_skip"] + jobs_to_do = indata["jobs_data"]["jobs_to_do"] + ci_cache = CiCache(s3, job_digests).update().fetch_records_data().print_status() # create GH status pr_info = PRInfo() commit = get_commit(Github(get_best_robot_token(), per_page=100), pr_info.sha) - def _run_create_status(job: str, batch: int, num_batches: int) -> None: + def _concurrent_create_status(job: str, batch: int, num_batches: int) -> None: job_status = ci_cache.get_successful(job, batch, num_batches) if not job_status: return - print(f"Going to re-create GH status for job [{job}] sha [{pr_info.sha}]") - assert job_status.status == SUCCESS, "BUG!" - commit.create_status( - state=job_status.status, - target_url=job_status.report_url, - description=format_description( - f"Reused from [{job_status.pr_num}-{job_status.sha[0:8]}]: " - f"{job_status.description}" - ), - context=get_check_name(job, batch=batch, num_batches=num_batches), - ) + _create_gh_status(commit, job, batch, num_batches, job_status) with concurrent.futures.ThreadPoolExecutor() as executor: futures = [] for job in job_digests: + if job not in jobs_to_skip or job not in jobs_to_do: + # no need to create status for job that are not supposed to be executed + continue if CI_CONFIG.is_build_job(job): # no GH status for build jobs continue num_batches = CI_CONFIG.get_job_config(job).num_batches for batch in range(num_batches): - future = executor.submit(_run_create_status, job, batch, num_batches) + future = executor.submit( + _concurrent_create_status, job, batch, num_batches + ) futures.append(future) done, _ = concurrent.futures.wait(futures) for future in done: @@ -1194,7 +1317,7 @@ def _upload_build_artifacts( ( get_release_or_pr(pr_info, get_version_from_repo())[1], pr_info.sha, - CI_CONFIG.normalize_string(build_name), + normalize_string(build_name), "performance.tar.zst", ) ) @@ -1509,30 +1632,51 @@ def main() -> int: if not args.skip_jobs and pr_info.has_changes_in_documentation_only(): _update_config_for_docs_only(jobs_data) - # TODO: await pending jobs - # wait for pending jobs to be finished, await_jobs is a long blocking call if any job has to be awaited - # awaited_jobs = ci_cache.await_jobs(jobs_data.get("jobs_to_wait", {})) - # for job in awaited_jobs: - # jobs_to_do = jobs_data["jobs_to_do"] - # if job in jobs_to_do: - # jobs_to_do.remove(job) - # else: - # assert False, "BUG" - - # set planned jobs as pending in the CI cache if on the master - if pr_info.is_master() and not args.skip_jobs: + if not args.skip_jobs: ci_cache = CiCache(s3, jobs_data["digests"]) - for job in jobs_data["jobs_to_do"]: - config = CI_CONFIG.get_job_config(job) - if config.run_always or config.run_by_label: - continue - job_params = jobs_data["jobs_params"][job] - ci_cache.push_pending( - job, - job_params["batches"], - config.num_batches, - release_branch=pr_info.is_release_branch(), + + if ( + pr_info.is_release_branch() + or pr_info.event.get("pull_request", {}) + .get("user", {}) + .get("login", "not_maxknv") + == "maxknv" + ): + # wait for pending jobs to be finished, await_jobs is a long blocking call + # wait pending jobs (for now only on release/master branches) + ready_jobs_batches_dict = ci_cache.await_jobs( + jobs_data.get("jobs_to_wait", {}), pr_info.is_release_branch() ) + jobs_to_do = jobs_data["jobs_to_do"] + jobs_to_skip = jobs_data["jobs_to_skip"] + jobs_params = jobs_data["jobs_params"] + for job, batches in ready_jobs_batches_dict.items(): + if job not in jobs_params: + print(f"WARNING: Job [{job}] is not in the params list") + continue + for batch in batches: + jobs_params[job]["batches"].remove(batch) + if not jobs_params[job]["batches"]: + jobs_to_do.remove(job) + jobs_to_skip.append(job) + del jobs_params[job] + + # set planned jobs as pending in the CI cache if on the master + if pr_info.is_master(): + for job in jobs_data["jobs_to_do"]: + config = CI_CONFIG.get_job_config(job) + if config.run_always or config.run_by_label: + continue + job_params = jobs_data["jobs_params"][job] + ci_cache.push_pending( + job, + job_params["batches"], + config.num_batches, + release_branch=pr_info.is_release_branch(), + ) + + if "jobs_to_wait" in jobs_data: + del jobs_data["jobs_to_wait"] # conclude results result["git_ref"] = git_ref @@ -1608,23 +1752,15 @@ def main() -> int: check_name, args.batch, job_config.num_batches ) assert job_status, "BUG" - commit.create_status( - state=job_status.status, - target_url=job_status.report_url, - description=format_description( - f"Reused from [{job_status.pr_num}-{job_status.sha[0:8]}]: " - f"{job_status.description}" - ), - context=get_check_name( - check_name, - batch=args.batch, - num_batches=job_config.num_batches, - ), + _create_gh_status( + commit, + check_name, + args.batch, + job_config.num_batches, + job_status, ) previous_status = job_status.status - print("::group::Commit Status Data") - print(job_status) - print("::endgroup::") + GHActions.print_in_group("Commit Status Data", job_status) if previous_status: print( @@ -1648,7 +1784,7 @@ def main() -> int: if CI_CONFIG.is_build_job(args.job_name): assert ( indata - ), "--infile with config must be provided for POST action of a build type job [{args.job_name}]" + ), f"--infile with config must be provided for POST action of a build type job [{args.job_name}]" build_name = args.job_name s3_path_prefix = "/".join( ( @@ -1676,7 +1812,7 @@ def main() -> int: ( get_release_or_pr(pr_info, get_version_from_repo())[0], pr_info.sha, - CI_CONFIG.normalize_string( + normalize_string( job_report.check_name or _get_ext_check_name(args.job_name) ), ) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 6036a04080c..7c8990e8d16 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -22,6 +22,8 @@ class Labels(metaclass=WithIter): CI_SET_ARM = "ci_set_arm" CI_SET_INTEGRATION = "ci_set_integration" + libFuzzer = "libFuzzer" + class Build(metaclass=WithIter): PACKAGE_RELEASE = "package_release" @@ -193,6 +195,8 @@ class JobConfig: required_on_release_branch: bool = False # job is for pr workflow only pr_only: bool = False + # job is for release/master branches only + release_only: bool = False @dataclass @@ -790,6 +794,7 @@ CI_CONFIG = CiConfig( name=Build.FUZZERS, compiler="clang-17", package_type="fuzzers", + job_config=JobConfig(run_by_label=Labels.libFuzzer), ), }, builds_report_config={ @@ -824,7 +829,7 @@ CI_CONFIG = CiConfig( }, other_jobs_configs={ JobNames.MARK_RELEASE_READY: TestConfig( - "", job_config=JobConfig(required_on_release_branch=True) + "", job_config=JobConfig(release_only=True) ), JobNames.DOCKER_SERVER: TestConfig( "", @@ -909,13 +914,6 @@ CI_CONFIG = CiConfig( JobNames.STATEFUL_TEST_AARCH64: TestConfig( Build.PACKAGE_AARCH64, job_config=JobConfig(**stateful_test_common_params) # type: ignore ), - # FIXME: delete? - # "Stateful tests (release, DatabaseOrdinary)": TestConfig( - # Build.PACKAGE_RELEASE, job_config=JobConfig(**stateful_test_common_params) # type: ignore - # ), - # "Stateful tests (release, DatabaseReplicated)": TestConfig( - # Build.PACKAGE_RELEASE, job_config=JobConfig(**stateful_test_common_params) # type: ignore - # ), # Stateful tests for parallel replicas JobNames.STATEFUL_TEST_PARALLEL_REPL_RELEASE: TestConfig( Build.PACKAGE_RELEASE, job_config=JobConfig(**stateful_test_common_params) # type: ignore @@ -997,16 +995,16 @@ CI_CONFIG = CiConfig( Build.PACKAGE_DEBUG, job_config=JobConfig(**stress_test_common_params) # type: ignore ), JobNames.UPGRADE_TEST_ASAN: TestConfig( - Build.PACKAGE_ASAN, job_config=JobConfig(**upgrade_test_common_params) # type: ignore + Build.PACKAGE_ASAN, job_config=JobConfig(pr_only=True, **upgrade_test_common_params) # type: ignore ), JobNames.UPGRADE_TEST_TSAN: TestConfig( - Build.PACKAGE_TSAN, job_config=JobConfig(**upgrade_test_common_params) # type: ignore + Build.PACKAGE_TSAN, job_config=JobConfig(pr_only=True, **upgrade_test_common_params) # type: ignore ), JobNames.UPGRADE_TEST_MSAN: TestConfig( - Build.PACKAGE_MSAN, job_config=JobConfig(**upgrade_test_common_params) # type: ignore + Build.PACKAGE_MSAN, job_config=JobConfig(pr_only=True, **upgrade_test_common_params) # type: ignore ), JobNames.UPGRADE_TEST_DEBUG: TestConfig( - Build.PACKAGE_DEBUG, job_config=JobConfig(**upgrade_test_common_params) # type: ignore + Build.PACKAGE_DEBUG, job_config=JobConfig(pr_only=True, **upgrade_test_common_params) # type: ignore ), JobNames.INTEGRATION_TEST_ASAN: TestConfig( Build.PACKAGE_ASAN, @@ -1033,7 +1031,7 @@ CI_CONFIG = CiConfig( job_config=JobConfig(num_batches=4, **integration_test_common_params), # type: ignore ), JobNames.INTEGRATION_TEST_FLAKY: TestConfig( - Build.PACKAGE_ASAN, job_config=JobConfig(**integration_test_common_params) # type: ignore + Build.PACKAGE_ASAN, job_config=JobConfig(pr_only=True, **integration_test_common_params) # type: ignore ), JobNames.COMPATIBILITY_TEST: TestConfig( Build.PACKAGE_RELEASE, @@ -1080,7 +1078,7 @@ CI_CONFIG = CiConfig( JobNames.STATELESS_TEST_FLAKY_ASAN: TestConfig( # replace to non-default Build.PACKAGE_ASAN, - job_config=JobConfig(**{**statless_test_common_params, "timeout": 3600}), # type: ignore + job_config=JobConfig(pr_only=True, **{**statless_test_common_params, "timeout": 3600}), # type: ignore ), JobNames.JEPSEN_KEEPER: TestConfig( Build.BINARY_RELEASE, @@ -1116,7 +1114,7 @@ CI_CONFIG = CiConfig( ), JobNames.CLCIKBENCH_TEST: TestConfig(Build.PACKAGE_RELEASE), JobNames.CLCIKBENCH_TEST_ARM: TestConfig(Build.PACKAGE_AARCH64), - JobNames.LIBFUZZER_TEST: TestConfig(Build.FUZZERS), # type: ignore + JobNames.LIBFUZZER_TEST: TestConfig(Build.FUZZERS, job_config=JobConfig(run_by_label=Labels.libFuzzer)), # type: ignore }, ) CI_CONFIG.validate() diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py index 7e2a3d11725..2967ec2f309 100644 --- a/tests/ci/ci_utils.py +++ b/tests/ci/ci_utils.py @@ -1,6 +1,6 @@ from contextlib import contextmanager import os -from typing import List, Union, Iterator +from typing import Any, List, Union, Iterator from pathlib import Path @@ -27,9 +27,22 @@ def is_hex(s): return False +def normalize_string(string: str) -> str: + lowercase_string = string.lower() + normalized_string = ( + lowercase_string.replace(" ", "_") + .replace("-", "_") + .replace("/", "_") + .replace("(", "") + .replace(")", "") + .replace(",", "") + ) + return normalized_string + + class GHActions: @staticmethod - def print_in_group(group_name: str, lines: Union[str, List[str]]) -> None: + def print_in_group(group_name: str, lines: Union[Any, List[Any]]) -> None: lines = list(lines) print(f"::group::{group_name}") for line in lines: diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 5dd2a33adaf..8a34d375d1e 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -370,6 +370,9 @@ class CommitStatusData: def is_ok(self): return self.status == SUCCESS + def is_failure(self): + return self.status == FAILURE + @staticmethod def cleanup(): STATUS_FILE_PATH.unlink(missing_ok=True) diff --git a/tests/ci/report.py b/tests/ci/report.py index ce20c7293f9..ef09e9738ee 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -23,6 +23,7 @@ from typing import ( from build_download_helper import get_gh_api from ci_config import CI_CONFIG, BuildConfig from env_helper import REPORT_PATH, TEMP_PATH +from ci_utils import normalize_string logger = logging.getLogger(__name__) @@ -550,7 +551,7 @@ class BuildResult: def write_json(self, directory: Union[Path, str] = REPORT_PATH) -> Path: path = Path(directory) / self.get_report_name( - self.build_name, self.pr_number or self.head_ref + self.build_name, self.pr_number or normalize_string(self.head_ref) ) path.write_text( json.dumps( diff --git a/tests/ci/test_ci_cache.py b/tests/ci/test_ci_cache.py index 0f8acf2656c..3cdd6c78390 100644 --- a/tests/ci/test_ci_cache.py +++ b/tests/ci/test_ci_cache.py @@ -96,16 +96,27 @@ class TestCiCache(unittest.TestCase): pr_num=PR_NUM, ) - ### add some pending statuses for two batches and on non-release branch + ### add some pending statuses for two batches, non-release branch for job in JobNames: - ci_cache.push_pending(job, [0, 1], NUM_BATCHES, release_branch=False) - ci_cache_2.push_pending(job, [0, 1], NUM_BATCHES, release_branch=False) + ci_cache.push_pending(job, [0, 1, 2], NUM_BATCHES, release_branch=False) + ci_cache_2.push_pending(job, [0, 1, 2], NUM_BATCHES, release_branch=False) ### add success status for 0 batch, non-release branch + batch = 0 for job in JobNames: - ci_cache.push_successful(job, 0, NUM_BATCHES, status, release_branch=False) + ci_cache.push_successful( + job, batch, NUM_BATCHES, status, release_branch=False + ) ci_cache_2.push_successful( - job, 0, NUM_BATCHES, status, release_branch=False + job, batch, NUM_BATCHES, status, release_branch=False + ) + + ### add failed status for 2 batch, non-release branch + batch = 2 + for job in JobNames: + ci_cache.push_failed(job, batch, NUM_BATCHES, status, release_branch=False) + ci_cache_2.push_failed( + job, batch, NUM_BATCHES, status, release_branch=False ) ### check all expected directories were created on s3 mock @@ -128,7 +139,7 @@ class TestCiCache(unittest.TestCase): ) ### check number of cache files is as expected - FILES_PER_JOB = 3 # 1 successful + 2 pending batches = 3 + FILES_PER_JOB = 5 # 1 successful + 1 failed + 3 pending batches = 5 self.assertEqual( len( s3_mock.files_on_s3_paths[ @@ -219,7 +230,7 @@ class TestCiCache(unittest.TestCase): ci_cache.push_successful(job, 0, NUM_BATCHES, status, release_branch=True) ### check number of cache files is as expected - FILES_PER_JOB = 6 # 1 successful + 1 successful_release + 2 pending batches + 2 pending batches release = 6 + FILES_PER_JOB = 8 # 1 successful + 1 failed + 1 successful_release + 3 pending batches + 2 pending batches release = 8 self.assertEqual( len( s3_mock.files_on_s3_paths[ @@ -252,6 +263,9 @@ class TestCiCache(unittest.TestCase): self.assertEqual(ci_cache.is_pending(job, 1, NUM_BATCHES, False), True) self.assertEqual(ci_cache.is_pending(job, 1, NUM_BATCHES, True), True) + self.assertEqual(ci_cache.is_failed(job, 2, NUM_BATCHES, False), True) + self.assertEqual(ci_cache.is_failed(job, 2, NUM_BATCHES, True), False) + status2 = ci_cache.get_successful(job, 0, NUM_BATCHES) assert status2 and status2.pr_num == PR_NUM status2 = ci_cache.get_successful(job, 1, NUM_BATCHES) @@ -273,6 +287,13 @@ class TestCiCache(unittest.TestCase): self.assertEqual(ci_cache.is_pending(job, 1, NUM_BATCHES, False), True) self.assertEqual(ci_cache.is_pending(job, 1, NUM_BATCHES, True), True) + self.assertEqual(ci_cache.is_failed(job, 2, NUM_BATCHES, False), True) + self.assertEqual(ci_cache.is_failed(job, 2, NUM_BATCHES, True), False) + + # is_pending() is false for failed jobs batches + self.assertEqual(ci_cache.is_pending(job, 2, NUM_BATCHES, False), False) + self.assertEqual(ci_cache.is_pending(job, 2, NUM_BATCHES, True), False) + status2 = ci_cache.get_successful(job, 0, NUM_BATCHES) assert status2 and status2.pr_num == PR_NUM status2 = ci_cache.get_successful(job, 1, NUM_BATCHES) From b112fd1e3cde6088ec403fe9c2fca6b2dd828f4f Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 8 Feb 2024 19:09:28 +0000 Subject: [PATCH 073/103] CI: ci test await #do_not_test --- tests/ci/ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 5c33d4e02a8..ce8d1c8c20e 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1274,7 +1274,7 @@ def _update_gh_statuses_action(indata: Dict, s3: S3Helper) -> None: with concurrent.futures.ThreadPoolExecutor() as executor: futures = [] for job in job_digests: - if job not in jobs_to_skip or job not in jobs_to_do: + if job not in jobs_to_skip and job not in jobs_to_do: # no need to create status for job that are not supposed to be executed continue if CI_CONFIG.is_build_job(job): From a2beae80f5077e0a477df9a136f8f32224e67246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 8 Feb 2024 20:57:33 +0100 Subject: [PATCH 074/103] Improve test --- .../02985_shard_query_start_time.reference | 4 ++-- .../02985_shard_query_start_time.sql | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/02985_shard_query_start_time.reference b/tests/queries/0_stateless/02985_shard_query_start_time.reference index 1957f3a9604..ff563ea1d53 100644 --- a/tests/queries/0_stateless/02985_shard_query_start_time.reference +++ b/tests/queries/0_stateless/02985_shard_query_start_time.reference @@ -1,2 +1,2 @@ -1 1 -1 1 +QueryStart 2 2 2 2 +QueryFinish 2 2 2 2 diff --git a/tests/queries/0_stateless/02985_shard_query_start_time.sql b/tests/queries/0_stateless/02985_shard_query_start_time.sql index b0d8d2b6e53..c31d81e58ae 100644 --- a/tests/queries/0_stateless/02985_shard_query_start_time.sql +++ b/tests/queries/0_stateless/02985_shard_query_start_time.sql @@ -1,29 +1,34 @@ DROP TABLE IF EXISTS sharded_table; CREATE TABLE sharded_table (dummy UInt8) ENGINE = Distributed('test_cluster_two_shards', 'system', 'one'); +SET prefer_localhost_replica=0; SELECT * FROM sharded_table FORMAT Null SETTINGS log_comment='02985_shard_query_start_time_query_1'; SYSTEM FLUSH LOGS; --- We do not test for query_start_time because that would conflict pretty easily +-- Check that there are 2 queries to shards and for each one query_start_time_microseconds is more recent +-- than initial_query_start_time_microseconds, and initial_query_start_time_microseconds matches the original query +-- query_start_time_microseconds WITH ( SELECT - (query_id, query_start_time_microseconds) + (query_id, query_start_time, query_start_time_microseconds) FROM system.query_log WHERE - event_date >= yesterday() + event_date >= yesterday() AND current_database = currentDatabase() AND log_comment = '02985_shard_query_start_time_query_1' AND type = 'QueryFinish' - ORDER BY query_start_time_microseconds DESC - LIMIT 1 ) AS id_and_start_tuple SELECT - query_start_time_microseconds > initial_query_start_time_microseconds, - initial_query_start_time_microseconds = id_and_start_tuple.2 + type, + countIf(query_start_time >= initial_query_start_time), -- Using >= because it's comparing seconds + countIf(query_start_time_microseconds > initial_query_start_time_microseconds), + countIf(initial_query_start_time = id_and_start_tuple.2), + countIf(initial_query_start_time_microseconds = id_and_start_tuple.3) FROM system.query_log WHERE - NOT is_initial_query AND initial_query_id = id_and_start_tuple.1; + NOT is_initial_query AND initial_query_id = id_and_start_tuple.1 +GROUP BY type; From 075da5602fdc03d4d5b15cd8d769704259b168a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 8 Feb 2024 21:10:49 +0100 Subject: [PATCH 075/103] Remove incorrect test --- ...nt_info_initial_query_start_time.reference | 8 --- ...de_client_info_initial_query_start_time.sh | 67 ------------------- 2 files changed, 75 deletions(-) delete mode 100644 tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.reference delete mode 100755 tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh diff --git a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.reference b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.reference deleted file mode 100644 index fbce8ae2026..00000000000 --- a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.reference +++ /dev/null @@ -1,8 +0,0 @@ -SELECT -3 0 0 -3 0 0 -INSERT -CHECK -1 -2 -6 0 2 diff --git a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh deleted file mode 100755 index 5da643bd17b..00000000000 --- a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest -# Tag no-fasttest: interserver mode requires SSL -# -# Test that checks that some of ClientInfo correctly passed in inter-server mode. -# NOTE: we need .sh test (.sql is not enough) because queries on remote nodes does not have current_database = currentDatabase() -# -# Check-style suppression: select * from system.query_log where current_database = currentDatabase(); - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -function get_query_id() { random_str 10; } - -$CLICKHOUSE_CLIENT -nm -q " - drop table if exists buf; - drop table if exists dist; - drop table if exists data; - - create table data (key Int) engine=Memory(); - create table dist as data engine=Distributed(test_cluster_interserver_secret, currentDatabase(), data, key); - create table dist_dist as data engine=Distributed(test_cluster_interserver_secret, currentDatabase(), dist, key); - system stop distributed sends dist; -" - -echo "SELECT" -query_id="$(get_query_id)" -# initialize connection, but actually if there are other tables that uses this -# cluster then, it will be created long time ago, but this is OK for this -# test, since we care about the difference between NOW() and there should -# not be any significant difference. -$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " - system flush logs; - select count(), countIf(initial_query_start_time_microseconds != query_start_time_microseconds), countIf(event_time - initial_query_start_time > 3) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; -" - -sleep 6 - -query_id="$(get_query_id)" -# this query (and all subsequent) should reuse the previous connection (at least most of the time) -$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" - -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " - system flush logs; - select count(), countIf(initial_query_start_time_microseconds != query_start_time_microseconds), countIf(event_time - initial_query_start_time > 3) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; -" - -echo "INSERT" -query_id="$(get_query_id)" -$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -nm -q " - insert into dist_dist values (1),(2); - select * from data; -" - -sleep 3 -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist_dist" -sleep 1 -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist" - -echo "CHECK" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " - select * from data order by key; - system flush logs; - select count(), countIf(initial_query_start_time_microseconds != query_start_time_microseconds), countIf(event_time - initial_query_start_time > 3) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; -" From e598ad9157e63cfd08800839c5c2d4d9a15ba95f Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Thu, 8 Feb 2024 21:26:48 +0000 Subject: [PATCH 076/103] Better logging for adaptive async timeouts Log the adaptive async timeout value after normalization. --- src/Interpreters/AsynchronousInsertQueue.cpp | 34 ++++++++------------ src/Interpreters/AsynchronousInsertQueue.h | 1 - 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index e7f292d9b77..44cc58cec84 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -353,7 +353,18 @@ AsynchronousInsertQueue::pushDataChunk(ASTPtr query, DataChunk chunk, ContextPtr auto [it, inserted] = shard.iterators.try_emplace(key.hash); auto now = std::chrono::steady_clock::now(); - auto timeout_ms = getBusyWaitTimeoutMs(settings, shard, shard_num, flush_time_points, now); + auto timeout_ms = getBusyWaitTimeoutMs(settings, shard, flush_time_points, now); + if (timeout_ms != shard.busy_timeout_ms) + { + LOG_TRACE( + log, + "Asynchronous timeout {} from {} to {} for queue shard {}.", + timeout_ms < shard.busy_timeout_ms ? "decreased" : "increased", + shard.busy_timeout_ms.count(), + timeout_ms.count(), + size_t(shard_num)); + } + if (inserted) it->second = shard.queue.emplace(now + timeout_ms, Container{key, std::make_unique(timeout_ms)}).first; @@ -431,7 +442,6 @@ AsynchronousInsertQueue::pushDataChunk(ASTPtr query, DataChunk chunk, ContextPtr AsynchronousInsertQueue::Milliseconds AsynchronousInsertQueue::getBusyWaitTimeoutMs( const Settings & settings, const QueueShard & shard, - size_t shard_num, const QueueShardFlushTimeHistory::TimePoints & flush_time_points, std::chrono::steady_clock::time_point now) const { @@ -460,13 +470,6 @@ AsynchronousInsertQueue::Milliseconds AsynchronousInsertQueue::getBusyWaitTimeou auto timeout_ms = std::max( std::chrono::duration_cast(shard.busy_timeout_ms * (1.0 + increase_rate)), shard.busy_timeout_ms + Milliseconds(1)); - if (timeout_ms != shard.busy_timeout_ms) - LOG_TRACE( - log, - "Async timeout increased from {} to {} for queue shard {}.", - shard.busy_timeout_ms.count(), - timeout_ms.count(), - shard_num); return normalize(timeout_ms); } @@ -475,18 +478,7 @@ AsynchronousInsertQueue::Milliseconds AsynchronousInsertQueue::getBusyWaitTimeou /// long enough (exceeding the adjusted timeout). /// This ensures the timeout value converges to the minimum over time for non-frequent inserts. else if (last_insert_time + decreased_timeout_ms < now && t1 + decreased_timeout_ms < t2) - { - auto timeout_ms = decreased_timeout_ms; - if (timeout_ms != shard.busy_timeout_ms) - LOG_TRACE( - log, - "Async timeout decreased from {} to {} for queue shard {}.", - shard.busy_timeout_ms.count(), - timeout_ms.count(), - shard_num); - - return normalize(timeout_ms); - } + return normalize(decreased_timeout_ms); return normalize(shard.busy_timeout_ms); } diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index c2c4755f192..17140030766 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -248,7 +248,6 @@ private: Milliseconds getBusyWaitTimeoutMs( const Settings & settings, const QueueShard & shard, - size_t shard_num, const QueueShardFlushTimeHistory::TimePoints & flush_time_points, std::chrono::steady_clock::time_point now) const; From dbc288cd51affc3f6758a66ac51130aa8b5b565f Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Thu, 8 Feb 2024 23:13:44 +0100 Subject: [PATCH 077/103] Update ne-tormozit.md Fix broken youtube video embedding --- docs/ru/faq/general/ne-tormozit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/faq/general/ne-tormozit.md b/docs/ru/faq/general/ne-tormozit.md index 0f888de839f..6d0803680a8 100644 --- a/docs/ru/faq/general/ne-tormozit.md +++ b/docs/ru/faq/general/ne-tormozit.md @@ -20,6 +20,6 @@ sidebar_position: 11 Если вы не видели наших футболок, посмотрите видео о ClickHouse. Например, вот это: -![iframe](https://www.youtube.com/embed/bSyQahMVZ7w) + P.S. Эти футболки не продаются, а распространяются бесплатно на большинстве митапов [ClickHouse](https://clickhouse.com/#meet), обычно в награду за самые интересные вопросы или другие виды активного участия. From fe6d8316a35ce5ca93759a3eac2e36193476af22 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 17:34:43 -0800 Subject: [PATCH 078/103] [Docs] Add default cloud core settings --- docs/en/operations/settings/settings.md | 44 ++++++++++++++++++++----- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 3a826b095d2..a552616b3ec 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -508,7 +508,9 @@ Possible values: - Any positive integer number of hops. - 0 — No hops allowed. -Default value: 0. +Default value: `0`. + +Cloud default value: `10`. ## insert_null_as_default {#insert_null_as_default} @@ -1126,7 +1128,9 @@ Possible values: - 0 (or 1) — `INSERT SELECT` no parallel execution. - Positive integer. Bigger than 1. -Default value: 0. +Default value: `0`. + +Cloud default value: `2`. Parallel `INSERT SELECT` has effect only if the `SELECT` part is executed in parallel, see [max_threads](#max_threads) setting. Higher values will lead to higher memory usage. @@ -1207,7 +1211,9 @@ Default value: 10000. Cancels HTTP read-only queries (e.g. SELECT) when a client closes the connection without waiting for the response. -Default value: 0 +Default value: `0`. + +Cloud default value: `1`. ## poll_interval {#poll-interval} @@ -1946,6 +1952,8 @@ Possible values: Default value: `200`. +Cloud default value: `1000`. + ### async_insert_poll_timeout_ms {#async-insert-poll-timeout-ms} Timeout in milliseconds for polling data from asynchronous insert queue. @@ -2132,6 +2140,8 @@ Possible values: Default value: 0 +Cloud default value: `20`. + Keeper request retries are done after some timeout. The timeout is controlled by the following settings: `insert_keeper_retry_initial_backoff_ms`, `insert_keeper_retry_max_backoff_ms`. The first retry is done after `insert_keeper_retry_initial_backoff_ms` timeout. The consequent timeouts will be calculated as follows: ``` @@ -2660,6 +2670,8 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Default value: 1000000000 nanoseconds (once a second). +Cloud default value: `0`. + See also: - System table [trace_log](../../operations/system-tables/trace_log.md/#system_tables-trace_log) @@ -2683,6 +2695,8 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Default value: 1000000000 nanoseconds. +Cloud default value: `0`. + See also: - System table [trace_log](../../operations/system-tables/trace_log.md/#system_tables-trace_log) @@ -2804,6 +2818,8 @@ Possible values: Default value: `0`. +Cloud default value: `1`. + **See Also** - [Distributed Table Engine](../../engines/table-engines/special/distributed.md/#distributed) @@ -3319,7 +3335,9 @@ Possible values: - a string representing any valid table engine name -Default value: `None` +Default value: `None`. + +Cloud default value: `SharedMergeTree`. **Example** @@ -3689,6 +3707,8 @@ Possible values: Default value: `0`. +Cloud default value: `1`. + ## live_view_heartbeat_interval {#live-view-heartbeat-interval} Sets the heartbeat interval in seconds to indicate [live view](../../sql-reference/statements/create/view.md/#live-view) is alive . @@ -3933,6 +3953,8 @@ Possible values: Default value: `throw`. +Cloud default value: `none`. + ## flatten_nested {#flatten-nested} Sets the data format of a [nested](../../sql-reference/data-types/nested-data-structures/index.md) columns. @@ -4068,6 +4090,8 @@ Possible values: Default value: `1`. +Cloud default value: `0`. + :::note `alter_sync` is applicable to `Replicated` tables only, it does nothing to alters of not `Replicated` tables. ::: @@ -4723,6 +4747,8 @@ other connections are cancelled. Queries with `max_parallel_replicas > 1` are su Enabled by default. +Disabled by default on Cloud. + ## hedged_connection_timeout {#hedged_connection_timeout} If we can't establish connection with replica after this timeout in hedged requests, we start working with the next replica without cancelling connection to the previous. @@ -5348,10 +5374,11 @@ Default value: `false`. ## max_partition_size_to_drop -Restriction on dropping partitions in query time. +Restriction on dropping partitions in query time. The value 0 means that you can drop partitions without any restrictions. Default value: 50 GB. -The value 0 means that you can drop partitions without any restrictions. + +Cloud default value: 1 TB. :::note This query setting overwrites its server setting equivalent, see [max_partition_size_to_drop](/docs/en/operations/server-configuration-parameters/settings.md/#max-partition-size-to-drop) @@ -5359,10 +5386,11 @@ This query setting overwrites its server setting equivalent, see [max_partition_ ## max_table_size_to_drop -Restriction on deleting tables in query time. +Restriction on deleting tables in query time. The value 0 means that you can delete all tables without any restrictions. Default value: 50 GB. -The value 0 means that you can delete all tables without any restrictions. + +Cloud default value: 1 TB. :::note This query setting overwrites its server setting equivalent, see [max_table_size_to_drop](/docs/en/operations/server-configuration-parameters/settings.md/#max-table-size-to-drop) From d17a12fe6451e02dc13101cbdacb4e81951c239e Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 17:46:37 -0800 Subject: [PATCH 079/103] [Docs] Fix for default cloud core settings --- docs/en/operations/settings/settings.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index a552616b3ec..b6db3ccc197 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3707,8 +3707,6 @@ Possible values: Default value: `0`. -Cloud default value: `1`. - ## live_view_heartbeat_interval {#live-view-heartbeat-interval} Sets the heartbeat interval in seconds to indicate [live view](../../sql-reference/statements/create/view.md/#live-view) is alive . @@ -3915,6 +3913,8 @@ Possible values: Default value: `0`. +Cloud default value: `1`. + ## database_replicated_initial_query_timeout_sec {#database_replicated_initial_query_timeout_sec} Sets how long initial DDL query should wait for Replicated database to process previous DDL queue entries in seconds. From 2954cdf200f9f03dc1fa4c1a5fff37454edba3b7 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 18:06:34 -0800 Subject: [PATCH 080/103] [Docs] Add cloud default values for query complexity --- .../operations/settings/query-complexity.md | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/settings/query-complexity.md b/docs/en/operations/settings/query-complexity.md index 9a80f977ed1..f6d5a2a5017 100644 --- a/docs/en/operations/settings/query-complexity.md +++ b/docs/en/operations/settings/query-complexity.md @@ -1,4 +1,4 @@ ---- +![image](https://github.com/ClickHouse/ClickHouse/assets/3936029/17039d32-4699-4498-bcff-947079345a66)--- slug: /en/operations/settings/query-complexity sidebar_position: 59 sidebar_label: Restrictions on Query Complexity @@ -28,6 +28,8 @@ The maximum amount of RAM to use for running a query on a single server. The default setting is unlimited (set to `0`). +Cloud default value: 17 GB. + The setting does not consider the volume of available memory or the total volume of memory on the machine. The restriction applies to a single query within a single server. You can use `SHOW PROCESSLIST` to see the current memory consumption for each query. @@ -104,7 +106,9 @@ Possible values: - Maximum volume of RAM (in bytes) that can be used by the single [GROUP BY](../../sql-reference/statements/select/group-by.md#select-group-by-clause) operation. - 0 — `GROUP BY` in external memory disabled. -Default value: 0. +Default value: `0`. + +Cloud default value: 8 GB. ## max_bytes_before_external_sort {#settings-max_bytes_before_external_sort} @@ -115,6 +119,8 @@ Enables or disables execution of `ORDER BY` clauses in external memory. See [ORD Default value: 0. +Cloud default value: 8 GB. + ## max_rows_to_sort {#max-rows-to-sort} A maximum number of rows before sorting. This allows you to limit memory consumption when sorting. @@ -129,7 +135,11 @@ What to do if the number of rows received before sorting exceeds one of the limi ## max_result_rows {#setting-max_result_rows} -Limit on the number of rows in the result. Also checked for subqueries, and on remote servers when running parts of a distributed query. +Limit on the number of rows in the result. Also checked for subqueries, and on remote servers when running parts of a distributed query. No limit is applied when value is `0`. + +Default value: `0`. + +Cloud default value: `500000`. ## max_result_bytes {#max-result-bytes} @@ -137,10 +147,14 @@ Limit on the number of bytes in the result. The same as the previous setting. ## result_overflow_mode {#result-overflow-mode} -What to do if the volume of the result exceeds one of the limits: ‘throw’ or ‘break’. By default, throw. +What to do if the volume of the result exceeds one of the limits: ‘throw’ or ‘break’. Using ‘break’ is similar to using LIMIT. `Break` interrupts execution only at the block level. This means that amount of returned rows is greater than [max_result_rows](#setting-max_result_rows), multiple of [max_block_size](../../operations/settings/settings.md#setting-max_block_size) and depends on [max_threads](../../operations/settings/settings.md#max_threads). +Default value: `throw`. + +Cloud default value: `break`. + Example: ``` sql From 32305adb0865f67f589565f8e2416dcdf991c693 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 18:10:01 -0800 Subject: [PATCH 081/103] [Docs] Remove accidental link on query complexity page --- docs/en/operations/settings/query-complexity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/query-complexity.md b/docs/en/operations/settings/query-complexity.md index f6d5a2a5017..9c54e22b01d 100644 --- a/docs/en/operations/settings/query-complexity.md +++ b/docs/en/operations/settings/query-complexity.md @@ -1,4 +1,4 @@ -![image](https://github.com/ClickHouse/ClickHouse/assets/3936029/17039d32-4699-4498-bcff-947079345a66)--- +--- slug: /en/operations/settings/query-complexity sidebar_position: 59 sidebar_label: Restrictions on Query Complexity From f2f8a63ba8aa4c8fe4013632a51b17fc38326c49 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 18:14:21 -0800 Subject: [PATCH 082/103] [Docs] Change cloud default value for enable_filesystem_cache_on_write_operations --- docs/en/operations/storing-data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index b3ef1128c42..003277c8d4f 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -206,7 +206,7 @@ Some of these settings will disable cache features per query/profile that are en - `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` - allows to use cache in query only if it already exists, otherwise query data will not be written to local cache storage. Default: `false`. -- `enable_filesystem_cache_on_write_operations` - turn on `write-through` cache. This setting works only if setting `cache_on_write_operations` in cache configuration is turned on. Default: `false`. +- `enable_filesystem_cache_on_write_operations` - turn on `write-through` cache. This setting works only if setting `cache_on_write_operations` in cache configuration is turned on. Default: `false`. Cloud default value: `true`. - `enable_filesystem_cache_log` - turn on logging to `system.filesystem_cache_log` table. Gives a detailed view of cache usage per query. It can be turn on for specific queries or enabled in a profile. Default: `false`. From af6ae28f3cf965f7d41db819688b255529e818f9 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 18:16:44 -0800 Subject: [PATCH 083/103] [Docs] Specify cloud default value for date_time_input_format --- docs/en/operations/settings/settings-formats.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index eb09af44efd..1682e1e4673 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -212,6 +212,8 @@ Possible values: Default value: `'basic'`. +Cloud default value: `'best_effort'`. + See also: - [DateTime data type.](../../sql-reference/data-types/datetime.md) From 2dfd310cf26740344ecc633a5ae438f834649ce7 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Fri, 9 Feb 2024 03:31:30 +0000 Subject: [PATCH 084/103] Hide URL/S3 'headers' argument in SHOW CREATE --- src/Parsers/ASTFunction.cpp | 121 +++++++++++++++--- .../0_stateless/02968_url_args.reference | 9 +- tests/queries/0_stateless/02968_url_args.sql | 17 ++- 3 files changed, 126 insertions(+), 21 deletions(-) diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index e309dec2131..e7f7b48091a 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -71,6 +71,13 @@ namespace size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`). /// In all known cases secret arguments are consecutive bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments. + /// E.g. "headers" in `url('..', headers('foo' = '[HIDDEN]'))` + std::vector nested_maps; + + bool hasSecrets() const + { + return count != 0 || !nested_maps.empty(); + } }; Result getResult() const { return result; } @@ -127,6 +134,10 @@ namespace /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) findEncryptionFunctionSecretArguments(); } + else if (function.name == "url") + { + findURLSecretArguments(); + } } void findMySQLFunctionSecretArguments() @@ -143,6 +154,25 @@ namespace } } + /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should + /// always be at the end). Marks "headers" as secret, if found. + size_t excludeS3OrURLNestedMaps() + { + size_t count = arguments->size(); + while (count > 0) + { + const ASTFunction * f = arguments->at(count - 1)->as(); + if (!f) + break; + if (f->name == "headers") + result.nested_maps.push_back(f->name); + else if (f->name != "extra_credentials") + break; + count -= 1; + } + return count; + } + void findS3FunctionSecretArguments(bool is_cluster_function) { /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument. @@ -156,9 +186,10 @@ namespace } /// We should check other arguments first because we don't need to do any replacement in case of - /// s3('url', NOSIGN, 'format' [, 'compression']) - /// s3('url', 'format', 'structure' [, 'compression']) - if ((url_arg_idx + 3 <= arguments->size()) && (arguments->size() <= url_arg_idx + 4)) + /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) + /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) + size_t count = excludeS3OrURLNestedMaps(); + if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4)) { String second_arg; if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg)) @@ -174,7 +205,14 @@ namespace /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - markSecretArgument(url_arg_idx + 2); + if (url_arg_idx + 2 < count) + markSecretArgument(url_arg_idx + 2); + } + + void findURLSecretArguments() + { + if (!isNamedCollectionName(0)) + excludeS3OrURLNestedMaps(); } bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const @@ -347,6 +385,10 @@ namespace /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...) findS3TableEngineSecretArguments(); } + else if (engine_name == "URL") + { + findURLSecretArguments(); + } } void findExternalDistributedTableEngineSecretArguments() @@ -373,9 +415,10 @@ namespace } /// We should check other arguments first because we don't need to do any replacement in case of - /// S3('url', NOSIGN, 'format' [, 'compression']) - /// S3('url', 'format', 'compression') - if ((3 <= arguments->size()) && (arguments->size() <= 4)) + /// S3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) + /// S3('url', 'format', 'compression' [, extra_credentials(..)] [, headers(..)]) + size_t count = excludeS3OrURLNestedMaps(); + if ((3 <= count) && (count <= 4)) { String second_arg; if (tryGetStringFromArgument(1, &second_arg)) @@ -383,7 +426,7 @@ namespace if (boost::iequals(second_arg, "NOSIGN")) return; /// The argument after 'url' is "NOSIGN". - if (arguments->size() == 3) + if (count == 3) { if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) return; /// The argument after 'url' is a format: S3('url', 'format', ...) @@ -391,11 +434,12 @@ namespace } } - /// We replace 'aws_secret_access_key' with '[HIDDEN'] for the following signatures: + /// We replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: /// S3('url', 'aws_access_key_id', 'aws_secret_access_key') /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - markSecretArgument(2); + if (2 < count) + markSecretArgument(2); } void findDatabaseEngineSecretArguments() @@ -724,6 +768,25 @@ ASTSelectWithUnionQuery * ASTFunction::tryGetQueryArgument() const } +static bool formatNamedArgWithHiddenValue(IAST * arg, const IAST::FormatSettings & settings, IAST::FormatState & state, IAST::FormatStateStacked frame) +{ + const auto * equals_func = arg->as(); + if (!equals_func || (equals_func->name != "equals")) + return false; + const auto * expr_list = equals_func->arguments->as(); + if (!expr_list) + return false; + const auto & equal_args = expr_list->children; + if (equal_args.size() != 2) + return false; + + equal_args[0]->formatImpl(settings, state, frame); + settings.ostr << (settings.hilite ? IAST::hilite_operator : "") << " = " << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << "'[HIDDEN]'"; + + return true; +} + void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.expression_list_prepend_whitespace = false; @@ -1133,17 +1196,37 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format if (argument->as()) settings.ostr << "SETTINGS "; - if (!settings.show_secrets && (secret_arguments.start <= i) && (i < secret_arguments.start + secret_arguments.count)) + if (!settings.show_secrets) { - if (secret_arguments.are_named) + if (secret_arguments.start <= i && i < secret_arguments.start + secret_arguments.count) { - assert_cast(argument.get())->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens); - settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : ""); + if (secret_arguments.are_named) + { + assert_cast(argument.get())->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens); + settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : ""); + } + settings.ostr << "'[HIDDEN]'"; + if (size <= secret_arguments.start + secret_arguments.count && !secret_arguments.are_named) + break; /// All other arguments should also be hidden. + continue; + } + + const ASTFunction * function = argument->as(); + if (function && function->arguments && std::count(secret_arguments.nested_maps.begin(), secret_arguments.nested_maps.end(), function->name) != 0) + { + /// headers('foo' = '[HIDDEN]', 'bar' = '[HIDDEN]') + settings.ostr << (settings.hilite ? hilite_function : "") << function->name << (settings.hilite ? hilite_none : "") << "("; + for (size_t j = 0; j < function->arguments->children.size(); ++j) + { + if (j != 0) + settings.ostr << ", "; + auto inner_arg = function->arguments->children[j]; + if (!formatNamedArgWithHiddenValue(inner_arg.get(), settings, state, nested_dont_need_parens)) + inner_arg->formatImpl(settings, state, nested_dont_need_parens); + } + settings.ostr << ")"; + continue; } - settings.ostr << "'[HIDDEN]'"; - if (size <= secret_arguments.start + secret_arguments.count && !secret_arguments.are_named) - break; /// All other arguments should also be hidden. - continue; } if ((i == 1) && special_hilite_regexp @@ -1166,7 +1249,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format bool ASTFunction::hasSecretParts() const { - return (FunctionSecretArgumentsFinder{*this}.getResult().count > 0) || childrenHaveSecretParts(); + return (FunctionSecretArgumentsFinder{*this}.getResult().hasSecrets()) || childrenHaveSecretParts(); } String getFunctionName(const IAST * ast) diff --git a/tests/queries/0_stateless/02968_url_args.reference b/tests/queries/0_stateless/02968_url_args.reference index aa19e45301c..1c3693e4a66 100644 --- a/tests/queries/0_stateless/02968_url_args.reference +++ b/tests/queries/0_stateless/02968_url_args.reference @@ -1 +1,8 @@ -CREATE TABLE default.a\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers(\'foo\' = \'bar\')) +CREATE TABLE default.a\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers(\'foo\' = \'[HIDDEN]\', \'a\' = \'[HIDDEN]\')) +CREATE TABLE default.b\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers()) +CREATE TABLE default.c\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', \'CSV\', headers(\'foo\' = \'[HIDDEN]\')) +CREATE TABLE default.d\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', headers(\'foo\' = \'[HIDDEN]\')) +CREATE VIEW default.e\n(\n `x` Int64\n) AS\nSELECT count()\nFROM url(\'https://example.com/\', CSV, headers(\'foo\' = \'[HIDDEN]\', \'a\' = \'[HIDDEN]\')) +CREATE VIEW default.f\n(\n `x` Int64\n) AS\nSELECT count()\nFROM url(\'https://example.com/\', CSV, headers()) +CREATE VIEW default.g\n(\n `x` Int64\n) AS\nSELECT count()\nFROM s3(\'https://example.s3.amazonaws.com/a.csv\', CSV, headers(\'foo\' = \'[HIDDEN]\')) +CREATE VIEW default.h\n(\n `x` Int64\n) AS\nSELECT count()\nFROM s3(\'https://example.s3.amazonaws.com/a.csv\', headers(\'foo\' = \'[HIDDEN]\')) diff --git a/tests/queries/0_stateless/02968_url_args.sql b/tests/queries/0_stateless/02968_url_args.sql index 8bee9fec0ac..e97ea381ea5 100644 --- a/tests/queries/0_stateless/02968_url_args.sql +++ b/tests/queries/0_stateless/02968_url_args.sql @@ -1,2 +1,17 @@ -create table a (x Int64) engine URL('https://example.com/', CSV, headers('foo' = 'bar')); +create table a (x Int64) engine URL('https://example.com/', CSV, headers('foo' = 'bar', 'a' = '13')); show create a; +create table b (x Int64) engine URL('https://example.com/', CSV, headers()); +show create b; +create table c (x Int64) engine S3('https://example.s3.amazonaws.com/a.csv', NOSIGN, CSV, headers('foo' = 'bar')); +show create c; +create table d (x Int64) engine S3('https://example.s3.amazonaws.com/a.csv', NOSIGN, headers('foo' = 'bar')); +show create d; + +create view e (x Int64) as select count() from url('https://example.com/', CSV, headers('foo' = 'bar', 'a' = '13')); +show create e; +create view f (x Int64) as select count() from url('https://example.com/', CSV, headers()); +show create f; +create view g (x Int64) as select count() from s3('https://example.s3.amazonaws.com/a.csv', CSV, headers('foo' = 'bar')); +show create g; +create view h (x Int64) as select count() from s3('https://example.s3.amazonaws.com/a.csv', headers('foo' = 'bar')); +show create h; From 2a34bbb0e027f40c65da7db475496c20a455e0a0 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 21:36:45 -0800 Subject: [PATCH 085/103] [Docs] Cloud core settings corrections Co-authored-by: Alexey Milovidov --- docs/en/operations/settings/settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index b6db3ccc197..9e259ea43b4 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1130,7 +1130,7 @@ Possible values: Default value: `0`. -Cloud default value: `2`. +Cloud default value: from `2` to `4`, depending on the service size. Parallel `INSERT SELECT` has effect only if the `SELECT` part is executed in parallel, see [max_threads](#max_threads) setting. Higher values will lead to higher memory usage. @@ -2138,7 +2138,7 @@ Possible values: - Positive integer. - 0 — Retries are disabled -Default value: 0 +Default value: 20 Cloud default value: `20`. @@ -3335,7 +3335,7 @@ Possible values: - a string representing any valid table engine name -Default value: `None`. +Default value: `MergeTree`. Cloud default value: `SharedMergeTree`. From 89d4b1e77b545c18d4ad6017aa04ebe2d66da192 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 21:39:08 -0800 Subject: [PATCH 086/103] [Docs] Specify that some query profiler settings are disabled in Cloud --- docs/en/operations/settings/settings.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 9e259ea43b4..a275878f32e 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2670,7 +2670,7 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Default value: 1000000000 nanoseconds (once a second). -Cloud default value: `0`. +**Temporarily disabled in ClickHouse Cloud.** See also: @@ -2695,7 +2695,7 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Default value: 1000000000 nanoseconds. -Cloud default value: `0`. +**Temporarily disabled in ClickHouse Cloud.** See also: From aa05f07130cb2fe6070a32198138348df4bf1a03 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Thu, 8 Feb 2024 21:41:11 -0800 Subject: [PATCH 087/103] [Docs] Corrections for default cloud values for query complexity Co-authored-by: Alexey Milovidov --- docs/en/operations/settings/query-complexity.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/operations/settings/query-complexity.md b/docs/en/operations/settings/query-complexity.md index 9c54e22b01d..d86f18ff982 100644 --- a/docs/en/operations/settings/query-complexity.md +++ b/docs/en/operations/settings/query-complexity.md @@ -28,7 +28,7 @@ The maximum amount of RAM to use for running a query on a single server. The default setting is unlimited (set to `0`). -Cloud default value: 17 GB. +Cloud default value: depends on the amount of RAM on the replica. The setting does not consider the volume of available memory or the total volume of memory on the machine. The restriction applies to a single query within a single server. @@ -108,7 +108,7 @@ Possible values: Default value: `0`. -Cloud default value: 8 GB. +Cloud default value: half the memory amount per replica. ## max_bytes_before_external_sort {#settings-max_bytes_before_external_sort} @@ -119,7 +119,7 @@ Enables or disables execution of `ORDER BY` clauses in external memory. See [ORD Default value: 0. -Cloud default value: 8 GB. +Cloud default value: half the memory amount per replica. ## max_rows_to_sort {#max-rows-to-sort} @@ -139,7 +139,7 @@ Limit on the number of rows in the result. Also checked for subqueries, and on r Default value: `0`. -Cloud default value: `500000`. +Cloud default value: `0`. ## max_result_bytes {#max-result-bytes} @@ -153,7 +153,7 @@ Using ‘break’ is similar to using LIMIT. `Break` interrupts execution only a Default value: `throw`. -Cloud default value: `break`. +Cloud default value: `throw`. Example: From e8fa8c3f7525c1507263b35222060dae81ae94f8 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Fri, 9 Feb 2024 06:36:45 +0000 Subject: [PATCH 088/103] no-fasttest --- tests/queries/0_stateless/02968_url_args.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02968_url_args.sql b/tests/queries/0_stateless/02968_url_args.sql index e97ea381ea5..a9ac96970e0 100644 --- a/tests/queries/0_stateless/02968_url_args.sql +++ b/tests/queries/0_stateless/02968_url_args.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + create table a (x Int64) engine URL('https://example.com/', CSV, headers('foo' = 'bar', 'a' = '13')); show create a; create table b (x Int64) engine URL('https://example.com/', CSV, headers()); From 359dda569337837be2ce08411188fa899ceb5c13 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Thu, 8 Feb 2024 22:38:53 -0800 Subject: [PATCH 089/103] Fix DB type check - now it'll refuse to create in Replicated databases (#59743) Fix DB type check - now it'll refuse to create in Replicated databases --- src/Interpreters/InterpreterCreateQuery.cpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 7133c9eef34..c491ee30321 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1076,15 +1076,22 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data { const auto * kind = create.is_dictionary ? "Dictionary" : "Table"; const auto * kind_upper = create.is_dictionary ? "DICTIONARY" : "TABLE"; + bool is_replicated_database_internal = database->getEngineName() == "Replicated" && getContext()->getClientInfo().is_replicated_database_internal; + bool from_path = create.attach_from_path.has_value(); + bool is_on_cluster = getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; - if (database->getEngineName() == "Replicated" && getContext()->getClientInfo().is_replicated_database_internal - && !internal) + if (is_replicated_database_internal && !internal) { if (create.uuid == UUIDHelpers::Nil) throw Exception(ErrorCodes::LOGICAL_ERROR, "Table UUID is not specified in DDL log"); } - bool from_path = create.attach_from_path.has_value(); + if (create.refresh_strategy && database->getEngineName() != "Atomic") + throw Exception(ErrorCodes::INCORRECT_QUERY, + "Refreshable materialized view requires Atomic database engine, but database {} has engine {}", create.getDatabase(), database->getEngineName()); + /// TODO: Support Replicated databases, only with Shared/ReplicatedMergeTree. + /// Figure out how to make the refreshed data appear all at once on other + /// replicas; maybe a replicated SYSTEM SYNC REPLICA query before the rename? if (database->getUUID() != UUIDHelpers::Nil) { @@ -1108,7 +1115,6 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data } else { - bool is_on_cluster = getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; bool has_uuid = create.uuid != UUIDHelpers::Nil || create.to_inner_uuid != UUIDHelpers::Nil; if (has_uuid && !is_on_cluster && !internal) { @@ -1121,13 +1127,6 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data "{} UUID specified, but engine of database {} is not Atomic", kind, create.getDatabase()); } - if (create.refresh_strategy && database->getEngineName() != "Atomic") - throw Exception(ErrorCodes::INCORRECT_QUERY, - "Refreshable materialized view requires Atomic database engine, but database {} has engine {}", create.getDatabase(), database->getEngineName()); - /// TODO: Support Replicated databases, only with Shared/ReplicatedMergeTree. - /// Figure out how to make the refreshed data appear all at once on other - /// replicas; maybe a replicated SYSTEM SYNC REPLICA query before the rename? - /// The database doesn't support UUID so we'll ignore it. The UUID could be set here because of either /// a) the initiator of `ON CLUSTER` query generated it to ensure the same UUIDs are used on different hosts; or /// b) `RESTORE from backup` query generated it to ensure the same UUIDs are used on different hosts. From 8655c11280c154f267b8f36fd24dc21c7b786aec Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 9 Feb 2024 11:45:15 +0100 Subject: [PATCH 090/103] Fix typo --- src/Storages/StorageMerge.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 09c38996b22..79d7b83cada 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -786,7 +786,7 @@ QueryTreeNodePtr replaceTableExpressionAndRemoveJoin( projection.clear(); NamesAndTypes projection_columns; - // Select only required columns from the table, because prjection list may contain: + // Select only required columns from the table, because projection list may contain: // 1. aggregate functions // 2. expressions referencing other tables of JOIN for (auto const & column_name : required_column_names) From 3ee2dda51142c5321837eeaf7a00404ac46c74f4 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 9 Feb 2024 12:59:06 +0100 Subject: [PATCH 091/103] Fix special build reports in release branches --- .github/workflows/release_branches.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index c5d8294b999..87fbf363f0b 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -206,13 +206,8 @@ jobs: if: ${{ !cancelled() }} needs: - RunConfig - - BuilderDebRelease - - BuilderDebAarch64 - - BuilderDebAsan - - BuilderDebTsan - - BuilderDebUBsan - - BuilderDebMsan - - BuilderDebDebug + - BuilderBinDarwin + - BuilderBinDarwinAarch64 uses: ./.github/workflows/reusable_test.yml with: test_name: ClickHouse special build check From 9dfe3f55590d3f094b6bb7dffda6e4d8f7b44739 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Fri, 9 Feb 2024 12:21:11 +0000 Subject: [PATCH 092/103] CI: do not reuse builds on release branches #job_package_debug #job_style_check --- .github/workflows/master.yml | 22 +++++++++++++--------- .github/workflows/release_branches.yml | 11 ++++++++--- .github/workflows/reusable_build.yml | 9 +++++++-- tests/ci/ci.py | 8 +++++++- tests/ci/ci_config.py | 5 ----- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index dac1332adc6..33f98e492b5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -318,15 +318,19 @@ jobs: run_command: | python3 build_report_check.py "$CHECK_NAME" MarkReleaseReady: - needs: [RunConfig, BuilderBinDarwin, BuilderBinDarwinAarch64, BuilderDebRelease, BuilderDebAarch64] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_test.yml - with: - test_name: Mark Commit Release Ready - runner_type: style-checker - data: ${{ needs.RunConfig.outputs.data }} - run_command: | - python3 mark_release_ready.py + needs: + - BuilderBinDarwin + - BuilderBinDarwinAarch64 + - BuilderDebRelease + - BuilderDebAarch64 + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + - name: Mark Commit Release Ready + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 mark_release_ready.py ############################################################################################ #################################### INSTALL PACKAGES ###################################### ############################################################################################ diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index c5d8294b999..c1e4a1800c7 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -91,6 +91,8 @@ jobs: build_name: package_release checkout_depth: 0 data: ${{ needs.RunConfig.outputs.data }} + # always rebuild on release branches to be able to publish from any commit + force: true BuilderDebAarch64: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} @@ -99,6 +101,8 @@ jobs: build_name: package_aarch64 checkout_depth: 0 data: ${{ needs.RunConfig.outputs.data }} + # always rebuild on release branches to be able to publish from any commit + force: true BuilderDebAsan: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} @@ -142,6 +146,8 @@ jobs: build_name: binary_darwin checkout_depth: 0 data: ${{ needs.RunConfig.outputs.data }} + # always rebuild on release branches to be able to publish from any commit + force: true BuilderBinDarwinAarch64: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} @@ -150,6 +156,8 @@ jobs: build_name: binary_darwin_aarch64 checkout_depth: 0 data: ${{ needs.RunConfig.outputs.data }} + # always rebuild on release branches to be able to publish from any commit + force: true ############################################################################################ ##################################### Docker images ####################################### ############################################################################################ @@ -225,7 +233,6 @@ jobs: run_command: | python3 build_report_check.py "$CHECK_NAME" MarkReleaseReady: - if: ${{ !failure() && !cancelled() }} needs: - BuilderBinDarwin - BuilderBinDarwinAarch64 @@ -235,8 +242,6 @@ jobs: steps: - name: Check out repository code uses: ClickHouse/checkout@v1 - with: - clear-repository: true - name: Mark Commit Release Ready run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index 6be9d30175e..80d78d93e1b 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -26,6 +26,10 @@ name: Build ClickHouse description: json ci data type: string required: true + force: + description: disallow job skipping + type: boolean + default: false additional_envs: description: additional ENV variables to setup the job type: string @@ -33,7 +37,7 @@ name: Build ClickHouse jobs: Build: name: Build-${{inputs.build_name}} - if: contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.build_name) + if: ${{ contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.build_name) || inputs.force }} env: GITHUB_JOB_OVERRIDDEN: Build-${{inputs.build_name}} runs-on: [self-hosted, '${{inputs.runner_type}}'] @@ -78,7 +82,8 @@ jobs: python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \ --infile ${{ toJson(inputs.data) }} \ --job-name "$BUILD_NAME" \ - --run + --run \ + ${{ inputs.force && '--force' || '' }} - name: Post # it still be build report to upload for failed build job if: ${{ !cancelled() }} diff --git a/tests/ci/ci.py b/tests/ci/ci.py index ce8d1c8c20e..213225d0ea8 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -816,6 +816,12 @@ def parse_args(parser: argparse.ArgumentParser) -> argparse.Namespace: default=False, help="skip fetching data about job runs, used in --configure action (for debugging and nigthly ci)", ) + parser.add_argument( + "--force", + action="store_true", + default=False, + help="Used with --run, force the job to run, omitting the ci cache", + ) # FIXME: remove, not used parser.add_argument( "--rebuild-all-binaries", @@ -1762,7 +1768,7 @@ def main() -> int: previous_status = job_status.status GHActions.print_in_group("Commit Status Data", job_status) - if previous_status: + if previous_status and not args.force: print( f"Commit status or Build Report is already present - job will be skipped with status: [{previous_status}]" ) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 7c8990e8d16..998d0876527 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -140,8 +140,6 @@ class JobNames(metaclass=WithIter): DOCS_CHECK = "Docs check" BUGFIX_VALIDATE = "tests bugfix validate check" - MARK_RELEASE_READY = "Mark Commit Release Ready" - # dynamically update JobName with Build jobs for attr_name in dir(Build): @@ -828,9 +826,6 @@ CI_CONFIG = CiConfig( ), }, other_jobs_configs={ - JobNames.MARK_RELEASE_READY: TestConfig( - "", job_config=JobConfig(release_only=True) - ), JobNames.DOCKER_SERVER: TestConfig( "", job_config=JobConfig( From c2320c2d160b3a04a03a3bff9fa738af09066c92 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Fri, 9 Feb 2024 13:17:49 +0000 Subject: [PATCH 093/103] CI: await tune ups #no_ci_cache #job_package_debug --- tests/ci/ci.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 213225d0ea8..47e20b3ec09 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -646,14 +646,14 @@ class CiCache: TIMEOUT = 3600 await_finished: Dict[str, List[int]] = {} round_cnt = 0 - while len(jobs_with_params) > 5 and round_cnt < 3: + while len(jobs_with_params) > 4 and round_cnt < 5: round_cnt += 1 GHActions.print_in_group( f"Wait pending jobs, round [{round_cnt}]:", list(jobs_with_params) ) # this is initial approach to wait pending jobs: - # start waiting for the next TIMEOUT seconds if there are more than X(=5) jobs to wait - # wait TIMEOUT seconds in rounds. Y(=3) is the max number of rounds + # start waiting for the next TIMEOUT seconds if there are more than X(=4) jobs to wait + # wait TIMEOUT seconds in rounds. Y(=5) is the max number of rounds expired_sec = 0 start_at = int(time.time()) while expired_sec < TIMEOUT and jobs_with_params: @@ -701,13 +701,11 @@ class CiCache: print( f"...awaiting continues... seconds left [{TIMEOUT - expired_sec}]" ) - if await_finished: - GHActions.print_in_group( - "Finished jobs:", - [f"{job}:{batches}" for job, batches in await_finished.items()], - ) - else: - print("Awaiting FAILED. No job has finished successfully.") + if await_finished: + GHActions.print_in_group( + f"Finished jobs, round [{round_cnt}]:", + [f"{job}:{batches}" for job, batches in await_finished.items()], + ) GHActions.print_in_group( "Remaining jobs:", [f"{job}:{params['batches']}" for job, params in jobs_with_params.items()], From 50828dade25fa0f431907f9a466c305ca3d6e1b2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 9 Feb 2024 14:53:03 +0100 Subject: [PATCH 094/103] Fix stacktraces on MacOS (#59690) --- CMakeLists.txt | 15 +++++--- contrib/libunwind-cmake/unwind-override.c | 4 +++ contrib/llvm-project | 2 +- src/Common/StackTrace.cpp | 43 ++++++++++++++++++++++- 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 063cfc77302..8b232aa12ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -254,10 +254,17 @@ endif() include(cmake/cpu_features.cmake) -# Asynchronous unwind tables are needed for Query Profiler. -# They are already by default on some platforms but possibly not on all platforms. -# Enable it explicitly. -set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables") + +# Query Profiler doesn't work on MacOS for several reasons +# - PHDR cache is not available +# - We use native functionality to get stacktraces which is not async signal safe +# and thus we don't need to generate asynchronous unwind tables +if (NOT OS_DARWIN) + # Asynchronous unwind tables are needed for Query Profiler. + # They are already by default on some platforms but possibly not on all platforms. + # Enable it explicitly. + set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables") +endif() # Reproducible builds. if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") diff --git a/contrib/libunwind-cmake/unwind-override.c b/contrib/libunwind-cmake/unwind-override.c index 616bab6ae4b..57928d817eb 100644 --- a/contrib/libunwind-cmake/unwind-override.c +++ b/contrib/libunwind-cmake/unwind-override.c @@ -1,6 +1,10 @@ #include +/// On MacOS this function will be replaced with a dynamic symbol +/// from the system library. +#if !defined(OS_DARWIN) int backtrace(void ** buffer, int size) { return unw_backtrace(buffer, size); } +#endif diff --git a/contrib/llvm-project b/contrib/llvm-project index 2568a7cd129..d2142eed980 160000 --- a/contrib/llvm-project +++ b/contrib/llvm-project @@ -1 +1 @@ -Subproject commit 2568a7cd1297c7c3044b0f3cc0c23a6f6444d856 +Subproject commit d2142eed98046a47ff7112e3cc1e197c8a5cd80f diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 8431630b16c..7e683ae91de 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,15 @@ #include "config.h" +#include + +#if defined(OS_DARWIN) +/// This header contains functions like `backtrace` and `backtrace_symbols` +/// Which will be used for stack unwinding on Mac. +/// Read: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/backtrace.3.html +#include "execinfo.h" +#endif + namespace { /// Currently this variable is set up once on server startup. @@ -262,6 +272,33 @@ void StackTrace::forEachFrame( callback(current_inline_frame); } + callback(current_frame); + } +#elif defined(OS_DARWIN) + UNUSED(fatal); + + /// This function returns an array of string in a special (a little bit weird format) + /// The frame number, library name, address in hex, mangled symbol name, `+` sign, the offset. + char** strs = ::backtrace_symbols(frame_pointers.data(), static_cast(size)); + SCOPE_EXIT_SAFE({free(strs);}); + + for (size_t i = offset; i < size; ++i) + { + StackTrace::Frame current_frame; + + std::vector split; + boost::split(split, strs[i], isWhitespaceASCII); + split.erase( + std::remove_if( + split.begin(), split.end(), + [](const std::string & x) { return x.empty(); }), + split.end()); + assert(split.size() == 6); + + current_frame.virtual_addr = frame_pointers[i]; + current_frame.physical_addr = frame_pointers[i]; + current_frame.object = split[1]; + current_frame.symbol = split[3]; callback(current_frame); } #else @@ -306,7 +343,11 @@ StackTrace::StackTrace(const ucontext_t & signal_context) void StackTrace::tryCapture() { +#if defined(OS_DARWIN) + size = backtrace(frame_pointers.data(), capacity); +#else size = unw_backtrace(frame_pointers.data(), capacity); +#endif __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); } @@ -376,7 +417,7 @@ toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & s return callback(""); size_t frame_index = stack_trace.offset; -#if defined(__ELF__) && !defined(OS_FREEBSD) +#if (defined(__ELF__) && !defined(OS_FREEBSD)) || defined(OS_DARWIN) size_t inline_frame_index = 0; auto callback_wrapper = [&](const StackTrace::Frame & frame) { From f056e8b2c400fd676a37ab59727ebaf6d929e27c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 9 Feb 2024 16:54:00 +0100 Subject: [PATCH 095/103] Handle different timestamp related aspects of zip-files --- .../build_and_deploy_archive.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/ci/team_keys_lambda/build_and_deploy_archive.sh b/tests/ci/team_keys_lambda/build_and_deploy_archive.sh index 5b377d3c184..3c6c8e0ac1e 100644 --- a/tests/ci/team_keys_lambda/build_and_deploy_archive.sh +++ b/tests/ci/team_keys_lambda/build_and_deploy_archive.sh @@ -22,18 +22,29 @@ mkdir "$PACKAGE" cp app.py "$PACKAGE" if [ -f requirements.txt ]; then VENV=lambda-venv - rm -rf "$VENV" lambda-package.zip + rm -rf "$VENV" docker run --net=host --rm --user="${UID}" -e HOME=/tmp --entrypoint=/bin/bash \ --volume="${WORKDIR}/..:/ci" --workdir="/ci/${DIR_NAME}" "${DOCKER_IMAGE}" \ -exc " '$PY_EXEC' -m venv '$VENV' && source '$VENV/bin/activate' && - pip install -r requirements.txt + pip install -r requirements.txt && + # To have consistent pyc files + find '$VENV/lib' -name '*.pyc' -delete + find '$VENV/lib' ! -type d -exec touch -t 201212121212 {} + + python -m compileall " cp -rT "$VENV/lib/$PY_EXEC/site-packages/" "$PACKAGE" rm -r "$PACKAGE"/{pip,pip-*,setuptools,setuptools-*} + # zip stores metadata about timestamps + find "$PACKAGE" ! -type d -exec touch -t 201212121212 {} + fi -( cd "$PACKAGE" && zip -9 -r ../"$PACKAGE".zip . ) +( + export LC_ALL=c + cd "$PACKAGE" + # zip uses random files order by default, so we sort the files alphabetically + find . ! -type d -print0 | sort -z | tr '\0' '\n' | zip -XD -0 ../"$PACKAGE".zip --names-stdin +) ECHO=() if [ -n "$DRY_RUN" ]; then From a76e07207aa7645316fdba83ae3db5398b650c1a Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 9 Feb 2024 18:58:15 +0100 Subject: [PATCH 096/103] Add new setting azure_max_single_part_copy_size to SettingsChangesHistory.h --- src/Core/SettingsChangesHistory.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 5bd38d600c1..c453dd837eb 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -92,7 +92,8 @@ static std::map sett {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"}, {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"}, {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, - {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}}, + {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}, + {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."}}}, {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"}, From 69e118e58734aa822f86d33c3596310509af3c42 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 9 Feb 2024 19:34:21 +0100 Subject: [PATCH 097/103] Fix potential endless loop during merge (#59812) --- src/Processors/Merges/Algorithms/MergedData.h | 4 ++-- src/Processors/Transforms/ColumnGathererTransform.h | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Processors/Merges/Algorithms/MergedData.h b/src/Processors/Merges/Algorithms/MergedData.h index f92d20d22e1..7ffde835ad0 100644 --- a/src/Processors/Merges/Algorithms/MergedData.h +++ b/src/Processors/Merges/Algorithms/MergedData.h @@ -100,7 +100,7 @@ public: merged_rows = 0; sum_blocks_granularity = 0; ++total_chunks; - total_allocated_bytes += chunk.allocatedBytes(); + total_allocated_bytes += chunk.bytes(); need_flush = false; return chunk; @@ -122,7 +122,7 @@ public: { size_t merged_bytes = 0; for (const auto & column : columns) - merged_bytes += column->allocatedBytes(); + merged_bytes += column->byteSize(); if (merged_bytes >= max_block_size_bytes) return true; } diff --git a/src/Processors/Transforms/ColumnGathererTransform.h b/src/Processors/Transforms/ColumnGathererTransform.h index 821d04db0df..4e56cffa46a 100644 --- a/src/Processors/Transforms/ColumnGathererTransform.h +++ b/src/Processors/Transforms/ColumnGathererTransform.h @@ -145,10 +145,14 @@ void ColumnGathererStream::gather(Column & column_res) next_required_source = -1; - while (row_source_pos < row_sources_end - && column_res.size() < block_preferred_size_rows - && column_res.allocatedBytes() < block_preferred_size_bytes) + + /// We use do ... while here to ensure there will be at least one iteration of this loop. + /// Because the column_res.byteSize() could be bigger than block_preferred_size_bytes already at this point. + do { + if (row_source_pos >= row_sources_end) + break; + RowSourcePart row_source = *row_source_pos; size_t source_num = row_source.getSourceNum(); Source & source = sources[source_num]; @@ -191,7 +195,7 @@ void ColumnGathererStream::gather(Column & column_res) } source.pos += len; - } + } while (column_res.size() < block_preferred_size_rows && column_res.byteSize() < block_preferred_size_bytes); } } From dc52def9ebfcd10ac826108ac846d3c4d00792e0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 5 Feb 2024 12:33:17 +0300 Subject: [PATCH 098/103] Revert "Revert "Poco Logger small refactoring"" --- base/poco/Foundation/CMakeLists.txt | 6 + base/poco/Foundation/include/Poco/Logger.h | 42 ++-- .../include/Poco/RefCountedObject.h | 3 +- base/poco/Foundation/src/Logger.cpp | 226 +++++++++++------- src/Common/Logger.h | 12 + src/Common/tests/gtest_log.cpp | 73 ++++++ .../ObjectStorages/ObjectStorageFactory.cpp | 2 +- .../ObjectStorages/Web/WebObjectStorage.cpp | 2 +- utils/keeper-data-dumper/main.cpp | 2 +- 9 files changed, 255 insertions(+), 113 deletions(-) diff --git a/base/poco/Foundation/CMakeLists.txt b/base/poco/Foundation/CMakeLists.txt index dfb41a33fb1..5fe644d3057 100644 --- a/base/poco/Foundation/CMakeLists.txt +++ b/base/poco/Foundation/CMakeLists.txt @@ -166,6 +166,12 @@ set (SRCS ) add_library (_poco_foundation ${SRCS}) +target_link_libraries (_poco_foundation + PUBLIC + boost::headers_only + boost::system +) + add_library (Poco::Foundation ALIAS _poco_foundation) # TODO: remove these warning exclusions diff --git a/base/poco/Foundation/include/Poco/Logger.h b/base/poco/Foundation/include/Poco/Logger.h index cf202718662..883294a071a 100644 --- a/base/poco/Foundation/include/Poco/Logger.h +++ b/base/poco/Foundation/include/Poco/Logger.h @@ -22,6 +22,9 @@ #include #include #include + +#include + #include "Poco/Channel.h" #include "Poco/Format.h" #include "Poco/Foundation.h" @@ -34,7 +37,7 @@ namespace Poco class Exception; class Logger; -using LoggerPtr = std::shared_ptr; +using LoggerPtr = boost::intrusive_ptr; class Foundation_API Logger : public Channel /// Logger is a special Channel that acts as the main @@ -871,21 +874,11 @@ public: /// If the Logger does not yet exist, it is created, based /// on its parent logger. - static LoggerPtr getShared(const std::string & name); + static LoggerPtr getShared(const std::string & name, bool should_be_owned_by_shared_ptr_if_created = true); /// Returns a shared pointer to the Logger with the given name. /// If the Logger does not yet exist, it is created, based /// on its parent logger. - static Logger & unsafeGet(const std::string & name); - /// Returns a reference to the Logger with the given name. - /// If the Logger does not yet exist, it is created, based - /// on its parent logger. - /// - /// WARNING: This method is not thread safe. You should - /// probably use get() instead. - /// The only time this method should be used is during - /// program initialization, when only one thread is running. - static Logger & create(const std::string & name, Channel * pChannel, int level = Message::PRIO_INFORMATION); /// Creates and returns a reference to a Logger with the /// given name. The Logger's Channel and log level as set as @@ -932,6 +925,16 @@ public: static const std::string ROOT; /// The name of the root logger (""). +public: + struct LoggerEntry + { + Poco::Logger * logger; + bool owned_by_shared_ptr = false; + }; + + using LoggerMap = std::unordered_map; + using LoggerMapIterator = LoggerMap::iterator; + protected: Logger(const std::string & name, Channel * pChannel, int level); ~Logger(); @@ -940,12 +943,19 @@ protected: void log(const std::string & text, Message::Priority prio, const char * file, int line); static std::string format(const std::string & fmt, int argc, std::string argv[]); - static Logger & unsafeCreate(const std::string & name, Channel * pChannel, int level = Message::PRIO_INFORMATION); - static Logger & parent(const std::string & name); - static void add(Logger * pLogger); - static Logger * find(const std::string & name); private: + static std::pair unsafeGet(const std::string & name, bool get_shared); + static Logger * unsafeGetRawPtr(const std::string & name); + static std::pair unsafeCreate(const std::string & name, Channel * pChannel, int level = Message::PRIO_INFORMATION); + static Logger & parent(const std::string & name); + static std::pair add(Logger * pLogger); + static std::optional find(const std::string & name); + static Logger * findRawPtr(const std::string & name); + + friend void intrusive_ptr_add_ref(Logger * ptr); + friend void intrusive_ptr_release(Logger * ptr); + Logger(); Logger(const Logger &); Logger & operator=(const Logger &); diff --git a/base/poco/Foundation/include/Poco/RefCountedObject.h b/base/poco/Foundation/include/Poco/RefCountedObject.h index db966089e00..1f806bdacb1 100644 --- a/base/poco/Foundation/include/Poco/RefCountedObject.h +++ b/base/poco/Foundation/include/Poco/RefCountedObject.h @@ -53,11 +53,10 @@ protected: virtual ~RefCountedObject(); /// Destroys the RefCountedObject. + mutable std::atomic _counter; private: RefCountedObject(const RefCountedObject &); RefCountedObject & operator=(const RefCountedObject &); - - mutable std::atomic _counter; }; diff --git a/base/poco/Foundation/src/Logger.cpp b/base/poco/Foundation/src/Logger.cpp index cfc063c8979..16fc3a0480e 100644 --- a/base/poco/Foundation/src/Logger.cpp +++ b/base/poco/Foundation/src/Logger.cpp @@ -38,14 +38,7 @@ std::mutex & getLoggerMutex() return *logger_mutex; } -struct LoggerEntry -{ - Poco::Logger * logger; - bool owned_by_shared_ptr = false; -}; - -using LoggerMap = std::unordered_map; -LoggerMap * _pLoggerMap = nullptr; +Poco::Logger::LoggerMap * _pLoggerMap = nullptr; } @@ -309,38 +302,9 @@ void Logger::formatDump(std::string& message, const void* buffer, std::size_t le namespace { -struct LoggerDeleter -{ - void operator()(Poco::Logger * logger) - { - std::lock_guard lock(getLoggerMutex()); - - /// If logger infrastructure is destroyed just decrement logger reference count - if (!_pLoggerMap) - { - logger->release(); - return; - } - - auto it = _pLoggerMap->find(logger->name()); - assert(it != _pLoggerMap->end()); - - /** If reference count is 1, this means this shared pointer owns logger - * and need destroy it. - */ - size_t reference_count_before_release = logger->release(); - if (reference_count_before_release == 1) - { - assert(it->second.owned_by_shared_ptr); - _pLoggerMap->erase(it); - } - } -}; - - inline LoggerPtr makeLoggerPtr(Logger & logger) { - return std::shared_ptr(&logger, LoggerDeleter()); + return LoggerPtr(&logger, false /*add_ref*/); } } @@ -350,64 +314,87 @@ Logger& Logger::get(const std::string& name) { std::lock_guard lock(getLoggerMutex()); - Logger & logger = unsafeGet(name); - - /** If there are already shared pointer created for this logger - * we need to increment Logger reference count and now logger - * is owned by logger infrastructure. - */ - auto it = _pLoggerMap->find(name); - if (it->second.owned_by_shared_ptr) - { - it->second.logger->duplicate(); - it->second.owned_by_shared_ptr = false; - } - - return logger; + auto [it, inserted] = unsafeGet(name, false /*get_shared*/); + return *it->second.logger; } -LoggerPtr Logger::getShared(const std::string & name) +LoggerPtr Logger::getShared(const std::string & name, bool should_be_owned_by_shared_ptr_if_created) { std::lock_guard lock(getLoggerMutex()); - bool logger_exists = _pLoggerMap && _pLoggerMap->contains(name); + auto [it, inserted] = unsafeGet(name, true /*get_shared*/); - Logger & logger = unsafeGet(name); - - /** If logger already exists, then this shared pointer does not own it. - * If logger does not exists, logger infrastructure could be already destroyed - * or logger was created. + /** If during `unsafeGet` logger was created, then this shared pointer owns it. + * If logger was already created, then this shared pointer does not own it. */ - if (logger_exists) + if (inserted) { - logger.duplicate(); - } - else if (_pLoggerMap) - { - _pLoggerMap->find(name)->second.owned_by_shared_ptr = true; + if (should_be_owned_by_shared_ptr_if_created) + it->second.owned_by_shared_ptr = true; + else + it->second.logger->duplicate(); } - return makeLoggerPtr(logger); + return makeLoggerPtr(*it->second.logger); } -Logger& Logger::unsafeGet(const std::string& name) +std::pair Logger::unsafeGet(const std::string& name, bool get_shared) { - Logger* pLogger = find(name); - if (!pLogger) + std::optional optional_logger_it = find(name); + + bool should_recreate_logger = false; + + if (optional_logger_it) { + auto & logger_it = *optional_logger_it; + std::optional reference_count_before; + + if (get_shared) + { + reference_count_before = logger_it->second.logger->duplicate(); + } + else if (logger_it->second.owned_by_shared_ptr) + { + reference_count_before = logger_it->second.logger->duplicate(); + logger_it->second.owned_by_shared_ptr = false; + } + + /// Other thread already decided to delete this logger, but did not yet remove it from map + if (reference_count_before && reference_count_before == 0) + should_recreate_logger = true; + } + + if (!optional_logger_it || should_recreate_logger) + { + Logger * logger = nullptr; + if (name == ROOT) { - pLogger = new Logger(name, 0, Message::PRIO_INFORMATION); + logger = new Logger(name, nullptr, Message::PRIO_INFORMATION); } else { Logger& par = parent(name); - pLogger = new Logger(name, par.getChannel(), par.getLevel()); + logger = new Logger(name, par.getChannel(), par.getLevel()); } - add(pLogger); + + if (should_recreate_logger) + { + (*optional_logger_it)->second.logger = logger; + return std::make_pair(*optional_logger_it, true); + } + + return add(logger); } - return *pLogger; + + return std::make_pair(*optional_logger_it, false); +} + + +Logger * Logger::unsafeGetRawPtr(const std::string & name) +{ + return unsafeGet(name, false /*get_shared*/).first->second.logger; } @@ -415,24 +402,24 @@ Logger& Logger::create(const std::string& name, Channel* pChannel, int level) { std::lock_guard lock(getLoggerMutex()); - return unsafeCreate(name, pChannel, level); + return *unsafeCreate(name, pChannel, level).first->second.logger; } LoggerPtr Logger::createShared(const std::string & name, Channel * pChannel, int level) { std::lock_guard lock(getLoggerMutex()); - Logger & logger = unsafeCreate(name, pChannel, level); - _pLoggerMap->find(name)->second.owned_by_shared_ptr = true; + auto [it, inserted] = unsafeCreate(name, pChannel, level); + it->second.owned_by_shared_ptr = true; - return makeLoggerPtr(logger); + return makeLoggerPtr(*it->second.logger); } Logger& Logger::root() { std::lock_guard lock(getLoggerMutex()); - return unsafeGet(ROOT); + return *unsafeGetRawPtr(ROOT); } @@ -440,7 +427,11 @@ Logger* Logger::has(const std::string& name) { std::lock_guard lock(getLoggerMutex()); - return find(name); + auto optional_it = find(name); + if (!optional_it) + return nullptr; + + return (*optional_it)->second.logger; } @@ -459,20 +450,69 @@ void Logger::shutdown() } delete _pLoggerMap; - _pLoggerMap = 0; + _pLoggerMap = nullptr; } } -Logger* Logger::find(const std::string& name) +std::optional Logger::find(const std::string& name) { if (_pLoggerMap) { LoggerMap::iterator it = _pLoggerMap->find(name); if (it != _pLoggerMap->end()) - return it->second.logger; + return it; + + return {}; } - return 0; + + return {}; +} + +Logger * Logger::findRawPtr(const std::string & name) +{ + auto optional_it = find(name); + if (!optional_it) + return nullptr; + + return (*optional_it)->second.logger; +} + + +void intrusive_ptr_add_ref(Logger * ptr) +{ + ptr->duplicate(); +} + + +void intrusive_ptr_release(Logger * ptr) +{ + size_t reference_count_before = ptr->_counter.fetch_sub(1, std::memory_order_acq_rel); + if (reference_count_before != 1) + return; + + { + std::lock_guard lock(getLoggerMutex()); + + if (_pLoggerMap) + { + auto it = _pLoggerMap->find(ptr->name()); + + /** It is possible that during release other thread created logger and + * updated iterator in map. + */ + if (it != _pLoggerMap->end() && ptr == it->second.logger) + { + /** If reference count is 0, this means this intrusive pointer owns logger + * and need destroy it. + */ + assert(it->second.owned_by_shared_ptr); + _pLoggerMap->erase(it); + } + } + } + + delete ptr; } @@ -490,28 +530,28 @@ void Logger::names(std::vector& names) } } -Logger& Logger::unsafeCreate(const std::string & name, Channel * pChannel, int level) + +std::pair Logger::unsafeCreate(const std::string & name, Channel * pChannel, int level) { if (find(name)) throw ExistsException(); Logger* pLogger = new Logger(name, pChannel, level); - add(pLogger); - - return *pLogger; + return add(pLogger); } + Logger& Logger::parent(const std::string& name) { std::string::size_type pos = name.rfind('.'); if (pos != std::string::npos) { std::string pname = name.substr(0, pos); - Logger* pParent = find(pname); + Logger* pParent = findRawPtr(pname); if (pParent) return *pParent; else return parent(pname); } - else return unsafeGet(ROOT); + else return *unsafeGetRawPtr(ROOT); } @@ -579,12 +619,14 @@ namespace } -void Logger::add(Logger* pLogger) +std::pair Logger::add(Logger* pLogger) { if (!_pLoggerMap) - _pLoggerMap = new LoggerMap; + _pLoggerMap = new Logger::LoggerMap; - _pLoggerMap->emplace(pLogger->name(), LoggerEntry{pLogger, false /*owned_by_shared_ptr*/}); + auto result = _pLoggerMap->emplace(pLogger->name(), LoggerEntry{pLogger, false /*owned_by_shared_ptr*/}); + assert(result.second); + return result; } diff --git a/src/Common/Logger.h b/src/Common/Logger.h index 6dcdea9a9d8..0425da8c847 100644 --- a/src/Common/Logger.h +++ b/src/Common/Logger.h @@ -2,6 +2,8 @@ #include +#include + #include #include #include @@ -24,6 +26,16 @@ using LoggerRawPtr = Poco::Logger *; */ LoggerPtr getLogger(const std::string & name); +/** Get Logger with specified name. If the Logger does not exists, it is created. + * This overload was added for specific purpose, when logger is constructed from constexpr string. + * Logger is destroyed only during program shutdown. + */ +template +ALWAYS_INLINE LoggerPtr getLogger(const char (&name)[n]) +{ + return Poco::Logger::getShared(name, false /*should_be_owned_by_shared_ptr_if_created*/); +} + /** Create Logger with specified name, channel and logging level. * If Logger already exists, throws exception. * Logger is destroyed, when last shared ptr that refers to Logger with specified name is destroyed. diff --git a/src/Common/tests/gtest_log.cpp b/src/Common/tests/gtest_log.cpp index 622497fe2f5..6d2bd56ad77 100644 --- a/src/Common/tests/gtest_log.cpp +++ b/src/Common/tests/gtest_log.cpp @@ -9,6 +9,7 @@ #include #include #include +#include TEST(Logger, Log) @@ -100,3 +101,75 @@ TEST(Logger, SideEffects) LOG_TRACE(log, "test no throw {}", getLogMessageParamOrThrow()); } + +TEST(Logger, SharedRawLogger) +{ + { + std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + auto stream_channel = Poco::AutoPtr(new Poco::StreamChannel(stream)); + + auto shared_logger = getLogger("Logger_1"); + shared_logger->setChannel(stream_channel.get()); + shared_logger->setLevel("trace"); + + LOG_TRACE(shared_logger, "SharedLogger1Log1"); + LOG_TRACE(getRawLogger("Logger_1"), "RawLogger1Log"); + LOG_TRACE(shared_logger, "SharedLogger1Log2"); + + auto actual = stream.str(); + EXPECT_EQ(actual, "SharedLogger1Log1\nRawLogger1Log\nSharedLogger1Log2\n"); + } + { + std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + auto stream_channel = Poco::AutoPtr(new Poco::StreamChannel(stream)); + + auto * raw_logger = getRawLogger("Logger_2"); + raw_logger->setChannel(stream_channel.get()); + raw_logger->setLevel("trace"); + + LOG_TRACE(getLogger("Logger_2"), "SharedLogger2Log1"); + LOG_TRACE(raw_logger, "RawLogger2Log"); + LOG_TRACE(getLogger("Logger_2"), "SharedLogger2Log2"); + + auto actual = stream.str(); + EXPECT_EQ(actual, "SharedLogger2Log1\nRawLogger2Log\nSharedLogger2Log2\n"); + } +} + +TEST(Logger, SharedLoggersThreadSafety) +{ + static size_t threads_count = std::thread::hardware_concurrency(); + static constexpr size_t loggers_count = 10; + static constexpr size_t logger_get_count = 1000; + + Poco::Logger::root(); + + std::vector names; + + Poco::Logger::names(names); + size_t loggers_size_before = names.size(); + + std::vector threads; + + for (size_t thread_index = 0; thread_index < threads_count; ++thread_index) + { + threads.emplace_back([]() + { + for (size_t logger_index = 0; logger_index < loggers_count; ++logger_index) + { + for (size_t iteration = 0; iteration < logger_get_count; ++iteration) + { + getLogger("Logger_" + std::to_string(logger_index)); + } + } + }); + } + + for (auto & thread : threads) + thread.join(); + + Poco::Logger::names(names); + size_t loggers_size_after = names.size(); + + EXPECT_EQ(loggers_size_before, loggers_size_after); +} diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp index 866373db44a..b3626135177 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -102,7 +102,7 @@ void checkS3Capabilities( if (s3_capabilities.support_batch_delete && !checkBatchRemove(storage, key_with_trailing_slash)) { LOG_WARNING( - &Poco::Logger::get("S3ObjectStorage"), + getLogger("S3ObjectStorage"), "Storage for disk {} does not support batch delete operations, " "so `s3_capabilities.support_batch_delete` was automatically turned off during the access check. " "To remove this message set `s3_capabilities.support_batch_delete` for the disk to `false`.", diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp index 0223c24973e..786b23caf48 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp @@ -82,7 +82,7 @@ WebObjectStorage::loadFiles(const String & path, const std::unique_lock::max()); SnapshotsQueue snapshots_queue{1}; CoordinationSettingsPtr settings = std::make_shared(); From cb702f72ef040e3974d3f25e228aaeb8971cbdb7 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 5 Feb 2024 16:30:04 +0300 Subject: [PATCH 099/103] Updated implementation --- base/poco/Foundation/CMakeLists.txt | 6 - base/poco/Foundation/include/Poco/Logger.h | 7 +- .../include/Poco/RefCountedObject.h | 3 +- base/poco/Foundation/src/Logger.cpp | 112 +++++++----------- 4 files changed, 46 insertions(+), 82 deletions(-) diff --git a/base/poco/Foundation/CMakeLists.txt b/base/poco/Foundation/CMakeLists.txt index 5fe644d3057..dfb41a33fb1 100644 --- a/base/poco/Foundation/CMakeLists.txt +++ b/base/poco/Foundation/CMakeLists.txt @@ -166,12 +166,6 @@ set (SRCS ) add_library (_poco_foundation ${SRCS}) -target_link_libraries (_poco_foundation - PUBLIC - boost::headers_only - boost::system -) - add_library (Poco::Foundation ALIAS _poco_foundation) # TODO: remove these warning exclusions diff --git a/base/poco/Foundation/include/Poco/Logger.h b/base/poco/Foundation/include/Poco/Logger.h index 883294a071a..2a1cb33b407 100644 --- a/base/poco/Foundation/include/Poco/Logger.h +++ b/base/poco/Foundation/include/Poco/Logger.h @@ -23,8 +23,6 @@ #include #include -#include - #include "Poco/Channel.h" #include "Poco/Format.h" #include "Poco/Foundation.h" @@ -37,7 +35,7 @@ namespace Poco class Exception; class Logger; -using LoggerPtr = boost::intrusive_ptr; +using LoggerPtr = std::shared_ptr; class Foundation_API Logger : public Channel /// Logger is a special Channel that acts as the main @@ -953,9 +951,6 @@ private: static std::optional find(const std::string & name); static Logger * findRawPtr(const std::string & name); - friend void intrusive_ptr_add_ref(Logger * ptr); - friend void intrusive_ptr_release(Logger * ptr); - Logger(); Logger(const Logger &); Logger & operator=(const Logger &); diff --git a/base/poco/Foundation/include/Poco/RefCountedObject.h b/base/poco/Foundation/include/Poco/RefCountedObject.h index 1f806bdacb1..db966089e00 100644 --- a/base/poco/Foundation/include/Poco/RefCountedObject.h +++ b/base/poco/Foundation/include/Poco/RefCountedObject.h @@ -53,10 +53,11 @@ protected: virtual ~RefCountedObject(); /// Destroys the RefCountedObject. - mutable std::atomic _counter; private: RefCountedObject(const RefCountedObject &); RefCountedObject & operator=(const RefCountedObject &); + + mutable std::atomic _counter; }; diff --git a/base/poco/Foundation/src/Logger.cpp b/base/poco/Foundation/src/Logger.cpp index 16fc3a0480e..779af384b0b 100644 --- a/base/poco/Foundation/src/Logger.cpp +++ b/base/poco/Foundation/src/Logger.cpp @@ -302,9 +302,40 @@ void Logger::formatDump(std::string& message, const void* buffer, std::size_t le namespace { -inline LoggerPtr makeLoggerPtr(Logger & logger) +struct LoggerDeleter { - return LoggerPtr(&logger, false /*add_ref*/); + void operator()(Poco::Logger * logger) + { + std::lock_guard lock(getLoggerMutex()); + + /// If logger infrastructure is destroyed just decrement logger reference count + if (!_pLoggerMap) + { + logger->release(); + return; + } + + auto it = _pLoggerMap->find(logger->name()); + assert(it != _pLoggerMap->end()); + + /** If reference count is 1, this means this shared pointer owns logger + * and need destroy it. + */ + size_t reference_count_before_release = logger->release(); + if (reference_count_before_release == 1) + { + assert(it->second.owned_by_shared_ptr); + _pLoggerMap->erase(it); + } + } +}; + +inline LoggerPtr makeLoggerPtr(Logger & logger, bool owned_by_shared_ptr) +{ + if (owned_by_shared_ptr) + return LoggerPtr(&logger, LoggerDeleter()); + + return LoggerPtr(std::shared_ptr{}, &logger); } } @@ -327,15 +358,10 @@ LoggerPtr Logger::getShared(const std::string & name, bool should_be_owned_by_sh /** If during `unsafeGet` logger was created, then this shared pointer owns it. * If logger was already created, then this shared pointer does not own it. */ - if (inserted) - { - if (should_be_owned_by_shared_ptr_if_created) - it->second.owned_by_shared_ptr = true; - else - it->second.logger->duplicate(); - } + if (inserted && should_be_owned_by_shared_ptr_if_created) + it->second.owned_by_shared_ptr = true; - return makeLoggerPtr(*it->second.logger); + return makeLoggerPtr(*it->second.logger, it->second.owned_by_shared_ptr); } @@ -343,29 +369,20 @@ std::pair Logger::unsafeGet(const std::string& { std::optional optional_logger_it = find(name); - bool should_recreate_logger = false; - if (optional_logger_it) { auto & logger_it = *optional_logger_it; - std::optional reference_count_before; - if (get_shared) + if (logger_it->second.owned_by_shared_ptr) { - reference_count_before = logger_it->second.logger->duplicate(); - } - else if (logger_it->second.owned_by_shared_ptr) - { - reference_count_before = logger_it->second.logger->duplicate(); - logger_it->second.owned_by_shared_ptr = false; - } + logger_it->second.logger->duplicate(); - /// Other thread already decided to delete this logger, but did not yet remove it from map - if (reference_count_before && reference_count_before == 0) - should_recreate_logger = true; + if (!get_shared) + logger_it->second.owned_by_shared_ptr = false; + } } - if (!optional_logger_it || should_recreate_logger) + if (!optional_logger_it) { Logger * logger = nullptr; @@ -379,12 +396,6 @@ std::pair Logger::unsafeGet(const std::string& logger = new Logger(name, par.getChannel(), par.getLevel()); } - if (should_recreate_logger) - { - (*optional_logger_it)->second.logger = logger; - return std::make_pair(*optional_logger_it, true); - } - return add(logger); } @@ -412,7 +423,7 @@ LoggerPtr Logger::createShared(const std::string & name, Channel * pChannel, int auto [it, inserted] = unsafeCreate(name, pChannel, level); it->second.owned_by_shared_ptr = true; - return makeLoggerPtr(*it->second.logger); + return makeLoggerPtr(*it->second.logger, it->second.owned_by_shared_ptr); } Logger& Logger::root() @@ -479,43 +490,6 @@ Logger * Logger::findRawPtr(const std::string & name) } -void intrusive_ptr_add_ref(Logger * ptr) -{ - ptr->duplicate(); -} - - -void intrusive_ptr_release(Logger * ptr) -{ - size_t reference_count_before = ptr->_counter.fetch_sub(1, std::memory_order_acq_rel); - if (reference_count_before != 1) - return; - - { - std::lock_guard lock(getLoggerMutex()); - - if (_pLoggerMap) - { - auto it = _pLoggerMap->find(ptr->name()); - - /** It is possible that during release other thread created logger and - * updated iterator in map. - */ - if (it != _pLoggerMap->end() && ptr == it->second.logger) - { - /** If reference count is 0, this means this intrusive pointer owns logger - * and need destroy it. - */ - assert(it->second.owned_by_shared_ptr); - _pLoggerMap->erase(it); - } - } - } - - delete ptr; -} - - void Logger::names(std::vector& names) { std::lock_guard lock(getLoggerMutex()); From 2fc8895ae8aae3cb861faf04e424482ffcd081b3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 7 Feb 2024 20:04:51 +0300 Subject: [PATCH 100/103] Fixed tests --- src/Server/GRPCServer.cpp | 10 +++++----- src/Server/GRPCServer.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 7c532312612..15765f99b4b 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -76,7 +76,7 @@ namespace static std::once_flag once_flag; std::call_once(once_flag, [&config] { - static LoggerPtr logger = getLogger("grpc"); + static LoggerRawPtr logger = getRawLogger("grpc"); gpr_set_log_function([](gpr_log_func_args* args) { if (args->severity == GPR_LOG_SEVERITY_DEBUG) @@ -622,7 +622,7 @@ namespace class Call { public: - Call(CallType call_type_, std::unique_ptr responder_, IServer & iserver_, LoggerPtr log_); + Call(CallType call_type_, std::unique_ptr responder_, IServer & iserver_, LoggerRawPtr log_); ~Call(); void start(const std::function & on_finish_call_callback); @@ -664,7 +664,7 @@ namespace const CallType call_type; std::unique_ptr responder; IServer & iserver; - LoggerPtr log = nullptr; + LoggerRawPtr log = nullptr; std::optional session; ContextMutablePtr query_context; @@ -726,7 +726,7 @@ namespace }; // NOLINTEND(clang-analyzer-optin.performance.Padding) - Call::Call(CallType call_type_, std::unique_ptr responder_, IServer & iserver_, LoggerPtr log_) + Call::Call(CallType call_type_, std::unique_ptr responder_, IServer & iserver_, LoggerRawPtr log_) : call_type(call_type_), responder(std::move(responder_)), iserver(iserver_), log(log_) { } @@ -1851,7 +1851,7 @@ private: GRPCServer::GRPCServer(IServer & iserver_, const Poco::Net::SocketAddress & address_to_listen_) : iserver(iserver_) , address_to_listen(address_to_listen_) - , log(getLogger("GRPCServer")) + , log(getRawLogger("GRPCServer")) , runner(std::make_unique(*this)) {} diff --git a/src/Server/GRPCServer.h b/src/Server/GRPCServer.h index a9c8161298f..f86c2fe4ab7 100644 --- a/src/Server/GRPCServer.h +++ b/src/Server/GRPCServer.h @@ -47,7 +47,7 @@ private: IServer & iserver; const Poco::Net::SocketAddress address_to_listen; - LoggerPtr log; + LoggerRawPtr log; GRPCService grpc_service; std::unique_ptr grpc_server; std::unique_ptr queue; From 9c3b363c81a2ea394a4b4d4c55468f6dcf7b8c8b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 9 Feb 2024 21:38:56 +0300 Subject: [PATCH 101/103] Fixed tests --- src/Server/GRPCServer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Server/GRPCServer.h b/src/Server/GRPCServer.h index f86c2fe4ab7..70c16d3e9af 100644 --- a/src/Server/GRPCServer.h +++ b/src/Server/GRPCServer.h @@ -3,10 +3,11 @@ #include "config.h" #if USE_GRPC + +#include "clickhouse_grpc.grpc.pb.h" #include #include #include -#include "clickhouse_grpc.grpc.pb.h" namespace Poco { class Logger; } From 8b3c27d45bf6721c97231de0bba063463ecf2e0f Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 10 Feb 2024 15:00:31 +0300 Subject: [PATCH 102/103] Updated a list of trusted contributors --- tests/ci/lambda_shared_package/lambda_shared/pr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/lambda_shared_package/lambda_shared/pr.py b/tests/ci/lambda_shared_package/lambda_shared/pr.py index ce38475b3ee..a23d4bbb4c2 100644 --- a/tests/ci/lambda_shared_package/lambda_shared/pr.py +++ b/tests/ci/lambda_shared_package/lambda_shared/pr.py @@ -44,6 +44,7 @@ TRUSTED_CONTRIBUTORS = { "kitaisreal", "k-morozov", # Konstantin Morozov, Yandex Cloud "justindeguzman", # ClickHouse, Inc + "jrdi", # ClickHouse contributor, TinyBird ] } From fb8723a15c00cc6eff7c2ea2765a528788d11b59 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sat, 10 Feb 2024 13:13:26 +0100 Subject: [PATCH 103/103] Update pr.py --- tests/ci/lambda_shared_package/lambda_shared/pr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/lambda_shared_package/lambda_shared/pr.py b/tests/ci/lambda_shared_package/lambda_shared/pr.py index a23d4bbb4c2..1b4f827cc0a 100644 --- a/tests/ci/lambda_shared_package/lambda_shared/pr.py +++ b/tests/ci/lambda_shared_package/lambda_shared/pr.py @@ -44,7 +44,7 @@ TRUSTED_CONTRIBUTORS = { "kitaisreal", "k-morozov", # Konstantin Morozov, Yandex Cloud "justindeguzman", # ClickHouse, Inc - "jrdi", # ClickHouse contributor, TinyBird + "jrdi", # ClickHouse contributor, TinyBird ] }