From 653d48a552a66cec3bf50f8e9af67c9868826aab Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 29 Dec 2023 19:50:14 +0100 Subject: [PATCH 01/14] Flexible disks configuration --- .../AzureBlobStorage/AzureObjectStorage.h | 5 +- .../registerDiskAzureBlobStorage.cpp | 67 ----- .../Cached/CachedObjectStorage.cpp | 1 - .../Cached/CachedObjectStorage.h | 4 + .../DiskObjectStorageCommon.cpp | 40 --- .../ObjectStorages/DiskObjectStorageCommon.h | 23 -- .../ObjectStorages/HDFS/HDFSObjectStorage.h | 7 +- .../ObjectStorages/HDFS/registerDiskHDFS.cpp | 66 ----- src/Disks/ObjectStorages/IObjectStorage.h | 4 + .../Local/LocalObjectStorage.cpp | 1 - .../ObjectStorages/Local/LocalObjectStorage.h | 6 +- .../Local/registerLocalObjectStorage.cpp | 46 ---- .../ObjectStorages/MetadataStorageFactory.cpp | 99 +++++++ .../ObjectStorages/MetadataStorageFactory.h | 33 +++ .../ObjectStorages/ObjectStorageFactory.cpp | 248 ++++++++++++++++++ .../ObjectStorages/ObjectStorageFactory.h | 34 +++ .../RegisterDiskObjectStorage.cpp | 70 +++++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 1 - src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 6 + .../ObjectStorages/S3/checkBatchRemove.cpp | 64 +++++ .../ObjectStorages/S3/checkBatchRemove.h | 11 + src/Disks/ObjectStorages/S3/diskSettings.cpp | 1 - .../ObjectStorages/S3/registerDiskS3.cpp | 188 ------------- .../ObjectStorages/Web/WebObjectStorage.h | 4 + .../Web/registerDiskWebServer.cpp | 65 ----- src/Disks/registerDisks.cpp | 33 +-- 26 files changed, 594 insertions(+), 533 deletions(-) delete mode 100644 src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp delete mode 100644 src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp delete mode 100644 src/Disks/ObjectStorages/DiskObjectStorageCommon.h delete mode 100644 src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp delete mode 100644 src/Disks/ObjectStorages/Local/registerLocalObjectStorage.cpp create mode 100644 src/Disks/ObjectStorages/MetadataStorageFactory.cpp create mode 100644 src/Disks/ObjectStorages/MetadataStorageFactory.h create mode 100644 src/Disks/ObjectStorages/ObjectStorageFactory.cpp create mode 100644 src/Disks/ObjectStorages/ObjectStorageFactory.h create mode 100644 src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp create mode 100644 src/Disks/ObjectStorages/S3/checkBatchRemove.cpp create mode 100644 src/Disks/ObjectStorages/S3/checkBatchRemove.h delete mode 100644 src/Disks/ObjectStorages/S3/registerDiskS3.cpp delete mode 100644 src/Disks/ObjectStorages/Web/registerDiskWebServer.cpp diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 8e3d50418d3..936c3fce4a3 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -3,7 +3,6 @@ #if USE_AZURE_BLOB_STORAGE -#include #include #include #include @@ -64,6 +63,10 @@ public: std::string getName() const override { return "AzureObjectStorage"; } + std::string getBasePath() const override { return ""; } /// No namespaces in azure. + + std::string getTypeName() const override { return "azure_blob_storage"; } + bool exists(const StoredObject & object) const override; std::unique_ptr readObject( /// NOLINT diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp deleted file mode 100644 index 7ba9d21db62..00000000000 --- a/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "config.h" - -#include - -#if USE_AZURE_BLOB_STORAGE - -#include -#include - -#include -#include -#include -#include - -namespace DB -{ - -void registerDiskAzureBlobStorage(DiskFactory & factory, bool global_skip_access_check) -{ - auto creator = [global_skip_access_check]( - const String & name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context, - const DisksMap & /*map*/) - { - auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context); - - ObjectStoragePtr azure_object_storage = std::make_unique( - name, - getAzureBlobContainerClient(config, config_prefix), - getAzureBlobStorageSettings(config, config_prefix, context)); - - String key_prefix; - auto metadata_storage = std::make_shared(metadata_disk, key_prefix); - - std::shared_ptr azure_blob_storage_disk = std::make_shared( - name, - /* no namespaces */ key_prefix, - "DiskAzureBlobStorage", - std::move(metadata_storage), - std::move(azure_object_storage), - config, - config_prefix - ); - - bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false); - azure_blob_storage_disk->startup(context, skip_access_check); - - return azure_blob_storage_disk; - }; - - factory.registerDiskType("azure_blob_storage", creator); -} - -} - -#else - -namespace DB -{ - -void registerDiskAzureBlobStorage(DiskFactory &, bool /* global_skip_access_check */) {} - -} - -#endif diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp index 742d735cc95..ad0366d8642 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp @@ -1,6 +1,5 @@ #include "CachedObjectStorage.h" -#include #include #include #include diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h index 20b3a42540b..568fa64105e 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h @@ -24,8 +24,12 @@ public: std::string getName() const override { return fmt::format("CachedObjectStorage-{}({})", cache_config_name, object_storage->getName()); } + std::string getBasePath() const override { return object_storage->getBasePath(); } + bool exists(const StoredObject & object) const override; + std::string getTypeName() const override { return object_storage->getTypeName(); } + std::unique_ptr readObject( /// NOLINT const StoredObject & object, const ReadSettings & read_settings = ReadSettings{}, diff --git a/src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp b/src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp deleted file mode 100644 index cc9e4b0b712..00000000000 --- a/src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include -#include -#include -#include -#include - -namespace DB -{ - -static String getDiskMetadataPath( - const String & name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context) -{ - return config.getString(config_prefix + ".metadata_path", fs::path(context->getPath()) / "disks" / name / ""); -} - -std::pair prepareForLocalMetadata( - const String & name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context) -{ - /// where the metadata files are stored locally - auto metadata_path = getDiskMetadataPath(name, config, config_prefix, context); - fs::create_directories(metadata_path); - auto metadata_disk = std::make_shared(name + "-metadata", metadata_path, 0, config, config_prefix); - return std::make_pair(metadata_path, metadata_disk); -} - -bool isFileWithPersistentCache(const String & path) -{ - auto path_extension = std::filesystem::path(path).extension(); - return path_extension == ".idx" // index files. - || path_extension == ".mrk" || path_extension == ".mrk2" || path_extension == ".mrk3" /// mark files. - || path_extension == ".txt" || path_extension == ".dat"; -} - -} diff --git a/src/Disks/ObjectStorages/DiskObjectStorageCommon.h b/src/Disks/ObjectStorages/DiskObjectStorageCommon.h deleted file mode 100644 index 0bdbe0dfd36..00000000000 --- a/src/Disks/ObjectStorages/DiskObjectStorageCommon.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include -#include - -#include -#include - -#include - - -namespace DB -{ - -std::pair prepareForLocalMetadata( - const String & name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context); - -bool isFileWithPersistentCache(const String & path); - -} diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index fe0893f963b..6f54b2495f1 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -48,6 +48,7 @@ public: , hdfs_builder(createHDFSBuilder(hdfs_root_path_, config)) , hdfs_fs(createHDFSFS(hdfs_builder.get())) , settings(std::move(settings_)) + , hdfs_root_path(hdfs_root_path_) { data_source_description.type = DataSourceType::HDFS; data_source_description.description = hdfs_root_path_; @@ -57,6 +58,10 @@ public: std::string getName() const override { return "HDFSObjectStorage"; } + std::string getBasePath() const override { return hdfs_root_path; } + + std::string getTypeName() const override { return "hdfs"; } + DataSourceDescription getDataSourceDescription() const override { return data_source_description; @@ -123,8 +128,8 @@ private: HDFSBuilderWrapper hdfs_builder; HDFSFSPtr hdfs_fs; - SettingsPtr settings; + const std::string hdfs_root_path; DataSourceDescription data_source_description; }; diff --git a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp deleted file mode 100644 index 96ff0a91564..00000000000 --- a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - -void registerDiskHDFS(DiskFactory & factory, bool global_skip_access_check) -{ - auto creator = [global_skip_access_check]( - const String & name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context, - const DisksMap & /*map*/) -> DiskPtr - { - String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); - String uri{endpoint}; - checkHDFSURL(uri); - - if (uri.back() != '/') - throw Exception(ErrorCodes::BAD_ARGUMENTS, "HDFS path must ends with '/', but '{}' doesn't.", uri); - - std::unique_ptr settings = std::make_unique( - config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), - config.getInt(config_prefix + ".objects_chunk_size_to_delete", 1000), - context->getSettingsRef().hdfs_replication - ); - - - /// FIXME Cache currently unsupported :( - ObjectStoragePtr hdfs_storage = std::make_unique(uri, std::move(settings), config); - - auto [_, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context); - - auto metadata_storage = std::make_shared(metadata_disk, uri); - bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false); - - DiskPtr disk = std::make_shared( - name, - uri, - "DiskHDFS", - std::move(metadata_storage), - std::move(hdfs_storage), - config, - config_prefix); - - disk->startup(context, skip_access_check); - - return disk; - }; - - factory.registerDiskType("hdfs", creator); -} - -} diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index 72a4c432b20..fcdb5e6d164 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -84,6 +84,10 @@ public: virtual std::string getName() const = 0; + virtual std::string getBasePath() const = 0; + + virtual std::string getTypeName() const = 0; + /// Object exists or not virtual bool exists(const StoredObject & object) const = 0; diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index c20a27e2384..a72e43952a8 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h index 263eb3f7832..24090ba3a54 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h @@ -16,12 +16,16 @@ namespace DB class LocalObjectStorage : public IObjectStorage { public: - LocalObjectStorage(String key_prefix_); + explicit LocalObjectStorage(String key_prefix_); DataSourceDescription getDataSourceDescription() const override { return data_source_description; } std::string getName() const override { return "LocalObjectStorage"; } + std::string getBasePath() const override { return key_prefix; } + + std::string getTypeName() const override { return "local_blob_storage"; } + bool exists(const StoredObject & object) const override; std::unique_ptr readObject( /// NOLINT diff --git a/src/Disks/ObjectStorages/Local/registerLocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/registerLocalObjectStorage.cpp deleted file mode 100644 index 0b2c71fa09d..00000000000 --- a/src/Disks/ObjectStorages/Local/registerLocalObjectStorage.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -namespace DB -{ -void registerDiskLocalObjectStorage(DiskFactory & factory, bool global_skip_access_check) -{ - auto creator = [global_skip_access_check]( - const String & name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context, - const DisksMap & /*map*/) -> DiskPtr - { - String object_key_prefix; - UInt64 keep_free_space_bytes; - loadDiskLocalConfig(name, config, config_prefix, context, object_key_prefix, keep_free_space_bytes); - /// keys are mapped to the fs, object_key_prefix is a directory also - fs::create_directories(object_key_prefix); - - String type = config.getString(config_prefix + ".type"); - chassert(type == "local_blob_storage"); - - std::shared_ptr local_storage = std::make_shared(object_key_prefix); - MetadataStoragePtr metadata_storage; - auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context); - metadata_storage = std::make_shared(metadata_disk, object_key_prefix); - - auto disk = std::make_shared( - name, object_key_prefix, "Local", metadata_storage, local_storage, config, config_prefix); - disk->startup(context, global_skip_access_check); - return disk; - - }; - factory.registerDiskType("local_blob_storage", creator); -} - -} diff --git a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp new file mode 100644 index 00000000000..60da0b7867e --- /dev/null +++ b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NO_ELEMENTS_IN_CONFIG; + extern const int UNKNOWN_ELEMENT_IN_CONFIG; +} + +MetadataStorageFactory & MetadataStorageFactory::instance() +{ + static MetadataStorageFactory factory; + return factory; +} + +void MetadataStorageFactory::registerMetadataStorageType(const std::string & metadata_type, Creator creator) +{ + if (!registry.emplace(metadata_type, creator).second) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "MetadataStorageFactory: the metadata type '{}' is not unique", + metadata_type); + } +} + +MetadataStoragePtr MetadataStorageFactory::create( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ObjectStoragePtr object_storage, + const std::string & compatibility_type_hint) const +{ + if (compatibility_type_hint.empty() && !config.has(config_prefix + ".metadata_type")) + { + throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Expected `metadata_type` in config"); + } + + const auto type = config.getString(config_prefix + ".metadata_type"); + const auto it = registry.find(type); + + if (it == registry.end()) + { + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, + "MetadataStorageFactory: unknown metadata storage type: {}", type); + } + + return it->second(name, config, config_prefix, object_storage); +} + +void registerMetadataStorageFromDisk(MetadataStorageFactory & factory) +{ + auto creator = []( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ObjectStoragePtr object_storage) -> MetadataStoragePtr + { + auto metadata_path = config.getString(config_prefix + ".metadata_path", + fs::path(Context::getGlobalContextInstance()->getPath()) / "disks" / name / ""); + fs::create_directories(metadata_path); + auto metadata_disk = std::make_shared(name + "-metadata", metadata_path, 0, config, config_prefix); + return std::make_shared(metadata_disk, object_storage->getBasePath()); + }; + factory.registerMetadataStorageType("local", creator); +} + +void registerMetadataStorageFromDiskPlain(MetadataStorageFactory & factory) +{ + auto creator = []( + const std::string & /* name */, + const Poco::Util::AbstractConfiguration & /* config */, + const std::string & /* config_prefix */, + ObjectStoragePtr object_storage) -> MetadataStoragePtr + { + return std::make_shared(object_storage, object_storage->getBasePath()); + }; + factory.registerMetadataStorageType("plain", creator); +} + +void registerMetadataStorageFromStaticFilesWebServer(MetadataStorageFactory & factory) +{ + auto creator = []( + const std::string & /* name */, + const Poco::Util::AbstractConfiguration & /* config */, + const std::string & /* config_prefix */, + ObjectStoragePtr object_storage) -> MetadataStoragePtr + { + return std::make_shared(assert_cast(*object_storage)); + }; + factory.registerMetadataStorageType("web", creator); +} + +} diff --git a/src/Disks/ObjectStorages/MetadataStorageFactory.h b/src/Disks/ObjectStorages/MetadataStorageFactory.h new file mode 100644 index 00000000000..5f61125c599 --- /dev/null +++ b/src/Disks/ObjectStorages/MetadataStorageFactory.h @@ -0,0 +1,33 @@ +#pragma once +#include +#include + +namespace DB +{ + +class MetadataStorageFactory final : private boost::noncopyable +{ +public: + using Creator = std::function; + + static MetadataStorageFactory & instance(); + + void registerMetadataStorageType(const std::string & metadata_type, Creator creator); + + MetadataStoragePtr create( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ObjectStoragePtr object_storage, + const std::string & compatibility_type_hint) const; + +private: + using Registry = std::unordered_map; + Registry registry; +}; + +} diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp new file mode 100644 index 00000000000..b9a2d85e15a --- /dev/null +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -0,0 +1,248 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NO_ELEMENTS_IN_CONFIG; + extern const int UNKNOWN_ELEMENT_IN_CONFIG; +} + +ObjectStorageFactory & ObjectStorageFactory::instance() +{ + static ObjectStorageFactory factory; + return factory; +} + +void ObjectStorageFactory::registerObjectStorageType(const std::string & metadata_type, Creator creator) +{ + if (!registry.emplace(metadata_type, creator).second) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ObjectStorageFactory: the metadata type '{}' is not unique", + metadata_type); + } +} + +ObjectStoragePtr ObjectStorageFactory::create( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context, + bool skip_access_check) const +{ + std::string type; + if (config.has(config_prefix + ".obejct_storage_type")) + { + type = config.getString(config_prefix + ".object_storage_type"); + } + else if (config.has(config_prefix + ".type")) /// .type -- for compatibility. + { + type = config.getString(config_prefix + ".type"); + } + else + { + throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Expected `object_storage_type` in config"); + } + + const auto it = registry.find(type); + + if (it == registry.end()) + { + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, + "ObjectStorageFactory: unknown object storage type: {}", type); + } + + return it->second(name, config, config_prefix, context, skip_access_check); +} + +static S3::URI getS3URI(const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context) +{ + String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); + S3::URI uri(endpoint); + if (!uri.key.ends_with('/')) + uri.key.push_back('/'); + return uri; +} + +#if USE_AWS_S3 +void registerS3ObjectStorage(ObjectStorageFactory & factory) +{ + auto creator = []( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context, + bool skip_access_check) -> ObjectStoragePtr + { + auto uri = getS3URI(config, config_prefix, context); + auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); + auto settings = getSettings(config, config_prefix, context); + auto client = getClient(config, config_prefix, context, *settings); + + auto object_storage = std::make_shared( + std::move(client), std::move(settings), uri.version_id, + s3_capabilities, uri.bucket, uri.endpoint, uri.key, name); + + /// NOTE: should we still perform this check for clickhouse-disks? + if (!skip_access_check) + { + /// If `support_batch_delete` is turned on (default), check and possibly switch it off. + if (s3_capabilities.support_batch_delete && !checkBatchRemove(*object_storage, uri.key)) + { + LOG_WARNING( + &Poco::Logger::get("S3ObjectStorage"), + "Storage for disk {} does not support batch delete operations, " + "so `s3_capabilities.support_batch_delete` was automatically turned off during the access check. " + "To remove this message set `s3_capabilities.support_batch_delete` for the disk to `false`.", + name + ); + object_storage->setCapabilitiesSupportBatchDelete(false); + } + } + return object_storage; + }; + factory.registerObjectStorageType("s3", creator); +} + +void registerS3PlainObjectStorage(ObjectStorageFactory & factory) +{ + auto creator = []( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context, + bool /* skip_access_check */) -> ObjectStoragePtr + { + /// send_metadata changes the filenames (includes revision), while + /// s3_plain do not care about this, and expect that the file name + /// will not be changed. + /// + /// And besides, send_metadata does not make sense for s3_plain. + if (config.getBool(config_prefix + ".send_metadata", false)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "s3_plain does not supports send_metadata"); + + auto uri = getS3URI(config, config_prefix, context); + auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); + auto settings = getSettings(config, config_prefix, context); + auto client = getClient(config, config_prefix, context, *settings); + + return std::make_shared( + std::move(client), std::move(settings), uri.version_id, + s3_capabilities, uri.bucket, uri.endpoint, uri.key, name); + }; + factory.registerObjectStorageType("s3_plain", creator); +} +#endif + +#if USE_HDFS +void registerHDFSObjectStorage(ObjectStorageFactory & factory) +{ + auto creator = []( + const std::string & /* name */, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context, + bool /* skip_access_check */) -> ObjectStoragePtr + { + auto uri = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); + checkHDFSURL(uri); + if (uri.back() != '/') + throw Exception(ErrorCodes::BAD_ARGUMENTS, "HDFS path must ends with '/', but '{}' doesn't.", uri); + + std::unique_ptr settings = std::make_unique( + config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), + config.getInt(config_prefix + ".objects_chunk_size_to_delete", 1000), + context->getSettingsRef().hdfs_replication + ); + + return std::make_unique(uri, std::move(settings), config); + }; + factory.registerObjectStorageType("hdfs", creator); +} +#endif + +#if USE_AZURE_BLOB_STORAGE +void registerAzureObjectStorage(ObjectStorageFactory & factory) +{ + auto creator = []( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context, + bool /* skip_access_check */) -> ObjectStoragePtr + { + return std::make_unique( + name, + getAzureBlobContainerClient(config, config_prefix), + getAzureBlobStorageSettings(config, config_prefix, context)); + + }; + factory.registerObjectStorageType("azure_blob_storage", creator); +} +#endif + +void registerWebObjectStorage(ObjectStorageFactory & factory) +{ + auto creator = []( + const std::string & /* name */, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context, + bool /* skip_access_check */) -> ObjectStoragePtr + { + auto uri = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); + if (!uri.ends_with('/')) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "URI must end with '/', but '{}' doesn't.", uri); + try + { + Poco::URI poco_uri(uri); + } + catch (const Poco::Exception & e) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Bad URI: `{}`. Error: {}", uri, e.what()); + } + + return std::make_shared(uri, context); + }; + factory.registerObjectStorageType("web", creator); +} + +void registerLocalObjectStorage(ObjectStorageFactory & factory) +{ + auto creator = []( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context, + bool /* skip_access_check */) -> ObjectStoragePtr + { + String object_key_prefix; + UInt64 keep_free_space_bytes; + loadDiskLocalConfig(name, config, config_prefix, context, object_key_prefix, keep_free_space_bytes); + /// keys are mapped to the fs, object_key_prefix is a directory also + fs::create_directories(object_key_prefix); + return std::make_shared(object_key_prefix); + }; + factory.registerObjectStorageType("local", creator); +} + +} diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.h b/src/Disks/ObjectStorages/ObjectStorageFactory.h new file mode 100644 index 00000000000..3c13b3bca86 --- /dev/null +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.h @@ -0,0 +1,34 @@ +#pragma once +#include +#include + +namespace DB +{ + +class ObjectStorageFactory final : private boost::noncopyable +{ +public: + using Creator = std::function; + + static ObjectStorageFactory & instance(); + + void registerObjectStorageType(const std::string & metadata_type, Creator creator); + + ObjectStoragePtr create( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context, + bool skip_access_check) const; + +private: + using Registry = std::unordered_map; + Registry registry; +}; + +} diff --git a/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp b/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp new file mode 100644 index 00000000000..a5f5a6238f4 --- /dev/null +++ b/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int UNKNOWN_ELEMENT_IN_CONFIG; +} + +static std::string getCompatibilityMetadataTypeHint(const std::string & object_storage_type) +{ + /// TODO: change type name to enum + if (object_storage_type == "s3" + || object_storage_type == "hdfs" + || object_storage_type == "azure_blob_storage" + || object_storage_type == "local_blob_storage") + return "local"; + else if (object_storage_type == "s3_plain") + return "plain"; + else if (object_storage_type == "web") + return "web"; + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown object storage type: {}", object_storage_type); +} + +void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_check) +{ + auto creator = [global_skip_access_check]( + const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + ContextPtr context, + const DisksMap & /*map*/) -> DiskPtr + { + bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false); + auto object_storage = ObjectStorageFactory::instance().create(name, config, config_prefix, context, skip_access_check); + auto compatibility_metadata_type_hint = config.has("metadata_type") ? "" : getCompatibilityMetadataTypeHint(object_storage->getTypeName()); + auto metadata_storage = MetadataStorageFactory::instance().create(name, config, config_prefix, object_storage, compatibility_metadata_type_hint); + + DiskObjectStoragePtr disk = std::make_shared( + name, + object_storage->getBasePath(), + "Disk" + object_storage->getTypeName(), + std::move(metadata_storage), + std::move(object_storage), + config, + config_prefix); + + disk->startup(context, skip_access_check); + return disk; + }; + factory.registerDiskType("object_storage", creator); +#if USE_AWS_S3 + factory.registerDiskType("s3", creator); /// For compatibility + factory.registerDiskType("s3_plain", creator); /// For compatibility +#endif +#if USE_HDFS + factory.registerDiskType("hdfs", creator); /// For compatibility +#endif +#if USE_AZURE_BLOB_STORAGE + factory.registerDiskType("azure_blob_storage", creator); /// For compatibility +#endif + factory.registerDiskType("local_blob_storage", creator); /// For compatibility + factory.registerDiskType("web", creator); /// For compatibility +} + +} diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index beb8a400632..95150f46cc6 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index c8b3aeaca28..4e76dedcfc4 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -85,6 +85,10 @@ public: std::string getName() const override { return "S3ObjectStorage"; } + std::string getBasePath() const override { return object_key_prefix; } + + std::string getTypeName() const override { return "s3"; } + bool exists(const StoredObject & object) const override; std::unique_ptr readObject( /// NOLINT @@ -206,6 +210,8 @@ public: std::string getName() const override { return "S3PlainObjectStorage"; } + std::string getTypeName() const override { return "s3_plain"; } + template explicit S3PlainObjectStorage(Args && ...args) : S3ObjectStorage("S3PlainObjectStorage", std::forward(args)...) diff --git a/src/Disks/ObjectStorages/S3/checkBatchRemove.cpp b/src/Disks/ObjectStorages/S3/checkBatchRemove.cpp new file mode 100644 index 00000000000..b806b56065d --- /dev/null +++ b/src/Disks/ObjectStorages/S3/checkBatchRemove.cpp @@ -0,0 +1,64 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +static String getServerUUID() +{ + UUID server_uuid = ServerUUID::get(); + if (server_uuid == UUIDHelpers::Nil) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Server UUID is not initialized"); + return toString(server_uuid); +} + +bool checkBatchRemove(S3ObjectStorage & storage, const String & key_with_trailing_slash) +{ + /// NOTE: key_with_trailing_slash is the disk prefix, it is required + /// because access is done via S3ObjectStorage not via IDisk interface + /// (since we don't have disk yet). + const String path = fmt::format("{}clickhouse_remove_objects_capability_{}", key_with_trailing_slash, getServerUUID()); + StoredObject object(path); + try + { + auto file = storage.writeObject(object, WriteMode::Rewrite); + file->write("test", 4); + file->finalize(); + } + catch (...) + { + try + { + storage.removeObject(object); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + } + /// We don't have write access, therefore no information about batch remove. + return true; + } + try + { + /// Uses `DeleteObjects` request (batch delete). + storage.removeObjects({object}); + return true; + } + catch (const Exception &) + { + try + { + storage.removeObject(object); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + } + return false; + } +} + +} diff --git a/src/Disks/ObjectStorages/S3/checkBatchRemove.h b/src/Disks/ObjectStorages/S3/checkBatchRemove.h new file mode 100644 index 00000000000..c782fa52e41 --- /dev/null +++ b/src/Disks/ObjectStorages/S3/checkBatchRemove.h @@ -0,0 +1,11 @@ +#pragma once +#include + +namespace DB +{ + +class S3ObjectStorage; + +bool checkBatchRemove(S3ObjectStorage & storage, const std::string & key_with_trailing_slash); + +} diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 13d26877de4..062345494c6 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #include diff --git a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp deleted file mode 100644 index 7543fb94331..00000000000 --- a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp +++ /dev/null @@ -1,188 +0,0 @@ -#include "config.h" - -#include -#include -#include -#include -#include - -#if USE_AWS_S3 - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; - extern const int LOGICAL_ERROR; -} - -namespace -{ - -class CheckAccess -{ -public: - static bool checkBatchRemove(S3ObjectStorage & storage, const String & key_with_trailing_slash) - { - /// NOTE: key_with_trailing_slash is the disk prefix, it is required - /// because access is done via S3ObjectStorage not via IDisk interface - /// (since we don't have disk yet). - const String path = fmt::format("{}clickhouse_remove_objects_capability_{}", key_with_trailing_slash, getServerUUID()); - StoredObject object(path); - try - { - auto file = storage.writeObject(object, WriteMode::Rewrite); - file->write("test", 4); - file->finalize(); - } - catch (...) - { - try - { - storage.removeObject(object); - } - catch (...) // NOLINT(bugprone-empty-catch) - { - } - return true; /// We don't have write access, therefore no information about batch remove. - } - try - { - /// Uses `DeleteObjects` request (batch delete). - storage.removeObjects({object}); - return true; - } - catch (const Exception &) - { - try - { - storage.removeObject(object); - } - catch (...) // NOLINT(bugprone-empty-catch) - { - } - return false; - } - } - -private: - static String getServerUUID() - { - UUID server_uuid = ServerUUID::get(); - if (server_uuid == UUIDHelpers::Nil) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Server UUID is not initialized"); - return toString(server_uuid); - } -}; - -} - -void registerDiskS3(DiskFactory & factory, bool global_skip_access_check) -{ - auto creator = [global_skip_access_check]( - const String & name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context, - const DisksMap & /*map*/) -> DiskPtr - { - String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); - S3::URI uri(endpoint); - if (!uri.key.ends_with('/')) - uri.key.push_back('/'); - - S3Capabilities s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); - std::shared_ptr s3_storage; - - String type = config.getString(config_prefix + ".type"); - chassert(type == "s3" || type == "s3_plain"); - - MetadataStoragePtr metadata_storage; - auto settings = getSettings(config, config_prefix, context); - auto client = getClient(config, config_prefix, context, *settings); - - if (type == "s3_plain") - { - /// send_metadata changes the filenames (includes revision), while - /// s3_plain do not care about this, and expect that the file name - /// will not be changed. - /// - /// And besides, send_metadata does not make sense for s3_plain. - if (config.getBool(config_prefix + ".send_metadata", false)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "s3_plain does not supports send_metadata"); - - s3_storage = std::make_shared( - std::move(client), std::move(settings), - uri.version_id, s3_capabilities, - uri.bucket, uri.endpoint, uri.key, name); - metadata_storage = std::make_shared(s3_storage, uri.key); - } - else - { - s3_storage = std::make_shared( - std::move(client), std::move(settings), - uri.version_id, s3_capabilities, - uri.bucket, uri.endpoint, uri.key, name); - - auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context); - metadata_storage = std::make_shared(metadata_disk, uri.key); - } - - /// NOTE: should we still perform this check for clickhouse-disks? - bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false); - if (!skip_access_check) - { - /// If `support_batch_delete` is turned on (default), check and possibly switch it off. - if (s3_capabilities.support_batch_delete && !CheckAccess::checkBatchRemove(*s3_storage, uri.key)) - { - LOG_WARNING( - &Poco::Logger::get("registerDiskS3"), - "Storage for disk {} does not support batch delete operations, " - "so `s3_capabilities.support_batch_delete` was automatically turned off during the access check. " - "To remove this message set `s3_capabilities.support_batch_delete` for the disk to `false`.", - name - ); - s3_storage->setCapabilitiesSupportBatchDelete(false); - } - } - - DiskObjectStoragePtr s3disk = std::make_shared( - name, - uri.key, - type == "s3" ? "DiskS3" : "DiskS3Plain", - std::move(metadata_storage), - std::move(s3_storage), - config, - config_prefix); - - s3disk->startup(context, skip_access_check); - - return s3disk; - }; - factory.registerDiskType("s3", creator); - factory.registerDiskType("s3_plain", creator); -} - -} - -#else - -void registerDiskS3(DB::DiskFactory &, bool /* global_skip_access_check */) {} - -#endif diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.h b/src/Disks/ObjectStorages/Web/WebObjectStorage.h index 2bc3b6983fe..c4c6ef522dc 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.h +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.h @@ -33,6 +33,10 @@ public: std::string getName() const override { return "WebObjectStorage"; } + std::string getBasePath() const override { return ""; } + + std::string getTypeName() const override { return "web"; } + bool exists(const StoredObject & object) const override; std::unique_ptr readObject( /// NOLINT diff --git a/src/Disks/ObjectStorages/Web/registerDiskWebServer.cpp b/src/Disks/ObjectStorages/Web/registerDiskWebServer.cpp deleted file mode 100644 index 442a399fc78..00000000000 --- a/src/Disks/ObjectStorages/Web/registerDiskWebServer.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - -void registerDiskWebServer(DiskFactory & factory, bool global_skip_access_check) -{ - auto creator = [global_skip_access_check]( - const String & disk_name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context, - const DisksMap & /*map*/) -> DiskPtr - { - String uri = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); - bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false); - - if (!uri.ends_with('/')) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "URI must end with '/', but '{}' doesn't.", uri); - try - { - Poco::URI poco_uri(uri); - } - catch (const Poco::Exception & e) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Bad URI: `{}`. Error: {}", uri, e.what()); - } - - auto object_storage = std::make_shared(uri, context); - auto metadata_storage = std::make_shared(assert_cast(*object_storage)); - std::string root_path; - - DiskPtr disk = std::make_shared( - disk_name, - root_path, - "DiskWebServer", - metadata_storage, - object_storage, - config, - config_prefix); - - disk->startup(context, skip_access_check); - return disk; - }; - - factory.registerDiskType("web", creator); -} - -} diff --git a/src/Disks/registerDisks.cpp b/src/Disks/registerDisks.cpp index 4e07fda1cc2..6cbcd0ba5ff 100644 --- a/src/Disks/registerDisks.cpp +++ b/src/Disks/registerDisks.cpp @@ -9,27 +9,12 @@ namespace DB void registerDiskLocal(DiskFactory & factory, bool global_skip_access_check); -#if USE_AWS_S3 -void registerDiskS3(DiskFactory & factory, bool global_skip_access_check); -#endif - -#if USE_AZURE_BLOB_STORAGE -void registerDiskAzureBlobStorage(DiskFactory & factory, bool global_skip_access_check); -#endif - #if USE_SSL void registerDiskEncrypted(DiskFactory & factory, bool global_skip_access_check); #endif -#if USE_HDFS -void registerDiskHDFS(DiskFactory & factory, bool global_skip_access_check); -#endif - -void registerDiskWebServer(DiskFactory & factory, bool global_skip_access_check); - void registerDiskCache(DiskFactory & factory, bool global_skip_access_check); - -void registerDiskLocalObjectStorage(DiskFactory & factory, bool global_skip_access_check); +void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_check); #ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD @@ -40,27 +25,13 @@ void registerDisks(bool global_skip_access_check) registerDiskLocal(factory, global_skip_access_check); -#if USE_AWS_S3 - registerDiskS3(factory, global_skip_access_check); -#endif - -#if USE_AZURE_BLOB_STORAGE - registerDiskAzureBlobStorage(factory, global_skip_access_check); -#endif - #if USE_SSL registerDiskEncrypted(factory, global_skip_access_check); #endif -#if USE_HDFS - registerDiskHDFS(factory, global_skip_access_check); -#endif - - registerDiskWebServer(factory, global_skip_access_check); - registerDiskCache(factory, global_skip_access_check); - registerDiskLocalObjectStorage(factory, global_skip_access_check); + registerDiskObjectStorage(factory, global_skip_access_check); } #else From 383d40ea0389c42e30e3ca96dc16de0f7c63f9a6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 10 Jan 2024 13:14:10 +0100 Subject: [PATCH 02/14] Finish changes --- src/Disks/DiskFactory.cpp | 8 +- .../AzureBlobStorage/AzureObjectStorage.h | 4 +- .../Cached/CachedObjectStorage.h | 4 +- .../Cached/registerDiskCache.cpp | 5 +- .../ObjectStorages/HDFS/HDFSObjectStorage.h | 4 +- src/Disks/ObjectStorages/IObjectStorage.h | 4 +- .../Local/LocalObjectStorage.cpp | 2 +- .../ObjectStorages/Local/LocalObjectStorage.h | 4 +- .../ObjectStorages/MetadataStorageFactory.cpp | 45 ++++--- .../ObjectStorages/ObjectStorageFactory.cpp | 79 +++++++----- .../ObjectStorages/ObjectStorageFactory.h | 2 +- .../RegisterDiskObjectStorage.cpp | 49 +++++--- src/Disks/ObjectStorages/S3/DiskS3Utils.cpp | 117 ++++++++++++++++++ src/Disks/ObjectStorages/S3/DiskS3Utils.h | 20 +++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 60 ++++----- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 23 ++-- src/Disks/ObjectStorages/S3/S3Utils.h | 0 .../ObjectStorages/S3/checkBatchRemove.cpp | 64 ---------- .../ObjectStorages/S3/checkBatchRemove.h | 11 -- .../ObjectStorages/Web/WebObjectStorage.h | 4 +- 20 files changed, 306 insertions(+), 203 deletions(-) create mode 100644 src/Disks/ObjectStorages/S3/DiskS3Utils.cpp create mode 100644 src/Disks/ObjectStorages/S3/DiskS3Utils.h create mode 100644 src/Disks/ObjectStorages/S3/S3Utils.h delete mode 100644 src/Disks/ObjectStorages/S3/checkBatchRemove.cpp delete mode 100644 src/Disks/ObjectStorages/S3/checkBatchRemove.h diff --git a/src/Disks/DiskFactory.cpp b/src/Disks/DiskFactory.cpp index 1cf71773c12..6f8ba7ed186 100644 --- a/src/Disks/DiskFactory.cpp +++ b/src/Disks/DiskFactory.cpp @@ -6,6 +6,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int UNKNOWN_ELEMENT_IN_CONFIG; + extern const int NO_ELEMENTS_IN_CONFIG; } DiskFactory & DiskFactory::instance() @@ -14,7 +15,7 @@ DiskFactory & DiskFactory::instance() return factory; } -void DiskFactory::registerDiskType(const String & disk_type, DB::DiskFactory::Creator creator) +void DiskFactory::registerDiskType(const String & disk_type, Creator creator) { if (!registry.emplace(disk_type, creator).second) throw Exception(ErrorCodes::LOGICAL_ERROR, "DiskFactory: the disk type '{}' is not unique", disk_type); @@ -31,7 +32,10 @@ DiskPtr DiskFactory::create( const auto found = registry.find(disk_type); if (found == registry.end()) - throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "DiskFactory: the disk '{}' has unknown disk type: {}", name, disk_type); + { + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, + "DiskFactory: the disk '{}' has unknown disk type: {}", name, disk_type); + } const auto & disk_creator = found->second; return disk_creator(name, config, config_prefix, context, map); diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 936c3fce4a3..1d8e489f2fb 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -63,9 +63,7 @@ public: std::string getName() const override { return "AzureObjectStorage"; } - std::string getBasePath() const override { return ""; } /// No namespaces in azure. - - std::string getTypeName() const override { return "azure_blob_storage"; } + std::string getDataPrefix() const override { return ""; } /// No namespaces in azure. bool exists(const StoredObject & object) const override; diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h index 568fa64105e..f6a699d1625 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h @@ -24,12 +24,10 @@ public: std::string getName() const override { return fmt::format("CachedObjectStorage-{}({})", cache_config_name, object_storage->getName()); } - std::string getBasePath() const override { return object_storage->getBasePath(); } + std::string getDataPrefix() const override { return object_storage->getDataPrefix(); } bool exists(const StoredObject & object) const override; - std::string getTypeName() const override { return object_storage->getTypeName(); } - std::unique_ptr readObject( /// NOLINT const StoredObject & object, const ReadSettings & read_settings = ReadSettings{}, diff --git a/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp b/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp index da01a82746f..11117de67e0 100644 --- a/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp +++ b/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp @@ -39,7 +39,10 @@ void registerDiskCache(DiskFactory & factory, bool /* global_skip_access_check * } FileCacheSettings file_cache_settings; - auto predefined_configuration = config.has("cache_name") ? NamedCollectionFactory::instance().tryGet(config.getString("cache_name")) : nullptr; + auto predefined_configuration = config.has("cache_name") + ? NamedCollectionFactory::instance().tryGet(config.getString("cache_name")) + : nullptr; + if (predefined_configuration) file_cache_settings.loadFromCollection(*predefined_configuration); else diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index 6f54b2495f1..043aff78069 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -58,9 +58,7 @@ public: std::string getName() const override { return "HDFSObjectStorage"; } - std::string getBasePath() const override { return hdfs_root_path; } - - std::string getTypeName() const override { return "hdfs"; } + std::string getDataPrefix() const override { return hdfs_root_path; } DataSourceDescription getDataSourceDescription() const override { diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index fcdb5e6d164..f3e1d86fbad 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -84,9 +84,7 @@ public: virtual std::string getName() const = 0; - virtual std::string getBasePath() const = 0; - - virtual std::string getTypeName() const = 0; + virtual std::string getDataPrefix() const = 0; /// Object exists or not virtual bool exists(const StoredObject & object) const = 0; diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index a72e43952a8..c4451ebabf4 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -27,7 +27,7 @@ LocalObjectStorage::LocalObjectStorage(String key_prefix_) : key_prefix(std::move(key_prefix_)) , log(&Poco::Logger::get("LocalObjectStorage")) { - data_source_description.type = DataSourceType::Local; + data_source_description.type = DataSourceType::LocalBlobStorage; if (auto block_device_id = tryGetBlockDeviceId("/"); block_device_id.has_value()) data_source_description.description = *block_device_id; else diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h index 24090ba3a54..96075f85c38 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h @@ -22,9 +22,7 @@ public: std::string getName() const override { return "LocalObjectStorage"; } - std::string getBasePath() const override { return key_prefix; } - - std::string getTypeName() const override { return "local_blob_storage"; } + std::string getDataPrefix() const override { return key_prefix; } bool exists(const StoredObject & object) const override; diff --git a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp index 60da0b7867e..5217929545b 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp @@ -41,7 +41,7 @@ MetadataStoragePtr MetadataStorageFactory::create( throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Expected `metadata_type` in config"); } - const auto type = config.getString(config_prefix + ".metadata_type"); + const auto type = config.getString(config_prefix + ".metadata_type", compatibility_type_hint); const auto it = registry.find(type); if (it == registry.end()) @@ -53,9 +53,17 @@ MetadataStoragePtr MetadataStorageFactory::create( return it->second(name, config, config_prefix, object_storage); } +static std::string getObjectStoragePrefix( + const IObjectStorage & object_storage, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix) +{ + return config.getString(config_prefix + ".key_compatibility_prefix", object_storage.getDataPrefix()); +} + void registerMetadataStorageFromDisk(MetadataStorageFactory & factory) { - auto creator = []( + factory.registerMetadataStorageType("local", []( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -65,35 +73,42 @@ void registerMetadataStorageFromDisk(MetadataStorageFactory & factory) fs::path(Context::getGlobalContextInstance()->getPath()) / "disks" / name / ""); fs::create_directories(metadata_path); auto metadata_disk = std::make_shared(name + "-metadata", metadata_path, 0, config, config_prefix); - return std::make_shared(metadata_disk, object_storage->getBasePath()); - }; - factory.registerMetadataStorageType("local", creator); + auto key_compatibility_prefix = getObjectStoragePrefix(*object_storage, config, config_prefix); + return std::make_shared(metadata_disk, key_compatibility_prefix); + }); } -void registerMetadataStorageFromDiskPlain(MetadataStorageFactory & factory) +void registerPlainMetadataStorage(MetadataStorageFactory & factory) { - auto creator = []( + factory.registerMetadataStorageType("plain", []( const std::string & /* name */, - const Poco::Util::AbstractConfiguration & /* config */, - const std::string & /* config_prefix */, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, ObjectStoragePtr object_storage) -> MetadataStoragePtr { - return std::make_shared(object_storage, object_storage->getBasePath()); - }; - factory.registerMetadataStorageType("plain", creator); + auto key_compatibility_prefix = getObjectStoragePrefix(*object_storage, config, config_prefix); + return std::make_shared(object_storage, key_compatibility_prefix); + }); } void registerMetadataStorageFromStaticFilesWebServer(MetadataStorageFactory & factory) { - auto creator = []( + factory.registerMetadataStorageType("web", []( const std::string & /* name */, const Poco::Util::AbstractConfiguration & /* config */, const std::string & /* config_prefix */, ObjectStoragePtr object_storage) -> MetadataStoragePtr { return std::make_shared(assert_cast(*object_storage)); - }; - factory.registerMetadataStorageType("web", creator); + }); +} + +void registerMetadataStorages() +{ + auto & factory = MetadataStorageFactory::instance(); + registerMetadataStorageFromDisk(factory); + registerPlainMetadataStorage(factory); + registerMetadataStorageFromStaticFilesWebServer(factory); } } diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp index b9a2d85e15a..d5a20e3fdd5 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -28,13 +28,12 @@ ObjectStorageFactory & ObjectStorageFactory::instance() return factory; } -void ObjectStorageFactory::registerObjectStorageType(const std::string & metadata_type, Creator creator) +void ObjectStorageFactory::registerObjectStorageType(const std::string & type, Creator creator) { - if (!registry.emplace(metadata_type, creator).second) + if (!registry.emplace(type, creator).second) { throw Exception(ErrorCodes::LOGICAL_ERROR, - "ObjectStorageFactory: the metadata type '{}' is not unique", - metadata_type); + "ObjectStorageFactory: the metadata type '{}' is not unique", type); } } @@ -46,7 +45,7 @@ ObjectStoragePtr ObjectStorageFactory::create( bool skip_access_check) const { std::string type; - if (config.has(config_prefix + ".obejct_storage_type")) + if (config.has(config_prefix + ".object_storage_type")) { type = config.getString(config_prefix + ".object_storage_type"); } @@ -76,15 +75,20 @@ static S3::URI getS3URI(const Poco::Util::AbstractConfiguration & config, { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); - if (!uri.key.ends_with('/')) + + /// An empty key remains empty. + if (!uri.key.empty() && !uri.key.ends_with('/')) uri.key.push_back('/'); + return uri; } #if USE_AWS_S3 void registerS3ObjectStorage(ObjectStorageFactory & factory) { - auto creator = []( + static constexpr auto disk_type = "s3"; + + factory.registerObjectStorageType(disk_type, []( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -95,10 +99,10 @@ void registerS3ObjectStorage(ObjectStorageFactory & factory) auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); auto settings = getSettings(config, config_prefix, context); auto client = getClient(config, config_prefix, context, *settings); + auto key_generator = getKeyGenerator(disk_type, uri, config, config_prefix); auto object_storage = std::make_shared( - std::move(client), std::move(settings), uri.version_id, - s3_capabilities, uri.bucket, uri.endpoint, uri.key, name); + std::move(client), std::move(settings), uri, s3_capabilities, key_generator, name); /// NOTE: should we still perform this check for clickhouse-disks? if (!skip_access_check) @@ -117,13 +121,14 @@ void registerS3ObjectStorage(ObjectStorageFactory & factory) } } return object_storage; - }; - factory.registerObjectStorageType("s3", creator); + }); } void registerS3PlainObjectStorage(ObjectStorageFactory & factory) { - auto creator = []( + static constexpr auto disk_type = "s3_plain"; + + factory.registerObjectStorageType(disk_type, []( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -142,19 +147,18 @@ void registerS3PlainObjectStorage(ObjectStorageFactory & factory) auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); auto settings = getSettings(config, config_prefix, context); auto client = getClient(config, config_prefix, context, *settings); + auto key_generator = getKeyGenerator(disk_type, uri, config, config_prefix); return std::make_shared( - std::move(client), std::move(settings), uri.version_id, - s3_capabilities, uri.bucket, uri.endpoint, uri.key, name); - }; - factory.registerObjectStorageType("s3_plain", creator); + std::move(client), std::move(settings), uri, s3_capabilities, key_generator, name); + }); } #endif #if USE_HDFS void registerHDFSObjectStorage(ObjectStorageFactory & factory) { - auto creator = []( + factory.registerObjectStorageType("hdfs", []( const std::string & /* name */, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -173,15 +177,14 @@ void registerHDFSObjectStorage(ObjectStorageFactory & factory) ); return std::make_unique(uri, std::move(settings), config); - }; - factory.registerObjectStorageType("hdfs", creator); + }); } #endif #if USE_AZURE_BLOB_STORAGE void registerAzureObjectStorage(ObjectStorageFactory & factory) { - auto creator = []( + factory.registerObjectStorageType("azure_blob_storage", []( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -193,14 +196,13 @@ void registerAzureObjectStorage(ObjectStorageFactory & factory) getAzureBlobContainerClient(config, config_prefix), getAzureBlobStorageSettings(config, config_prefix, context)); - }; - factory.registerObjectStorageType("azure_blob_storage", creator); + }); } #endif void registerWebObjectStorage(ObjectStorageFactory & factory) { - auto creator = []( + factory.registerObjectStorageType("web", []( const std::string & /* name */, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -222,13 +224,12 @@ void registerWebObjectStorage(ObjectStorageFactory & factory) } return std::make_shared(uri, context); - }; - factory.registerObjectStorageType("web", creator); + }); } void registerLocalObjectStorage(ObjectStorageFactory & factory) { - auto creator = []( + factory.registerObjectStorageType("local_blob_storage", []( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, @@ -241,8 +242,28 @@ void registerLocalObjectStorage(ObjectStorageFactory & factory) /// keys are mapped to the fs, object_key_prefix is a directory also fs::create_directories(object_key_prefix); return std::make_shared(object_key_prefix); - }; - factory.registerObjectStorageType("local", creator); + }); +} + +void registerObjectStorages() +{ + auto & factory = ObjectStorageFactory::instance(); + +#if USE_AWS_S3 + registerS3ObjectStorage(factory); + registerS3PlainObjectStorage(factory); +#endif + +#if USE_HDFS + registerHDFSObjectStorage(factory); +#endif + +#if USE_AZURE_BLOB_STORAGE + registerAzureObjectStorage(factory); +#endif + + registerLocalObjectStorage(factory); + registerWebObjectStorage(factory); } } diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.h b/src/Disks/ObjectStorages/ObjectStorageFactory.h index 3c13b3bca86..179f1a46262 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.h +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.h @@ -17,7 +17,7 @@ public: static ObjectStorageFactory & instance(); - void registerObjectStorageType(const std::string & metadata_type, Creator creator); + void registerObjectStorageType(const std::string & type, Creator creator); ObjectStoragePtr create( const std::string & name, diff --git a/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp b/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp index a5f5a6238f4..13271cda9db 100644 --- a/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp @@ -11,23 +11,35 @@ namespace ErrorCodes extern const int UNKNOWN_ELEMENT_IN_CONFIG; } -static std::string getCompatibilityMetadataTypeHint(const std::string & object_storage_type) +void registerObjectStorages(); +void registerMetadataStorages(); + +static std::string getCompatibilityMetadataTypeHint(const DataSourceDescription & description) { - /// TODO: change type name to enum - if (object_storage_type == "s3" - || object_storage_type == "hdfs" - || object_storage_type == "azure_blob_storage" - || object_storage_type == "local_blob_storage") - return "local"; - else if (object_storage_type == "s3_plain") - return "plain"; - else if (object_storage_type == "web") - return "web"; - throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown object storage type: {}", object_storage_type); + switch (description.type) + { + case DataSourceType::S3: + case DataSourceType::HDFS: + case DataSourceType::LocalBlobStorage: + case DataSourceType::AzureBlobStorage: + return "local"; + case DataSourceType::S3_Plain: + return "plain"; + case DataSourceType::WebServer: + return "web"; + default: + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, + "Cannot get compatibility metadata hint: " + "no such object storage type: {}", toString(description.type)); + } + UNREACHABLE(); } void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_check) { + registerObjectStorages(); + registerMetadataStorages(); + auto creator = [global_skip_access_check]( const String & name, const Poco::Util::AbstractConfiguration & config, @@ -37,13 +49,17 @@ void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_ch { bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false); auto object_storage = ObjectStorageFactory::instance().create(name, config, config_prefix, context, skip_access_check); - auto compatibility_metadata_type_hint = config.has("metadata_type") ? "" : getCompatibilityMetadataTypeHint(object_storage->getTypeName()); - auto metadata_storage = MetadataStorageFactory::instance().create(name, config, config_prefix, object_storage, compatibility_metadata_type_hint); + auto compatibility_metadata_type_hint = config.has("metadata_type") + ? "" + : getCompatibilityMetadataTypeHint(object_storage->getDataSourceDescription()); + + auto metadata_storage = MetadataStorageFactory::instance().create( + name, config, config_prefix, object_storage, compatibility_metadata_type_hint); DiskObjectStoragePtr disk = std::make_shared( name, - object_storage->getBasePath(), - "Disk" + object_storage->getTypeName(), + object_storage->getDataPrefix(), + fmt::format("Disk_{}({})", toString(object_storage->getDataSourceDescription().type), name), std::move(metadata_storage), std::move(object_storage), config, @@ -52,6 +68,7 @@ void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_ch disk->startup(context, skip_access_check); return disk; }; + factory.registerDiskType("object_storage", creator); #if USE_AWS_S3 factory.registerDiskType("s3", creator); /// For compatibility diff --git a/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp b/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp new file mode 100644 index 00000000000..06e02a31e89 --- /dev/null +++ b/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp @@ -0,0 +1,117 @@ +#include "DiskS3Utils.h" +#include +#include +#include + +namespace DB +{ + +ObjectStorageKeysGeneratorPtr getKeyGenerator( + String type, + const S3::URI & uri, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix) +{ + if (type == "s3_plain") + return createObjectStorageKeysGeneratorAsIsWithPrefix(uri.key); + + chassert(type == "s3"); + + bool storage_metadata_write_full_object_key = DiskObjectStorageMetadata::getWriteFullObjectKeySetting(); + bool send_metadata = config.getBool(config_prefix + ".send_metadata", false); + + if (send_metadata && storage_metadata_write_full_object_key) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Wrong configuration in {}. " + "s3 does not supports feature 'send_metadata' with feature 'storage_metadata_write_full_object_key'.", + config_prefix); + + String object_key_compatibility_prefix = config.getString(config_prefix + ".key_compatibility_prefix", String()); + String object_key_template = config.getString(config_prefix + ".key_template", String()); + + if (object_key_template.empty()) + { + if (!object_key_compatibility_prefix.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Wrong configuration in {}. " + "Setting 'key_compatibility_prefix' can be defined only with setting 'key_template'.", + config_prefix); + + return createObjectStorageKeysGeneratorByPrefix(uri.key); + } + + if (send_metadata) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Wrong configuration in {}. " + "s3 does not supports send_metadata with setting 'key_template'.", + config_prefix); + + if (!storage_metadata_write_full_object_key) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Wrong configuration in {}. " + "Feature 'storage_metadata_write_full_object_key' has to be enabled in order to use setting 'key_template'.", + config_prefix); + + if (!uri.key.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Wrong configuration in {}. " + "URI.key is forbidden with settings 'key_template', use setting 'key_compatibility_prefix' instead'. " + "URI.key: '{}', bucket: '{}'. ", + config_prefix, + uri.key, uri.bucket); + + return createObjectStorageKeysGeneratorByTemplate(object_key_template); +} + +static String getServerUUID() +{ + UUID server_uuid = ServerUUID::get(); + if (server_uuid == UUIDHelpers::Nil) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Server UUID is not initialized"); + return toString(server_uuid); +} + +bool checkBatchRemove(S3ObjectStorage & storage, const String & key_with_trailing_slash) +{ + /// NOTE: key_with_trailing_slash is the disk prefix, it is required + /// because access is done via S3ObjectStorage not via IDisk interface + /// (since we don't have disk yet). + const String path = fmt::format("{}clickhouse_remove_objects_capability_{}", key_with_trailing_slash, getServerUUID()); + StoredObject object(path); + try + { + auto file = storage.writeObject(object, WriteMode::Rewrite); + file->write("test", 4); + file->finalize(); + } + catch (...) + { + try + { + storage.removeObject(object); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + } + /// We don't have write access, therefore no information about batch remove. + return true; + } + try + { + /// Uses `DeleteObjects` request (batch delete). + storage.removeObjects({object}); + return true; + } + catch (const Exception &) + { + try + { + storage.removeObject(object); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + } + return false; + } +} +} diff --git a/src/Disks/ObjectStorages/S3/DiskS3Utils.h b/src/Disks/ObjectStorages/S3/DiskS3Utils.h new file mode 100644 index 00000000000..433da9d141b --- /dev/null +++ b/src/Disks/ObjectStorages/S3/DiskS3Utils.h @@ -0,0 +1,20 @@ +#pragma once +#include +#include +#include + +namespace Poco::Util { class AbstractConfiguration; } + +namespace DB +{ + +ObjectStorageKeysGeneratorPtr getKeyGenerator( + String type, + const S3::URI & uri, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix); + +class S3ObjectStorage; +bool checkBatchRemove(S3ObjectStorage & storage, const std::string & key_with_trailing_slash); + +} diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index a85e628d06f..49943554384 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -99,7 +99,7 @@ class S3IteratorAsync final : public IObjectStorageIteratorAsync { public: S3IteratorAsync( - const std::string & bucket, + const std::string & bucket_, const std::string & path_prefix, std::shared_ptr client_, size_t max_list_size) @@ -110,7 +110,7 @@ public: "ListObjectS3") , client(client_) { - request.SetBucket(bucket); + request.SetBucket(bucket_); request.SetPrefix(path_prefix); request.SetMaxKeys(static_cast(max_list_size)); } @@ -155,7 +155,7 @@ private: bool S3ObjectStorage::exists(const StoredObject & object) const { auto settings_ptr = s3_settings.get(); - return S3::objectExists(*client.get(), bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + return S3::objectExists(*client.get(), uri.bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); } std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT @@ -175,9 +175,9 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT { return std::make_unique( client.get(), - bucket, + uri.bucket, path, - version_id, + uri.version_id, settings_ptr->request_settings, disk_read_settings, /* use_external_buffer */true, @@ -225,9 +225,9 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT auto settings_ptr = s3_settings.get(); return std::make_unique( client.get(), - bucket, + uri.bucket, object.remote_path, - version_id, + uri.version_id, settings_ptr->request_settings, patchSettings(read_settings)); } @@ -256,7 +256,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN return std::make_unique( client.get(), - bucket, + uri.bucket, object.remote_path, buf_size, settings_ptr->request_settings, @@ -270,7 +270,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix) const { auto settings_ptr = s3_settings.get(); - return std::make_shared(bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size); + return std::make_shared(uri.bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size); } void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const @@ -278,7 +278,7 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet auto settings_ptr = s3_settings.get(); S3::ListObjectsV2Request request; - request.SetBucket(bucket); + request.SetBucket(uri.bucket); request.SetPrefix(path); if (max_keys) request.SetMaxKeys(max_keys); @@ -324,12 +324,12 @@ void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exis ProfileEvents::increment(ProfileEvents::S3DeleteObjects); ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); S3::DeleteObjectRequest request; - request.SetBucket(bucket); + request.SetBucket(uri.bucket); request.SetKey(object.remote_path); auto outcome = client.get()->DeleteObject(request); if (auto blob_storage_log = BlobStorageLogWriter::create(disk_name)) blob_storage_log->addEvent(BlobStorageLogElement::EventType::Delete, - bucket, object.remote_path, object.local_path, object.bytes_size, + uri.bucket, object.remote_path, object.local_path, object.bytes_size, outcome.IsSuccess() ? nullptr : &outcome.GetError()); throwIfUnexpectedError(outcome, if_exists); @@ -376,7 +376,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e ProfileEvents::increment(ProfileEvents::S3DeleteObjects); ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); S3::DeleteObjectsRequest request; - request.SetBucket(bucket); + request.SetBucket(uri.bucket); request.SetDelete(delkeys); auto outcome = client.get()->DeleteObjects(request); @@ -386,7 +386,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e auto time_now = std::chrono::system_clock::now(); for (const auto & object : objects) blob_storage_log->addEvent(BlobStorageLogElement::EventType::Delete, - bucket, object.remote_path, object.local_path, object.bytes_size, + uri.bucket, object.remote_path, object.local_path, object.bytes_size, outcome_error, time_now); } @@ -419,7 +419,7 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) std::optional S3ObjectStorage::tryGetObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); + auto object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); if (object_info.size == 0 && object_info.last_modification_time == 0 && object_info.metadata.empty()) return {}; @@ -435,7 +435,7 @@ std::optional S3ObjectStorage::tryGetObjectMetadata(const std::s ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); + auto object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); ObjectMetadata result; result.size_bytes = object_info.size; @@ -456,18 +456,18 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) { - auto client_ = dest_s3->client.get(); + auto current_client = dest_s3->client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*current_client, uri.bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); try { copyS3File( - client_, - bucket, + current_client, + uri.bucket, object_from.remote_path, 0, size, - dest_s3->bucket, + dest_s3->uri.bucket, object_to.remote_path, settings_ptr->request_settings, patchSettings(read_settings), @@ -498,16 +498,16 @@ void S3ObjectStorage::copyObject( // NOLINT const WriteSettings &, std::optional object_to_attributes) { - auto client_ = client.get(); + auto current_client = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*current_client, uri.bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(client_, - bucket, + copyS3File(current_client, + uri.bucket, object_from.remote_path, 0, size, - bucket, + uri.bucket, object_to.remote_path, settings_ptr->request_settings, patchSettings(read_settings), @@ -551,10 +551,12 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( auto new_s3_settings = getSettings(config, config_prefix, context); auto new_client = getClient(config, config_prefix, context, *new_s3_settings); String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); + + auto new_uri{uri}; + new_uri.bucket = new_namespace; + return std::make_unique( - std::move(new_client), std::move(new_s3_settings), - version_id, s3_capabilities, new_namespace, - endpoint, key_generator, disk_name); + std::move(new_client), std::move(new_s3_settings), new_uri, s3_capabilities, key_generator, disk_name); } ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string & path) const diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index 9eba9bfbc1c..f3791d1ff4c 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -49,22 +49,19 @@ private: const char * logger_name, std::unique_ptr && client_, std::unique_ptr && s3_settings_, - String version_id_, + S3::URI uri_, const S3Capabilities & s3_capabilities_, - String bucket_, - String connection_string, ObjectStorageKeysGeneratorPtr key_generator_, const String & disk_name_) - : bucket(std::move(bucket_)) + : uri(uri_) , key_generator(std::move(key_generator_)) , disk_name(disk_name_) , client(std::move(client_)) , s3_settings(std::move(s3_settings_)) , s3_capabilities(s3_capabilities_) - , version_id(std::move(version_id_)) { data_source_description.type = DataSourceType::S3; - data_source_description.description = connection_string; + data_source_description.description = uri_.endpoint; data_source_description.is_cached = false; data_source_description.is_encrypted = false; @@ -85,9 +82,7 @@ public: std::string getName() const override { return "S3ObjectStorage"; } - std::string getBasePath() const override { return object_key_prefix; } - - std::string getTypeName() const override { return "s3"; } + std::string getDataPrefix() const override { return uri.key; } bool exists(const StoredObject & object) const override; @@ -157,7 +152,7 @@ public: const std::string & config_prefix, ContextPtr context) override; - std::string getObjectsNamespace() const override { return bucket; } + std::string getObjectsNamespace() const override { return uri.bucket; } bool isRemote() const override { return true; } @@ -181,8 +176,8 @@ private: void removeObjectImpl(const StoredObject & object, bool if_exists); void removeObjectsImpl(const StoredObjects & objects, bool if_exists); -private: - std::string bucket; + const S3::URI uri; + ObjectStorageKeysGeneratorPtr key_generator; std::string disk_name; @@ -190,8 +185,6 @@ private: MultiVersion s3_settings; S3Capabilities s3_capabilities; - const String version_id; - Poco::Logger * log; DataSourceDescription data_source_description; }; @@ -205,8 +198,6 @@ class S3PlainObjectStorage : public S3ObjectStorage public: std::string getName() const override { return "S3PlainObjectStorage"; } - std::string getTypeName() const override { return "s3_plain"; } - template explicit S3PlainObjectStorage(Args && ...args) : S3ObjectStorage("S3PlainObjectStorage", std::forward(args)...) diff --git a/src/Disks/ObjectStorages/S3/S3Utils.h b/src/Disks/ObjectStorages/S3/S3Utils.h new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/Disks/ObjectStorages/S3/checkBatchRemove.cpp b/src/Disks/ObjectStorages/S3/checkBatchRemove.cpp deleted file mode 100644 index b806b56065d..00000000000 --- a/src/Disks/ObjectStorages/S3/checkBatchRemove.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -static String getServerUUID() -{ - UUID server_uuid = ServerUUID::get(); - if (server_uuid == UUIDHelpers::Nil) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Server UUID is not initialized"); - return toString(server_uuid); -} - -bool checkBatchRemove(S3ObjectStorage & storage, const String & key_with_trailing_slash) -{ - /// NOTE: key_with_trailing_slash is the disk prefix, it is required - /// because access is done via S3ObjectStorage not via IDisk interface - /// (since we don't have disk yet). - const String path = fmt::format("{}clickhouse_remove_objects_capability_{}", key_with_trailing_slash, getServerUUID()); - StoredObject object(path); - try - { - auto file = storage.writeObject(object, WriteMode::Rewrite); - file->write("test", 4); - file->finalize(); - } - catch (...) - { - try - { - storage.removeObject(object); - } - catch (...) // NOLINT(bugprone-empty-catch) - { - } - /// We don't have write access, therefore no information about batch remove. - return true; - } - try - { - /// Uses `DeleteObjects` request (batch delete). - storage.removeObjects({object}); - return true; - } - catch (const Exception &) - { - try - { - storage.removeObject(object); - } - catch (...) // NOLINT(bugprone-empty-catch) - { - } - return false; - } -} - -} diff --git a/src/Disks/ObjectStorages/S3/checkBatchRemove.h b/src/Disks/ObjectStorages/S3/checkBatchRemove.h deleted file mode 100644 index c782fa52e41..00000000000 --- a/src/Disks/ObjectStorages/S3/checkBatchRemove.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once -#include - -namespace DB -{ - -class S3ObjectStorage; - -bool checkBatchRemove(S3ObjectStorage & storage, const std::string & key_with_trailing_slash); - -} diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.h b/src/Disks/ObjectStorages/Web/WebObjectStorage.h index c4c6ef522dc..79e79ad7dbe 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.h +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.h @@ -33,9 +33,7 @@ public: std::string getName() const override { return "WebObjectStorage"; } - std::string getBasePath() const override { return ""; } - - std::string getTypeName() const override { return "web"; } + std::string getDataPrefix() const override { return ""; } bool exists(const StoredObject & object) const override; From 967d9c899df04453aa32380bcc806d6e52dc66c1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 10 Jan 2024 15:57:55 +0100 Subject: [PATCH 03/14] Add a test --- ...test_flexible_disk_configuration.reference | 0 ...02963_test_flexible_disk_configuration.sql | 58 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 tests/queries/0_stateless/02963_test_flexible_disk_configuration.reference create mode 100644 tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql diff --git a/tests/queries/0_stateless/02963_test_flexible_disk_configuration.reference b/tests/queries/0_stateless/02963_test_flexible_disk_configuration.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql b/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql new file mode 100644 index 00000000000..9a08c341836 --- /dev/null +++ b/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql @@ -0,0 +1,58 @@ +-- Tags: no-fasttest + +drop table if exists test; +create table test (a Int32) engine = MergeTree() order by tuple() +settings disk=disk(name='test1', type = object_storage, object_storage_type = local_blob_storage, path='./02963_test1/'); + +drop table test; +create table test (a Int32) engine = MergeTree() order by tuple() +settings disk='s3_disk_02963'; + +drop table test; +create table test (a Int32) engine = MergeTree() order by tuple() +settings disk=disk(name='test1', + type = object_storage, + object_storage_type = s3, + endpoint = 'http://localhost:11111/test/common/', + access_key_id = clickhouse, + secret_access_key = clickhouse); + +drop table test; +create table test (a Int32) engine = MergeTree() order by tuple() +settings disk=disk(name='test2', + type = object_storage, + object_storage_type = s3, + metadata_storage_type = local, + endpoint = 'http://localhost:11111/test/common/', + access_key_id = clickhouse, + secret_access_key = clickhouse); + +drop table test; +create table test (a Int32) engine = MergeTree() order by tuple() +settings disk=disk(name='test3', + type = object_storage, + object_storage_type = s3, + metadata_type = lll, + endpoint = 'http://localhost:11111/test/common/', + access_key_id = clickhouse, + secret_access_key = clickhouse); -- { serverError UNKNOWN_ELEMENT_IN_CONFIG } + +drop table test; +create table test (a Int32) engine = MergeTree() order by tuple() +settings disk=disk(name='test4', + type = object_storage, + object_storage_type = kkk, + metadata_type = local, + endpoint = 'http://localhost:11111/test/common/', + access_key_id = clickhouse, + secret_access_key = clickhouse); -- { serverError UNKNOWN_ELEMENT_IN_CONFIG } + +drop table test; +create table test (a Int32) engine = MergeTree() order by tuple() +settings disk=disk(name='test5', + type = kkk, + object_storage_type = s3, + metadata_type = local, + endpoint = 'http://localhost:11111/test/common/', + access_key_id = clickhouse, + secret_access_key = clickhouse); -- { serverError UNKNOWN_ELEMENT_IN_CONFIG } From 23659bec8f63833fb19110a954770aaca779c080 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 10 Jan 2024 16:44:21 +0100 Subject: [PATCH 04/14] Fix style check --- src/Disks/DiskFactory.cpp | 1 - src/Disks/ObjectStorages/MetadataStorageFactory.cpp | 1 + src/Disks/ObjectStorages/ObjectStorageFactory.cpp | 2 ++ src/Disks/ObjectStorages/S3/DiskS3Utils.cpp | 5 +++++ src/Disks/ObjectStorages/S3/S3Utils.h | 0 5 files changed, 8 insertions(+), 1 deletion(-) delete mode 100644 src/Disks/ObjectStorages/S3/S3Utils.h diff --git a/src/Disks/DiskFactory.cpp b/src/Disks/DiskFactory.cpp index 6f8ba7ed186..2261b24272c 100644 --- a/src/Disks/DiskFactory.cpp +++ b/src/Disks/DiskFactory.cpp @@ -6,7 +6,6 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int UNKNOWN_ELEMENT_IN_CONFIG; - extern const int NO_ELEMENTS_IN_CONFIG; } DiskFactory & DiskFactory::instance() diff --git a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp index 5217929545b..03f30cc631d 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp @@ -11,6 +11,7 @@ namespace ErrorCodes { extern const int NO_ELEMENTS_IN_CONFIG; extern const int UNKNOWN_ELEMENT_IN_CONFIG; + extern const int LOGICAL_ERROR; } MetadataStorageFactory & MetadataStorageFactory::instance() diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp index d5a20e3fdd5..280a30a5216 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -20,6 +20,8 @@ namespace ErrorCodes { extern const int NO_ELEMENTS_IN_CONFIG; extern const int UNKNOWN_ELEMENT_IN_CONFIG; + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; } ObjectStorageFactory & ObjectStorageFactory::instance() diff --git a/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp b/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp index 06e02a31e89..e0d56d1b4b5 100644 --- a/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp +++ b/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp @@ -5,6 +5,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} ObjectStorageKeysGeneratorPtr getKeyGenerator( String type, diff --git a/src/Disks/ObjectStorages/S3/S3Utils.h b/src/Disks/ObjectStorages/S3/S3Utils.h deleted file mode 100644 index e69de29bb2d..00000000000 From 0527d18f3f5554d4ffae5ad772f90f6bdc656784 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 10 Jan 2024 18:28:11 +0100 Subject: [PATCH 05/14] Fix build without s3 --- src/Disks/ObjectStorages/S3/DiskS3Utils.cpp | 5 +++++ src/Disks/ObjectStorages/S3/DiskS3Utils.h | 7 ++++++- tests/config/config.d/storage_conf_02963.xml | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 tests/config/config.d/storage_conf_02963.xml diff --git a/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp b/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp index e0d56d1b4b5..bb7b53b2d22 100644 --- a/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp +++ b/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp @@ -1,7 +1,10 @@ #include "DiskS3Utils.h" + +#if USE_AWS_S3 #include #include #include +#include namespace DB { @@ -120,3 +123,5 @@ bool checkBatchRemove(S3ObjectStorage & storage, const String & key_with_trailin } } } + +#endif diff --git a/src/Disks/ObjectStorages/S3/DiskS3Utils.h b/src/Disks/ObjectStorages/S3/DiskS3Utils.h index 433da9d141b..29e39d4bc1b 100644 --- a/src/Disks/ObjectStorages/S3/DiskS3Utils.h +++ b/src/Disks/ObjectStorages/S3/DiskS3Utils.h @@ -1,12 +1,15 @@ #pragma once +#include "config.h" #include #include -#include + +#if USE_AWS_S3 namespace Poco::Util { class AbstractConfiguration; } namespace DB { +namespace S3 { struct URI; } ObjectStorageKeysGeneratorPtr getKeyGenerator( String type, @@ -18,3 +21,5 @@ class S3ObjectStorage; bool checkBatchRemove(S3ObjectStorage & storage, const std::string & key_with_trailing_slash); } + +#endif diff --git a/tests/config/config.d/storage_conf_02963.xml b/tests/config/config.d/storage_conf_02963.xml new file mode 100644 index 00000000000..0672965e99d --- /dev/null +++ b/tests/config/config.d/storage_conf_02963.xml @@ -0,0 +1,15 @@ + + + + + object_storage + s3 + s3_disk/ + http://localhost:11111/test/common/ + clickhouse + clickhouse + 20000 + + + + From df1d4458b481cd6afe10f2013279087d50d60b97 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 11 Jan 2024 12:04:54 +0100 Subject: [PATCH 06/14] Fix build without s3 --- src/Disks/ObjectStorages/ObjectStorageFactory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp index 280a30a5216..db230588963 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -71,6 +71,7 @@ ObjectStoragePtr ObjectStorageFactory::create( return it->second(name, config, config_prefix, context, skip_access_check); } +#if USE_AWS_S3 static S3::URI getS3URI(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const ContextPtr & context) @@ -85,7 +86,6 @@ static S3::URI getS3URI(const Poco::Util::AbstractConfiguration & config, return uri; } -#if USE_AWS_S3 void registerS3ObjectStorage(ObjectStorageFactory & factory) { static constexpr auto disk_type = "s3"; From c211b3bb847662e22f42fcdb389e6485eb0da564 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 11 Jan 2024 15:53:35 +0100 Subject: [PATCH 07/14] Update configs --- docker/test/upgrade/run.sh | 2 ++ tests/config/install.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index f014fce49f6..861fd51a3fa 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -78,6 +78,7 @@ remove_keeper_config "create_if_not_exists" "[01]" rm /etc/clickhouse-server/config.d/merge_tree.xml rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml rm /etc/clickhouse-server/config.d/zero_copy_destructive_operations.xml +rm /etc/clickhouse-server/config.d/storage_conf_02963.xml rm /etc/clickhouse-server/users.d/nonconst_timezone.xml rm /etc/clickhouse-server/users.d/s3_cache_new.xml rm /etc/clickhouse-server/users.d/replicated_ddl_entry.xml @@ -117,6 +118,7 @@ sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_defau rm /etc/clickhouse-server/config.d/merge_tree.xml rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml rm /etc/clickhouse-server/config.d/zero_copy_destructive_operations.xml +rm /etc/clickhouse-server/config.d/storage_conf_02963.xml rm /etc/clickhouse-server/users.d/nonconst_timezone.xml rm /etc/clickhouse-server/users.d/s3_cache_new.xml rm /etc/clickhouse-server/users.d/replicated_ddl_entry.xml diff --git a/tests/config/install.sh b/tests/config/install.sh index 3a0744a298d..2314c5d8b67 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -175,6 +175,7 @@ if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then ln -sf $SRC_PATH/config.d/storage_conf.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf_02944.xml $DEST_SERVER_PATH/config.d/ + ln -sf $SRC_PATH/config.d/storage_conf_02963.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/users.d/s3_cache.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/s3_cache_new.xml $DEST_SERVER_PATH/users.d/ fi From a9ee80108f7e33851cea175840f104146f8130ce Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 11 Jan 2024 17:58:25 +0100 Subject: [PATCH 08/14] Update 02963_test_flexible_disk_configuration.sql --- .../0_stateless/02963_test_flexible_disk_configuration.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql b/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql index 9a08c341836..552291b2f83 100644 --- a/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql +++ b/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql @@ -37,7 +37,6 @@ settings disk=disk(name='test3', access_key_id = clickhouse, secret_access_key = clickhouse); -- { serverError UNKNOWN_ELEMENT_IN_CONFIG } -drop table test; create table test (a Int32) engine = MergeTree() order by tuple() settings disk=disk(name='test4', type = object_storage, @@ -47,7 +46,6 @@ settings disk=disk(name='test4', access_key_id = clickhouse, secret_access_key = clickhouse); -- { serverError UNKNOWN_ELEMENT_IN_CONFIG } -drop table test; create table test (a Int32) engine = MergeTree() order by tuple() settings disk=disk(name='test5', type = kkk, From f4546a5eeebd538a95d6126de5558150a1f70004 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 12 Jan 2024 10:53:40 +0100 Subject: [PATCH 09/14] Fix build --- programs/keeper/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index b8f538f821c..6c4c3e82162 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -126,7 +126,6 @@ if (BUILD_STANDALONE_KEEPER) ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/DiskObjectStorage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/ObjectStorageIterator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/StoredObject.cpp From 7b674c31c52a278d115c856270165d7c0e97ca56 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 12 Jan 2024 10:55:27 +0100 Subject: [PATCH 10/14] Fix for issue 58697 --- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 6075b385a6c..0e739425a26 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -73,8 +73,14 @@ AzureBlobStorageEndpoint processAzureBlobStorageEndpoint(const Poco::Util::Abstr } else { - storage_url = config.getString(config_prefix + ".connection_string"); + if (config.has(config_prefix + ".connection_string")) + storage_url = config.getString(config_prefix + ".connection_string"); + else if (config.has(config_prefix + ".endpoint")) + storage_url = config.getString(config_prefix + ".endpoint"); + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected either `connection_string` or `endpoint` in config"); } + String container_name = config.getString(config_prefix + ".container_name", "default-container"); validateContainerName(container_name); std::optional container_already_exists {}; @@ -108,11 +114,14 @@ template std::unique_ptr getAzureBlobStorageClientWithAuth( const String & url, const String & container_name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix) { + std::string connection_str; if (config.has(config_prefix + ".connection_string")) - { - String connection_str = config.getString(config_prefix + ".connection_string"); + connection_str = config.getString(config_prefix + ".connection_string"); + else if (config.has(config_prefix + ".endpoint")) + connection_str = config.getString(config_prefix + ".endpoint"); + + if (!connection_str.empty()) return getClientWithConnectionString(connection_str, container_name); - } if (config.has(config_prefix + ".account_key") && config.has(config_prefix + ".account_name")) { From 7c92d69820337ad8d9a0ba11c44b76510f399fe1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 12 Jan 2024 17:27:45 +0100 Subject: [PATCH 11/14] Rename a method --- src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h | 2 +- src/Disks/ObjectStorages/Cached/CachedObjectStorage.h | 2 +- src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h | 2 +- src/Disks/ObjectStorages/IObjectStorage.h | 2 +- src/Disks/ObjectStorages/Local/LocalObjectStorage.h | 2 +- src/Disks/ObjectStorages/MetadataStorageFactory.cpp | 2 +- src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp | 2 +- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 2 +- src/Disks/ObjectStorages/Web/WebObjectStorage.h | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 1d8e489f2fb..4718dce9bf9 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -63,7 +63,7 @@ public: std::string getName() const override { return "AzureObjectStorage"; } - std::string getDataPrefix() const override { return ""; } /// No namespaces in azure. + std::string getCommonKeyPrefix() const override { return ""; } /// No namespaces in azure. bool exists(const StoredObject & object) const override; diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h index f6a699d1625..4c185db051d 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h @@ -24,7 +24,7 @@ public: std::string getName() const override { return fmt::format("CachedObjectStorage-{}({})", cache_config_name, object_storage->getName()); } - std::string getDataPrefix() const override { return object_storage->getDataPrefix(); } + std::string getCommonKeyPrefix() const override { return object_storage->getCommonKeyPrefix(); } bool exists(const StoredObject & object) const override; diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index 043aff78069..82458974278 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -58,7 +58,7 @@ public: std::string getName() const override { return "HDFSObjectStorage"; } - std::string getDataPrefix() const override { return hdfs_root_path; } + std::string getCommonKeyPrefix() const override { return hdfs_root_path; } DataSourceDescription getDataSourceDescription() const override { diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index f3e1d86fbad..f405be72287 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -84,7 +84,7 @@ public: virtual std::string getName() const = 0; - virtual std::string getDataPrefix() const = 0; + virtual std::string getCommonKeyPrefix() const = 0; /// Object exists or not virtual bool exists(const StoredObject & object) const = 0; diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h index 96075f85c38..313fd1190a3 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h @@ -22,7 +22,7 @@ public: std::string getName() const override { return "LocalObjectStorage"; } - std::string getDataPrefix() const override { return key_prefix; } + std::string getCommonKeyPrefix() const override { return key_prefix; } bool exists(const StoredObject & object) const override; diff --git a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp index 03f30cc631d..6f490013209 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp @@ -59,7 +59,7 @@ static std::string getObjectStoragePrefix( const Poco::Util::AbstractConfiguration & config, const String & config_prefix) { - return config.getString(config_prefix + ".key_compatibility_prefix", object_storage.getDataPrefix()); + return config.getString(config_prefix + ".key_compatibility_prefix", object_storage.getCommonKeyPrefix()); } void registerMetadataStorageFromDisk(MetadataStorageFactory & factory) diff --git a/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp b/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp index 13271cda9db..c2e599f4608 100644 --- a/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp @@ -58,7 +58,7 @@ void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_ch DiskObjectStoragePtr disk = std::make_shared( name, - object_storage->getDataPrefix(), + object_storage->getCommonKeyPrefix(), fmt::format("Disk_{}({})", toString(object_storage->getDataSourceDescription().type), name), std::move(metadata_storage), std::move(object_storage), diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index f3791d1ff4c..1323f3d01f0 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -82,7 +82,7 @@ public: std::string getName() const override { return "S3ObjectStorage"; } - std::string getDataPrefix() const override { return uri.key; } + std::string getCommonKeyPrefix() const override { return uri.key; } bool exists(const StoredObject & object) const override; diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.h b/src/Disks/ObjectStorages/Web/WebObjectStorage.h index 79e79ad7dbe..9b5c9ae2b84 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.h +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.h @@ -33,7 +33,7 @@ public: std::string getName() const override { return "WebObjectStorage"; } - std::string getDataPrefix() const override { return ""; } + std::string getCommonKeyPrefix() const override { return ""; } bool exists(const StoredObject & object) const override; From dc61f68f655994ca1c3e8e913adc81639d94a1a3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 12 Jan 2024 17:55:57 +0100 Subject: [PATCH 12/14] Rename one more method --- src/Disks/ObjectStorages/MetadataStorageFactory.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp index 6f490013209..7a4d3335f28 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp @@ -54,7 +54,7 @@ MetadataStoragePtr MetadataStorageFactory::create( return it->second(name, config, config_prefix, object_storage); } -static std::string getObjectStoragePrefix( +static std::string getObjectKeyCompatiblePrefix( const IObjectStorage & object_storage, const Poco::Util::AbstractConfiguration & config, const String & config_prefix) @@ -74,7 +74,7 @@ void registerMetadataStorageFromDisk(MetadataStorageFactory & factory) fs::path(Context::getGlobalContextInstance()->getPath()) / "disks" / name / ""); fs::create_directories(metadata_path); auto metadata_disk = std::make_shared(name + "-metadata", metadata_path, 0, config, config_prefix); - auto key_compatibility_prefix = getObjectStoragePrefix(*object_storage, config, config_prefix); + auto key_compatibility_prefix = getObjectKeyCompatiblePrefix(*object_storage, config, config_prefix); return std::make_shared(metadata_disk, key_compatibility_prefix); }); } @@ -87,7 +87,7 @@ void registerPlainMetadataStorage(MetadataStorageFactory & factory) const std::string & config_prefix, ObjectStoragePtr object_storage) -> MetadataStoragePtr { - auto key_compatibility_prefix = getObjectStoragePrefix(*object_storage, config, config_prefix); + auto key_compatibility_prefix = getObjectKeyCompatiblePrefix(*object_storage, config, config_prefix); return std::make_shared(object_storage, key_compatibility_prefix); }); } From 341a9b3b7067afa81a37c6819a259dd40f7ca682 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 13 Jan 2024 12:56:27 +0100 Subject: [PATCH 13/14] Fix keeper build --- programs/keeper/CMakeLists.txt | 5 ++++- .../ObjectStorages/MetadataStorageFactory.cpp | 6 ++++++ .../ObjectStorages/ObjectStorageFactory.cpp | 16 ++++++++++++++-- src/Disks/registerDisks.cpp | 4 +--- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 6c4c3e82162..2adab53d092 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -130,10 +130,13 @@ if (BUILD_STANDALONE_KEEPER) ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/ObjectStorageIterator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/StoredObject.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/registerDiskS3.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/S3Capabilities.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/diskSettings.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/DiskS3Utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/ObjectStorageFactory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataStorageFactory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/createReadBufferFromFileBase.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/ReadBufferFromRemoteFSGather.cpp diff --git a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp index 7a4d3335f28..52a0b9ec268 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp @@ -1,7 +1,9 @@ #include #include #include +#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD #include +#endif #include #include @@ -92,6 +94,7 @@ void registerPlainMetadataStorage(MetadataStorageFactory & factory) }); } +#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD void registerMetadataStorageFromStaticFilesWebServer(MetadataStorageFactory & factory) { factory.registerMetadataStorageType("web", []( @@ -103,13 +106,16 @@ void registerMetadataStorageFromStaticFilesWebServer(MetadataStorageFactory & fa return std::make_shared(assert_cast(*object_storage)); }); } +#endif void registerMetadataStorages() { auto & factory = MetadataStorageFactory::instance(); registerMetadataStorageFromDisk(factory); registerPlainMetadataStorage(factory); +#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD registerMetadataStorageFromStaticFilesWebServer(factory); +#endif } } diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp index db230588963..bdae975020d 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -1,15 +1,23 @@ #include "config.h" #include +#if USE_AWS_S3 #include #include #include +#endif +#if USE_HDFS #include +#include +#endif +#if USE_AZURE_BLOB_STORAGE #include #include +#endif +#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD #include #include #include -#include +#endif #include #include @@ -202,6 +210,7 @@ void registerAzureObjectStorage(ObjectStorageFactory & factory) } #endif +#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD void registerWebObjectStorage(ObjectStorageFactory & factory) { factory.registerObjectStorageType("web", []( @@ -246,6 +255,7 @@ void registerLocalObjectStorage(ObjectStorageFactory & factory) return std::make_shared(object_key_prefix); }); } +#endif void registerObjectStorages() { @@ -264,8 +274,10 @@ void registerObjectStorages() registerAzureObjectStorage(factory); #endif - registerLocalObjectStorage(factory); +#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD registerWebObjectStorage(factory); + registerLocalObjectStorage(factory); +#endif } } diff --git a/src/Disks/registerDisks.cpp b/src/Disks/registerDisks.cpp index 6cbcd0ba5ff..b8da93ff9f2 100644 --- a/src/Disks/registerDisks.cpp +++ b/src/Disks/registerDisks.cpp @@ -42,9 +42,7 @@ void registerDisks(bool global_skip_access_check) registerDiskLocal(factory, global_skip_access_check); -#if USE_AWS_S3 - registerDiskS3(factory, global_skip_access_check); -#endif + registerDiskObjectStorage(factory, global_skip_access_check); } #endif From abd800dfa68c53fffbbd4bb228997445816356ad Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 15 Jan 2024 10:39:08 +0100 Subject: [PATCH 14/14] Fix keeper build --- src/Disks/ObjectStorages/ObjectStorageFactory.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp index bdae975020d..54f8fb31acc 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -5,11 +5,11 @@ #include #include #endif -#if USE_HDFS +#if USE_HDFS && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD) #include #include #endif -#if USE_AZURE_BLOB_STORAGE +#if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD) #include #include #endif @@ -165,7 +165,7 @@ void registerS3PlainObjectStorage(ObjectStorageFactory & factory) } #endif -#if USE_HDFS +#if USE_HDFS && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD) void registerHDFSObjectStorage(ObjectStorageFactory & factory) { factory.registerObjectStorageType("hdfs", []( @@ -191,7 +191,7 @@ void registerHDFSObjectStorage(ObjectStorageFactory & factory) } #endif -#if USE_AZURE_BLOB_STORAGE +#if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD) void registerAzureObjectStorage(ObjectStorageFactory & factory) { factory.registerObjectStorageType("azure_blob_storage", []( @@ -266,11 +266,11 @@ void registerObjectStorages() registerS3PlainObjectStorage(factory); #endif -#if USE_HDFS +#if USE_HDFS && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD) registerHDFSObjectStorage(factory); #endif -#if USE_AZURE_BLOB_STORAGE +#if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD) registerAzureObjectStorage(factory); #endif