diff --git a/src/Disks/IStoragePolicy.h b/src/Disks/IStoragePolicy.h new file mode 100644 index 00000000000..a41ea87c328 --- /dev/null +++ b/src/Disks/IStoragePolicy.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include +#include + +namespace DB +{ +class IStoragePolicy; +using StoragePolicyPtr = std::shared_ptr; +class IVolume; +using VolumePtr = std::shared_ptr; +using Volumes = std::vector; +class IDisk; +using DiskPtr = std::shared_ptr; +using Disks = std::vector; +class IReservation; +using ReservationPtr = std::unique_ptr; +using Reservations = std::vector; + +using String = std::string; + +class IStoragePolicy +{ +public: + virtual ~IStoragePolicy() = default; + virtual const String & getName() const = 0; + virtual const Volumes & getVolumes() const = 0; + /// Returns number [0., 1.] -- fraction of free space on disk + /// which should be kept with help of background moves + virtual double getMoveFactor() const = 0; + virtual bool isDefaultPolicy() const = 0; + /// Returns disks ordered by volumes priority + virtual Disks getDisks() const = 0; + /// Returns any disk + /// Used when it's not important, for example for + /// mutations files + virtual DiskPtr getAnyDisk() const = 0; + virtual DiskPtr getDiskByName(const String & disk_name) const = 0; + /// Get free space from most free disk + virtual UInt64 getMaxUnreservedFreeSpace() const = 0; + /// Reserves space on any volume with index > min_volume_index or returns nullptr + virtual ReservationPtr reserve(UInt64 bytes, size_t min_volume_index) const = 0; + /// Returns valid reservation or nullptr + virtual ReservationPtr reserve(UInt64 bytes) const = 0; + /// Reserves space on any volume or throws + virtual ReservationPtr reserveAndCheck(UInt64 bytes) const = 0; + /// Reserves 0 bytes on disk with max available space + /// Do not use this function when it is possible to predict size. + virtual ReservationPtr makeEmptyReservationOnLargestDisk() const = 0; + /// Get volume by index. + virtual VolumePtr getVolume(size_t index) const = 0; + virtual VolumePtr getVolumeByName(const String & volume_name) const = 0; + /// Checks if storage policy can be replaced by another one. + virtual void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const = 0; + /// Find volume index, which contains disk + virtual size_t getVolumeIndexByDisk(const DiskPtr & disk_ptr) const = 0; + /// Check if we have any volume with stopped merges + virtual bool hasAnyVolumeWithDisabledMerges() const = 0; + virtual bool containsVolume(const String & volume_name) const = 0; +}; + +} diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index e3a937cae55..a1345879c83 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -93,17 +93,17 @@ StoragePolicy::StoragePolicy(String name_, Volumes volumes_, double move_factor_ } -StoragePolicy::StoragePolicy(const StoragePolicy & storage_policy, +StoragePolicy::StoragePolicy(StoragePolicyPtr storage_policy, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks) - : StoragePolicy(storage_policy.getName(), config, config_prefix, disks) + : StoragePolicy(storage_policy->getName(), config, config_prefix, disks) { for (auto & volume : volumes) { - if (storage_policy.volume_index_by_volume_name.count(volume->getName()) > 0) + if (storage_policy->containsVolume(volume->getName())) { - auto old_volume = storage_policy.getVolumeByName(volume->getName()); + auto old_volume = storage_policy->getVolumeByName(volume->getName()); try { auto new_volume = updateVolumeFromConfig(old_volume, config, config_prefix + ".volumes." + volume->getName(), disks); @@ -112,7 +112,7 @@ StoragePolicy::StoragePolicy(const StoragePolicy & storage_policy, catch (Exception & e) { /// Default policies are allowed to be missed in configuration. - if (e.code() != ErrorCodes::NO_ELEMENTS_IN_CONFIG || storage_policy.getName() != DEFAULT_STORAGE_POLICY_NAME) + if (e.code() != ErrorCodes::NO_ELEMENTS_IN_CONFIG || storage_policy->getName() != DEFAULT_STORAGE_POLICY_NAME) throw; Poco::Util::AbstractConfiguration::Keys keys; @@ -331,6 +331,11 @@ bool StoragePolicy::hasAnyVolumeWithDisabledMerges() const return false; } +bool StoragePolicy::containsVolume(const String & volume_name) const +{ + return volume_index_by_volume_name.contains(volume_name); +} + StoragePolicySelector::StoragePolicySelector( const Poco::Util::AbstractConfiguration & config, const String & config_prefix, @@ -345,6 +350,13 @@ StoragePolicySelector::StoragePolicySelector( throw Exception( "Storage policy name can contain only alphanumeric and '_' (" + backQuote(name) + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); + /* + * A customization point for StoragePolicy, here one can add his own policy, for example, based on policy's name + * if (name == "MyCustomPolicy") + * policies.emplace(name, std::make_shared(name, config, config_prefix + "." + name, disks)); + * else + */ + policies.emplace(name, std::make_shared(name, config, config_prefix + "." + name, disks)); LOG_INFO(&Poco::Logger::get("StoragePolicySelector"), "Storage policy {} loaded", backQuote(name)); } @@ -374,7 +386,7 @@ StoragePolicySelectorPtr StoragePolicySelector::updateFromConfig(const Poco::Uti /// Second pass, load. for (const auto & [name, policy] : policies) { - result->policies[name] = std::make_shared(*policy, config, config_prefix + "." + name, disks); + result->policies[name] = std::make_shared(policy, config, config_prefix + "." + name, disks); } return result; diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index 9135c27d1c0..6676ab19043 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -23,14 +24,11 @@ namespace DB { -class StoragePolicy; -using StoragePolicyPtr = std::shared_ptr; - /** * Contains all information about volumes configuration for Storage. * Can determine appropriate Volume and Disk for each reservation. */ -class StoragePolicy +class StoragePolicy : public IStoragePolicy { public: StoragePolicy(String name_, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks); @@ -38,62 +36,63 @@ public: StoragePolicy(String name_, Volumes volumes_, double move_factor_); StoragePolicy( - const StoragePolicy & storage_policy, + StoragePolicyPtr storage_policy, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks ); - bool isDefaultPolicy() const; + bool isDefaultPolicy() const override; /// Returns disks ordered by volumes priority - Disks getDisks() const; + Disks getDisks() const override; /// Returns any disk /// Used when it's not important, for example for /// mutations files - DiskPtr getAnyDisk() const; + DiskPtr getAnyDisk() const override; - DiskPtr getDiskByName(const String & disk_name) const; + DiskPtr getDiskByName(const String & disk_name) const override; /// Get free space from most free disk - UInt64 getMaxUnreservedFreeSpace() const; + UInt64 getMaxUnreservedFreeSpace() const override; - const String & getName() const { return name; } + const String & getName() const override{ return name; } /// Returns valid reservation or nullptr - ReservationPtr reserve(UInt64 bytes) const; + ReservationPtr reserve(UInt64 bytes) const override; /// Reserves space on any volume or throws - ReservationPtr reserveAndCheck(UInt64 bytes) const; + ReservationPtr reserveAndCheck(UInt64 bytes) const override; /// Reserves space on any volume with index > min_volume_index or returns nullptr - ReservationPtr reserve(UInt64 bytes, size_t min_volume_index) const; + ReservationPtr reserve(UInt64 bytes, size_t min_volume_index) const override; /// Find volume index, which contains disk - size_t getVolumeIndexByDisk(const DiskPtr & disk_ptr) const; + size_t getVolumeIndexByDisk(const DiskPtr & disk_ptr) const override; /// Reserves 0 bytes on disk with max available space /// Do not use this function when it is possible to predict size. - ReservationPtr makeEmptyReservationOnLargestDisk() const; + ReservationPtr makeEmptyReservationOnLargestDisk() const override; - const Volumes & getVolumes() const { return volumes; } + const Volumes & getVolumes() const override{ return volumes; } /// Returns number [0., 1.] -- fraction of free space on disk /// which should be kept with help of background moves - double getMoveFactor() const { return move_factor; } + double getMoveFactor() const override{ return move_factor; } /// Get volume by index. - VolumePtr getVolume(size_t index) const; + VolumePtr getVolume(size_t index) const override; - VolumePtr getVolumeByName(const String & volume_name) const; + VolumePtr getVolumeByName(const String & volume_name) const override; /// Checks if storage policy can be replaced by another one. - void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const; + void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const override; /// Check if we have any volume with stopped merges - bool hasAnyVolumeWithDisabledMerges() const; + bool hasAnyVolumeWithDisabledMerges() const override; + bool containsVolume(const String & volume_name) const override; private: Volumes volumes; const String name; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index d83fef72882..8040091256c 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 83a46ef5a2b..5801cc2b949 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -102,8 +102,8 @@ using DiskPtr = std::shared_ptr; class DiskSelector; using DiskSelectorPtr = std::shared_ptr; using DisksMap = std::map; -class StoragePolicy; -using StoragePolicyPtr = std::shared_ptr; +class IStoragePolicy; +using StoragePolicyPtr = std::shared_ptr; using StoragePoliciesMap = std::map; class StoragePolicySelector; using StoragePolicySelectorPtr = std::shared_ptr; diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index 35df9cd4516..b12616dba1e 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace DB { diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index ce6d0ad1f6c..1806693db3a 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include namespace ProfileEvents diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 5006a817b5b..e161b5752ae 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 031b960fac1..1c0149ac261 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -50,8 +50,8 @@ class Pipe; class QueryPlan; using QueryPlanPtr = std::unique_ptr; -class StoragePolicy; -using StoragePolicyPtr = std::shared_ptr; +class IStoragePolicy; +using StoragePolicyPtr = std::shared_ptr; struct StreamLocalLimits; class EnabledQuota; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index f6581574ede..c00a2cbfa08 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index afd7d6b876e..5227cd8a33e 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 9fca8c49e81..11a159d4a6c 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a6fd50c4c8e..69cbe0d7062 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -27,7 +27,6 @@ #include #include -#include #include diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 363a2a20828..132ed234323 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include