From 99e9c0c4864894de5986d4c627769c18c6d0fb9c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 6 Oct 2017 14:10:01 +0300 Subject: [PATCH] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] --- dbms/src/Dictionaries/IDictionary.h | 13 ++- .../src/Interpreters/ExternalDictionaries.cpp | 6 +- dbms/src/Interpreters/ExternalDictionaries.h | 4 +- dbms/src/Interpreters/ExternalLoader.cpp | 34 +++++-- dbms/src/Interpreters/ExternalLoader.h | 97 +++++++++---------- dbms/src/Interpreters/IExternalLoadable.h | 12 ++- 6 files changed, 94 insertions(+), 72 deletions(-) diff --git a/dbms/src/Dictionaries/IDictionary.h b/dbms/src/Dictionaries/IDictionary.h index f378a9a7b07..48b8206fbca 100644 --- a/dbms/src/Dictionaries/IDictionary.h +++ b/dbms/src/Dictionaries/IDictionary.h @@ -8,16 +8,14 @@ #include #include #include +#include namespace DB { -class IDictionarySource; - struct IDictionaryBase; using DictionaryPtr = std::unique_ptr; -struct DictionaryLifetime; struct DictionaryStructure; class ColumnString; @@ -57,9 +55,14 @@ struct IDictionaryBase : public IExternalLoadable bool supportUpdates() const override { return !isCached(); } - virtual std::shared_ptr cloneObject() const override + bool isModified() const override + { auto source = getSource(); + return source && source->isModified(); + } + + std::unique_ptr cloneObject() const override { - return std::static_pointer_cast(clone()); + return clone(); }; std::shared_ptr shared_from_this() diff --git a/dbms/src/Interpreters/ExternalDictionaries.cpp b/dbms/src/Interpreters/ExternalDictionaries.cpp index b42d703d090..4c1e563e1f2 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.cpp +++ b/dbms/src/Interpreters/ExternalDictionaries.cpp @@ -44,13 +44,13 @@ ExternalDictionaries::ExternalDictionaries(Context & context, bool throw_on_erro getExternalDictionariesUpdateSettings(), getExternalDictionariesConfigSettings(), &Logger::get("ExternalDictionaries"), - "external dictionary", throw_on_error), + "external dictionary"), context(context) { - + init(throw_on_error); } -ExternalDictionaries::LoadablePtr ExternalDictionaries::create( +std::unique_ptr ExternalDictionaries::create( const std::string & name, const Configuration & config, const std::string & config_prefix) { return DictionaryFactory::instance().create(name, config, config_prefix, context); diff --git a/dbms/src/Interpreters/ExternalDictionaries.h b/dbms/src/Interpreters/ExternalDictionaries.h index 31ff6ad5341..9ed41f89d50 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.h +++ b/dbms/src/Interpreters/ExternalDictionaries.h @@ -30,8 +30,8 @@ public: protected: - LoadablePtr create(const std::string & name, const Configuration & config, - const std::string & config_prefix) override; + std::unique_ptr create(const std::string & name, const Configuration & config, + const std::string & config_prefix) override; using ExternalLoader::getObjectsMap; diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 42967f541c8..9b784639366 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -1,7 +1,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -46,10 +48,19 @@ void ExternalLoader::reloadPeriodically() ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config, const ExternalLoaderUpdateSettings & update_settings, const ExternalLoaderConfigSettings & config_settings, - Logger * log, const std::string & loadable_object_name, bool throw_on_error) + Logger * log, const std::string & loadable_object_name) : config(config), update_settings(update_settings), config_settings(config_settings), log(log), object_name(loadable_object_name) { +} + +void ExternalLoader::init(bool throw_on_error) +{ + if (is_initialized) + return; + + is_initialized = true; + { /// During synchronous loading of external dictionaries at moment of query execution, /// we should not use per query memory limit. @@ -116,7 +127,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) try { - auto loadable_ptr = failed_loadable_object.second.loadable->clone(); + auto loadable_ptr = failed_loadable_object.second.loadable->cloneObject(); if (const auto exception_ptr = loadable_ptr->getCreationException()) { /// recalculate next attempt time @@ -143,7 +154,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) const auto dict_it = loadable_objects.find(name); dict_it->second.loadable.reset(); - dict_it->second.loadable = loadable_ptr; + dict_it->second.loadable = std::move(loadable_ptr); /// clear stored exception on success dict_it->second.exception = std::exception_ptr{}; @@ -200,13 +211,13 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) if (current->isModified()) { /// create new version of loadable object - auto new_version = current->clone(); + auto new_version = current->cloneObject(); if (const auto exception_ptr = new_version->getCreationException()) std::rethrow_exception(exception_ptr); loadable_object.second.loadable.reset(); - loadable_object.second.loadable = new_version; + loadable_object.second.loadable = std::move(new_version); } } @@ -321,9 +332,9 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const const auto failed_dict_it = failed_loadable_objects.find(name); FailedLoadableInfo info{std::move(object_ptr), std::chrono::system_clock::now() + delay, 0}; if (failed_dict_it != std::end(failed_loadable_objects)) - failed_dict_it->second = info; + (*failed_dict_it).second = std::move(info); else - failed_loadable_objects.emplace(name, info); + failed_loadable_objects.emplace(name, std::move(info)); std::rethrow_exception(exception_ptr); } @@ -343,12 +354,12 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const /// add new loadable object or update an existing version if (object_it == std::end(loadable_objects)) - loadable_objects.emplace(name, LoadableInfo{object_ptr, config_path}); + loadable_objects.emplace(name, LoadableInfo{std::move(object_ptr), config_path}); else { if (object_it->second.loadable) object_it->second.loadable.reset(); - object_it->second.loadable = object_ptr; + object_it->second.loadable = std::move(object_ptr); /// erase stored exception on success object_it->second.exception = std::exception_ptr{}; @@ -414,4 +425,9 @@ ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name return it->second.loadable; } +std::tuple, const ExternalLoader::ObjectsMap &> ExternalLoader::getObjectsMap() const +{ + return std::make_tuple(std::unique_lock(map_mutex), std::cref(loadable_objects)); +} + } diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index d330bae2633..672c801424e 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -1,20 +1,16 @@ #pragma once -#include -#include -#include -#include #include #include -#include -#include #include #include #include #include -#include -#include +#include #include +#include +#include +#include namespace DB @@ -65,43 +61,9 @@ struct ExternalLoaderConfigSettings class ExternalLoader { public: - using Configuration = Poco::Util::AbstractConfiguration; - using ObjectsMap = std::unordered_map; - - /// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. - ExternalLoader(const Configuration & config, - const ExternalLoaderUpdateSettings & update_settings, - const ExternalLoaderConfigSettings & config_settings, - Logger * log, const std::string & loadable_object_name, bool throw_on_error); - ~ExternalLoader(); - - /// Forcibly reloads all loadable objects. - void reload(); - - /// Forcibly reloads specified loadable object. - void reload(const std::string & name); - - LoadablePtr getLoadable(const std::string & name) const; - -protected: - virtual LoadablePtr create(const std::string & name, const Configuration & config, - const std::string & config_prefix) = 0; - - /// Direct access to objects. - std::tuple, const ObjectsMap &> getObjectsMap() - { - return std::make_tuple(std::lock_guard(map_mutex), std::cref(loadable_objects)); - } + using LoadablePtr = std::shared_ptr; private: - - /// Protects only dictionaries map. - mutable std::mutex map_mutex; - - /// Protects all data, currently used to avoid races between updating thread and SYSTEM queries - mutable std::mutex all_mutex; - - using LoadablePtr = std::shared_ptr; struct LoadableInfo final { LoadablePtr loadable; @@ -116,16 +78,53 @@ private: UInt64 error_count; }; +public: + using Configuration = Poco::Util::AbstractConfiguration; + using ObjectsMap = std::unordered_map; + + /// Objects will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. + ExternalLoader(const Configuration & config, + const ExternalLoaderUpdateSettings & update_settings, + const ExternalLoaderConfigSettings & config_settings, + Logger * log, const std::string & loadable_object_name); + virtual ~ExternalLoader(); + + /// Forcibly reloads all loadable objects. + void reload(); + + /// Forcibly reloads specified loadable object. + void reload(const std::string & name); + + LoadablePtr getLoadable(const std::string & name) const; + +protected: + virtual std::unique_ptr create(const std::string & name, const Configuration & config, + const std::string & config_prefix) = 0; + + /// Direct access to objects. + std::tuple, const ObjectsMap &> getObjectsMap() const; + + /// Should be called in derived constructor (to avoid pure virtual call). + void init(bool throw_on_error); + +private: + + bool is_initialized = false; + + /// Protects only objects map. + mutable std::mutex map_mutex; + + /// Protects all data, currently used to avoid races between updating thread and SYSTEM queries + mutable std::mutex all_mutex; + /// name -> loadable. ObjectsMap loadable_objects; - /** Here are loadable objects, that has been never loaded successfully. - * They are also in 'loadable_objects', but with nullptr as 'loadable'. - */ + /// Here are loadable objects, that has been never loaded successfully. + /// They are also in 'loadable_objects', but with nullptr as 'loadable'. std::unordered_map failed_loadable_objects; - /** Both for dictionaries and failed_dictionaries. - */ + /// Both for loadable_objects and failed_loadable_objects. std::unordered_map update_times; pcg64 rnd_engine{randomSeed()}; @@ -143,7 +142,7 @@ private: std::unordered_map last_modification_times; - /// Check dictionaries definitions in config files and reload or/and add new ones if the definition is changed + /// Check objects definitions in config files and reload or/and add new ones if the definition is changed /// If loadable_name is not empty, load only loadable object with name loadable_name void reloadFromConfigFiles(bool throw_on_error, bool force_reload = false, const std::string & loadable_name = ""); void reloadFromConfigFile(const std::string & config_path, bool throw_on_error, bool force_reload, diff --git a/dbms/src/Interpreters/IExternalLoadable.h b/dbms/src/Interpreters/IExternalLoadable.h index 841aba86dfa..9e52b9e6864 100644 --- a/dbms/src/Interpreters/IExternalLoadable.h +++ b/dbms/src/Interpreters/IExternalLoadable.h @@ -1,14 +1,18 @@ +#pragma once #include #include +#include -namespace DB -{ namespace Poco::Util { -class AbstractConfiguration; + class AbstractConfiguration; } + +namespace DB +{ + /// Min and max lifetimes for a loadable object or it's entry struct ExternalLoadableLifetime final { @@ -31,7 +35,7 @@ public: virtual bool isModified() const = 0; - virtual std::shared_ptr cloneObject() const = 0; + virtual std::unique_ptr cloneObject() const = 0; virtual std::exception_ptr getCreationException() const = 0; };