Merge pull request #1555 from rlipovsky/external_dicts_repo

[clickhouse-yt] abstract config repository for user-defined objects (dictionaries, etc)

dbms/src/Interpreters/Context.cpp

@@ -1113,7 +1113,13 @@ ExternalDictionaries & Context::getExternalDictionariesImpl(const bool throw_on_
     {
         if (!this->global_context)
             throw Exception("Logical error: there is no global context", ErrorCodes::LOGICAL_ERROR);
-        shared->external_dictionaries = std::make_shared<ExternalDictionaries>(*this->global_context, throw_on_error);
+
+        auto config_repository = runtime_components_factory->createExternalDictionariesConfigRepository();
+
+        shared->external_dictionaries = std::make_shared<ExternalDictionaries>(
+            std::move(config_repository),
+            *this->global_context,
+            throw_on_error);
     }
 
     return *shared->external_dictionaries;
@@ -1127,7 +1133,13 @@ ExternalModels & Context::getExternalModelsImpl(bool throw_on_error) const
     {
         if (!this->global_context)
             throw Exception("Logical error: there is no global context", ErrorCodes::LOGICAL_ERROR);
-        shared->external_models = std::make_shared<ExternalModels>(*this->global_context, throw_on_error);
+
+        auto config_repository = runtime_components_factory->createExternalModelsConfigRepository();
+
+        shared->external_models = std::make_shared<ExternalModels>(
+            std::move(config_repository),
+            *this->global_context,
+            throw_on_error);
     }
 
     return *shared->external_models;

dbms/src/Interpreters/ExternalDictionaries.cpp

@@ -26,10 +26,14 @@ namespace
 }
 
 
-ExternalDictionaries::ExternalDictionaries(Context & context, bool throw_on_error)
+ExternalDictionaries::ExternalDictionaries(
+    std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
+    Context & context,
+    bool throw_on_error)
         : ExternalLoader(context.getConfigRef(),
                          externalDictionariesUpdateSettings,
                          getExternalDictionariesConfigSettings(),
+                         std::move(config_repository),
                          &Logger::get("ExternalDictionaries"),
                          "external dictionary"),
           context(context)

dbms/src/Interpreters/ExternalDictionaries.h

@@ -18,7 +18,10 @@ public:
     using DictPtr = std::shared_ptr<IDictionaryBase>;
 
     /// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
-    ExternalDictionaries(Context & context, bool throw_on_error);
+    ExternalDictionaries(
+        std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
+        Context & context,
+        bool throw_on_error);
 
     /// Forcibly reloads specified dictionary.
     void reloadDictionary(const std::string & name) { reload(name); }

dbms/src/Interpreters/ExternalLoader.cpp

@@ -2,15 +2,10 @@
 #include <Common/StringUtils.h>
 #include <Common/MemoryTracker.h>
 #include <Common/Exception.h>
-#include <Common/getMultipleKeysFromConfig.h>
 #include <Common/setThreadName.h>
 #include <ext/scope_guard.h>
 #include <Poco/Util/Application.h>
-#include <Poco/Glob.h>
-#include <Poco/File.h>
 #include <cmath>
-#include <Poco/Util/XMLConfiguration.h>
-#include <Common/ConfigProcessor.h>
 
 namespace DB
 {
@@ -49,9 +44,14 @@ void ExternalLoader::reloadPeriodically()
 ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config,
                                const ExternalLoaderUpdateSettings & update_settings,
                                const ExternalLoaderConfigSettings & config_settings,
+                               std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
                                Logger * log, const std::string & loadable_object_name)
-        : config(config), update_settings(update_settings), config_settings(config_settings),
-          log(log), object_name(loadable_object_name)
+        : config(config)
+        , update_settings(update_settings)
+        , config_settings(config_settings)
+        , config_repository(std::move(config_repository))
+        , log(log)
+        , object_name(loadable_object_name)
 {
 }
 
@@ -80,34 +80,6 @@ ExternalLoader::~ExternalLoader()
     reloading_thread.join();
 }
 
-namespace
-{
-std::set<std::string> getConfigPaths(const Poco::Util::AbstractConfiguration & config,
-                                     const std::string & external_config_paths_setting)
-{
-    std::set<std::string> files;
-    auto patterns = getMultipleValuesFromConfig(config, "", external_config_paths_setting);
-    for (auto & pattern : patterns)
-    {
-        if (pattern.empty())
-            continue;
-
-        if (pattern[0] != '/')
-        {
-            const auto app_config_path = config.getString("config-file", "config.xml");
-            const auto config_dir = Poco::Path{app_config_path}.parent().toString();
-            const auto absolute_path = config_dir + pattern;
-            Poco::Glob::glob(absolute_path, files, 0);
-            if (!files.empty())
-                continue;
-        }
-
-        Poco::Glob::glob(pattern, files, 0);
-    }
-
-    return files;
-}
-}
 
 void ExternalLoader::reloadAndUpdate(bool throw_on_error)
 {
@@ -239,7 +211,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
 
 void ExternalLoader::reloadFromConfigFiles(const bool throw_on_error, const bool force_reload, const std::string & only_dictionary)
 {
-    const auto config_paths = getConfigPaths(config, config_settings.path_setting_name);
+    const auto config_paths = config_repository->list(config, config_settings.path_setting_name);
 
     for (const auto & config_path : config_paths)
     {
@@ -260,9 +232,7 @@ void ExternalLoader::reloadFromConfigFiles(const bool throw_on_error, const bool
 void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const bool throw_on_error,
                                           const bool force_reload, const std::string & loadable_name)
 {
-    const Poco::File config_file{config_path};
-
-    if (config_path.empty() || !config_file.exists())
+    if (config_path.empty() || !config_repository->exists(config_path))
     {
         LOG_WARNING(log, "config file '" + config_path + "' does not exist");
     }
@@ -275,14 +245,10 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const
             modification_time_it = last_modification_times.emplace(config_path, Poco::Timestamp{0}).first;
         auto & config_last_modified = modification_time_it->second;
 
-        const auto last_modified = config_file.getLastModified();
+        const auto last_modified = config_repository->getLastModificationTime(config_path);
         if (force_reload || last_modified > config_last_modified)
         {
-            ConfigProcessor config_processor = ConfigProcessor(config_path);
-            ConfigProcessor::LoadedConfig cfg = config_processor.loadConfig();
-            config_processor.savePreprocessedConfig(cfg);
-            
-            Poco::AutoPtr<Poco::Util::AbstractConfiguration> config = cfg.configuration;
+            auto config = config_repository->load(config_path);
 
             /// Definitions of loadable objects may have changed, recreate all of them
 

dbms/src/Interpreters/ExternalLoader.h

@@ -8,6 +8,7 @@
 #include <chrono>
 #include <tuple>
 #include <Interpreters/IExternalLoadable.h>
+#include <Interpreters/IExternalLoaderConfigRepository.h>
 #include <Core/Types.h>
 #include <pcg_random.hpp>
 #include <Common/randomSeed.h>
@@ -92,6 +93,7 @@ public:
     ExternalLoader(const Configuration & config,
                    const ExternalLoaderUpdateSettings & update_settings,
                    const ExternalLoaderConfigSettings & config_settings,
+                   std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
                    Logger * log, const std::string & loadable_object_name);
     virtual ~ExternalLoader();
 
@@ -149,6 +151,8 @@ private:
     const ExternalLoaderUpdateSettings & update_settings;
     const ExternalLoaderConfigSettings & config_settings;
 
+    std::unique_ptr<IExternalLoaderConfigRepository> config_repository;
+
     std::thread reloading_thread;
     Poco::Event destroy;
 

dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp

@@ -0,0 +1,62 @@
+#include <Interpreters/ExternalLoaderConfigRepository.h>
+
+#include <Common/ConfigProcessor.h>
+#include <Common/getMultipleKeysFromConfig.h>
+
+#include <Poco/Glob.h>
+#include <Poco/File.h>
+#include <Poco/Path.h>
+
+namespace DB
+{
+
+ExternalLoaderConfigRepository::Files ExternalLoaderConfigRepository::list(
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & path_key) const
+{
+    Files files;
+
+    auto patterns = getMultipleValuesFromConfig(config, "", path_key);
+
+    for (auto & pattern : patterns)
+    {
+        if (pattern.empty())
+            continue;
+
+        if (pattern[0] != '/')
+        {
+            const auto app_config_path = config.getString("config-file", "config.xml");
+            const auto config_dir = Poco::Path{app_config_path}.parent().toString();
+            const auto absolute_path = config_dir + pattern;
+            Poco::Glob::glob(absolute_path, files, 0);
+            if (!files.empty())
+                continue;
+        }
+
+        Poco::Glob::glob(pattern, files, 0);
+    }
+
+    return files;
+}
+
+bool ExternalLoaderConfigRepository::exists(const std::string & config_file) const
+{
+    return Poco::File(config_file).exists();
+}
+
+Poco::Timestamp ExternalLoaderConfigRepository::getLastModificationTime(
+    const std::string & config_file) const
+{
+    return Poco::File(config_file).getLastModified();
+}
+
+Poco::AutoPtr<Poco::Util::AbstractConfiguration> ExternalLoaderConfigRepository::load(
+    const std::string & config_file) const
+{
+    ConfigProcessor config_processor{config_file};
+    ConfigProcessor::LoadedConfig preprocessed = config_processor.loadConfig();
+    config_processor.savePreprocessedConfig(preprocessed);
+    return preprocessed.configuration;
+}
+
+}

dbms/src/Interpreters/ExternalLoaderConfigRepository.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <Interpreters/IExternalLoaderConfigRepository.h>
+
+namespace DB
+{
+
+/** Default implementation of config repository used by native server application.
+  * Represents files in local filesystem.
+  */
+class ExternalLoaderConfigRepository : public IExternalLoaderConfigRepository
+{
+public:
+    Files list(
+        const Poco::Util::AbstractConfiguration & config,
+        const std::string & path_key) const override;
+
+    bool exists(const std::string & config_file) const override;
+
+    Poco::Timestamp getLastModificationTime(const std::string & config_file) const override;
+
+    Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file) const override;
+};
+
+}

dbms/src/Interpreters/ExternalModels.cpp

@@ -31,10 +31,14 @@ namespace
 }
 
 
-ExternalModels::ExternalModels(Context & context, bool throw_on_error)
+ExternalModels::ExternalModels(
+    std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
+    Context & context,
+    bool throw_on_error)
         : ExternalLoader(context.getConfigRef(),
                          externalModelsUpdateSettings,
                          getExternalModelsConfigSettings(),
+                         std::move(config_repository),
                          &Logger::get("ExternalModels"),
                          "external model"),
           context(context)

dbms/src/Interpreters/ExternalModels.h

@@ -18,7 +18,10 @@ public:
     using ModelPtr = std::shared_ptr<IModel>;
 
     /// Models will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
-    ExternalModels(Context & context, bool throw_on_error);
+    ExternalModels(
+        std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
+        Context & context,
+        bool throw_on_error);
 
     /// Forcibly reloads specified model.
     void reloadModel(const std::string & name) { reload(name); }

dbms/src/Interpreters/IExternalLoaderConfigRepository.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <Poco/AutoPtr.h>
+#include <Poco/Timestamp.h>
+#include <Poco/Util/AbstractConfiguration.h>
+
+#include <string>
+#include <set>
+
+namespace DB
+{
+
+/** Repository with configurations of user-defined objects (dictionaries, models).
+  * Used by ExternalLoader.
+  */ 
+class IExternalLoaderConfigRepository
+{
+public:
+    using Files = std::set<std::string>;
+    virtual Files list(const Poco::Util::AbstractConfiguration & config, const std::string & path_key) const = 0;
+
+    virtual bool exists(const std::string & config_file) const = 0;
+
+    virtual Poco::Timestamp getLastModificationTime(const std::string & config_file) const = 0;
+
+    virtual Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file) const = 0;
+
+    virtual ~IExternalLoaderConfigRepository() {}
+};
+
+}

dbms/src/Interpreters/IRuntimeComponentsFactory.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <Interpreters/IExternalLoaderConfigRepository.h>
 #include <Interpreters/ISecurityManager.h>
 
 #include <memory>
@@ -16,6 +17,11 @@ class IRuntimeComponentsFactory
 public:
     virtual std::unique_ptr<ISecurityManager> createSecurityManager() = 0;
 
+    // Repositories with configurations of user-defined objects (dictionaries, models)
+    virtual std::unique_ptr<IExternalLoaderConfigRepository> createExternalDictionariesConfigRepository() = 0;
+
+    virtual std::unique_ptr<IExternalLoaderConfigRepository> createExternalModelsConfigRepository() = 0;
+
     virtual ~IRuntimeComponentsFactory() {}
 };
 

dbms/src/Interpreters/RuntimeComponentsFactory.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Interpreters/IRuntimeComponentsFactory.h>
+#include <Interpreters/ExternalLoaderConfigRepository.h>
 #include <Interpreters/SecurityManager.h>
 
 namespace DB
@@ -16,6 +17,16 @@ public:
     {
         return std::make_unique<SecurityManager>();
     }
+
+    std::unique_ptr<IExternalLoaderConfigRepository> createExternalDictionariesConfigRepository() override
+    {
+        return std::make_unique<ExternalLoaderConfigRepository>();
+    }
+
+    std::unique_ptr<IExternalLoaderConfigRepository> createExternalModelsConfigRepository() override
+    {
+        return std::make_unique<ExternalLoaderConfigRepository>();
+    }
 };
 
 }