Merge pull request #46607 from kssenii/delay-loading-of-named-collections

Do not load named collections on server startup (on first access instead)

programs/local/LocalServer.cpp

@@ -37,7 +37,6 @@
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
 #include <Storages/registerStorages.h>
-#include <Common/NamedCollections/NamedCollectionUtils.h>
 #include <Dictionaries/registerDictionaries.h>
 #include <Disks/registerDisks.h>
 #include <Formats/registerFormats.h>
@@ -131,8 +130,6 @@ void LocalServer::initialize(Poco::Util::Application & self)
         config().getUInt("max_io_thread_pool_size", 100),
         config().getUInt("max_io_thread_pool_free_size", 0),
         config().getUInt("io_thread_pool_queue_size", 10000));
-
-    NamedCollectionUtils::loadFromConfig(config());
 }
 
 
@@ -224,8 +221,6 @@ void LocalServer::tryInitPath()
 
     global_context->setUserFilesPath(""); // user's files are everywhere
 
-    NamedCollectionUtils::loadFromSQL(global_context);
-
     /// top_level_domains_lists
     const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/");
     if (!top_level_domains_path.empty())

programs/server/Server.cpp

@@ -770,8 +770,6 @@ try
         config().getUInt("max_io_thread_pool_free_size", 0),
         config().getUInt("io_thread_pool_queue_size", 10000));
 
-    NamedCollectionUtils::loadFromConfig(config());
-
     /// Initialize global local cache for remote filesystem.
     if (config().has("local_cache_for_remote_fs"))
     {
@@ -1177,8 +1175,6 @@ try
         SensitiveDataMasker::setInstance(std::make_unique<SensitiveDataMasker>(config(), "query_masking_rules"));
     }
 
-    NamedCollectionUtils::loadFromSQL(global_context);
-
     auto main_config_reloader = std::make_unique<ConfigReloader>(
         config_path,
         include_from_path,

src/Common/NamedCollections/NamedCollectionUtils.cpp

@@ -32,6 +32,9 @@ namespace ErrorCodes
 namespace NamedCollectionUtils
 {
 
+static std::atomic<bool> is_loaded_from_config = false;
+static std::atomic<bool> is_loaded_from_sql = false;
+
 class LoadFromConfig
 {
 private:
@@ -329,10 +332,21 @@ std::unique_lock<std::mutex> lockNamedCollectionsTransaction()
     return std::unique_lock(transaction_lock);
 }
 
+void loadFromConfigUnlocked(const Poco::Util::AbstractConfiguration & config, std::unique_lock<std::mutex> &)
+{
+    auto named_collections = LoadFromConfig(config).getAll();
+    LOG_TRACE(
+        &Poco::Logger::get("NamedCollectionsUtils"),
+        "Loaded {} collections from config", named_collections.size());
+
+    NamedCollectionFactory::instance().add(std::move(named_collections));
+    is_loaded_from_config = true;
+}
+
 void loadFromConfig(const Poco::Util::AbstractConfiguration & config)
 {
     auto lock = lockNamedCollectionsTransaction();
-    NamedCollectionFactory::instance().add(LoadFromConfig(config).getAll());
+    loadFromConfigUnlocked(config, lock);
 }
 
 void reloadFromConfig(const Poco::Util::AbstractConfiguration & config)
@@ -342,17 +356,47 @@ void reloadFromConfig(const Poco::Util::AbstractConfiguration & config)
     auto & instance = NamedCollectionFactory::instance();
     instance.removeById(SourceId::CONFIG);
     instance.add(collections);
+    is_loaded_from_config = true;
+}
+
+void loadFromSQLUnlocked(ContextPtr context, std::unique_lock<std::mutex> &)
+{
+    auto named_collections = LoadFromSQL(context).getAll();
+    LOG_TRACE(
+        &Poco::Logger::get("NamedCollectionsUtils"),
+        "Loaded {} collections from SQL", named_collections.size());
+
+    NamedCollectionFactory::instance().add(std::move(named_collections));
+    is_loaded_from_sql = true;
 }
 
 void loadFromSQL(ContextPtr context)
 {
     auto lock = lockNamedCollectionsTransaction();
-    NamedCollectionFactory::instance().add(LoadFromSQL(context).getAll());
+    loadFromSQLUnlocked(context, lock);
+}
+
+void loadIfNotUnlocked(std::unique_lock<std::mutex> & lock)
+{
+    auto global_context = Context::getGlobalContextInstance();
+    if (!is_loaded_from_config)
+        loadFromConfigUnlocked(global_context->getConfigRef(), lock);
+    if (!is_loaded_from_sql)
+        loadFromSQLUnlocked(global_context, lock);
+}
+
+void loadIfNot()
+{
+    if (is_loaded_from_sql && is_loaded_from_config)
+        return;
+    auto lock = lockNamedCollectionsTransaction();
+    return loadIfNotUnlocked(lock);
 }
 
 void removeFromSQL(const std::string & collection_name, ContextPtr context)
 {
     auto lock = lockNamedCollectionsTransaction();
+    loadIfNotUnlocked(lock);
     LoadFromSQL(context).remove(collection_name);
     NamedCollectionFactory::instance().remove(collection_name);
 }
@@ -360,6 +404,7 @@ void removeFromSQL(const std::string & collection_name, ContextPtr context)
 void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr context)
 {
     auto lock = lockNamedCollectionsTransaction();
+    loadIfNotUnlocked(lock);
     LoadFromSQL(context).removeIfExists(collection_name);
     NamedCollectionFactory::instance().removeIfExists(collection_name);
 }
@@ -367,12 +412,14 @@ void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr conte
 void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr context)
 {
     auto lock = lockNamedCollectionsTransaction();
+    loadIfNotUnlocked(lock);
     NamedCollectionFactory::instance().add(query.collection_name, LoadFromSQL(context).create(query));
 }
 
 void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context)
 {
     auto lock = lockNamedCollectionsTransaction();
+    loadIfNotUnlocked(lock);
     LoadFromSQL(context).update(query);
 
     auto collection = NamedCollectionFactory::instance().getMutable(query.collection_name);

src/Common/NamedCollections/NamedCollectionUtils.h

@@ -35,6 +35,8 @@ void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr conte
 /// Update definition of already existing collection from AST and update result in `context->getPath() / named_collections /`.
 void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context);
 
+void loadIfNot();
+
 }
 
 }

src/Storages/NamedCollectionsHelpers.cpp

@@ -58,6 +58,8 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts)
     if (asts.empty())
         return nullptr;
 
+    NamedCollectionUtils::loadIfNot();
+
     auto collection = tryGetNamedCollectionFromASTs(asts);
     if (!collection)
         return nullptr;

src/Storages/System/StorageSystemNamedCollections.cpp

@@ -31,6 +31,8 @@ void StorageSystemNamedCollections::fillData(MutableColumns & res_columns, Conte
 {
     context->checkAccess(AccessType::SHOW_NAMED_COLLECTIONS);
 
+    NamedCollectionUtils::loadIfNot();
+
     auto collections = NamedCollectionFactory::instance().getAll();
     for (const auto & [name, collection] : collections)
     {