From 9d16205c8a532f5b2600fe9049c99372535a6dc9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Feb 2023 15:29:09 +0100 Subject: [PATCH] Load named collections on first access --- programs/local/LocalServer.cpp | 5 -- programs/server/Server.cpp | 4 -- .../NamedCollections/NamedCollectionUtils.cpp | 51 ++++++++++++++++++- .../NamedCollections/NamedCollectionUtils.h | 2 + src/Storages/NamedCollectionsHelpers.cpp | 2 + .../System/StorageSystemNamedCollections.cpp | 2 + 6 files changed, 55 insertions(+), 11 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 133d629bbb1..23ba65fed44 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -131,8 +130,6 @@ void LocalServer::initialize(Poco::Util::Application & self) config().getUInt("max_io_thread_pool_size", 100), config().getUInt("max_io_thread_pool_free_size", 0), config().getUInt("io_thread_pool_queue_size", 10000)); - - NamedCollectionUtils::loadFromConfig(config()); } @@ -224,8 +221,6 @@ void LocalServer::tryInitPath() global_context->setUserFilesPath(""); // user's files are everywhere - NamedCollectionUtils::loadFromSQL(global_context); - /// top_level_domains_lists const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/"); if (!top_level_domains_path.empty()) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b97b48d9c68..10710d61b84 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -770,8 +770,6 @@ try config().getUInt("max_io_thread_pool_free_size", 0), config().getUInt("io_thread_pool_queue_size", 10000)); - NamedCollectionUtils::loadFromConfig(config()); - /// Initialize global local cache for remote filesystem. if (config().has("local_cache_for_remote_fs")) { @@ -1177,8 +1175,6 @@ try SensitiveDataMasker::setInstance(std::make_unique(config(), "query_masking_rules")); } - NamedCollectionUtils::loadFromSQL(global_context); - auto main_config_reloader = std::make_unique( config_path, include_from_path, diff --git a/src/Common/NamedCollections/NamedCollectionUtils.cpp b/src/Common/NamedCollections/NamedCollectionUtils.cpp index 8beaa38210e..6ec09fb8a77 100644 --- a/src/Common/NamedCollections/NamedCollectionUtils.cpp +++ b/src/Common/NamedCollections/NamedCollectionUtils.cpp @@ -32,6 +32,9 @@ namespace ErrorCodes namespace NamedCollectionUtils { +static std::atomic is_loaded_from_config = false; +static std::atomic is_loaded_from_sql = false; + class LoadFromConfig { private: @@ -329,10 +332,21 @@ std::unique_lock lockNamedCollectionsTransaction() return std::unique_lock(transaction_lock); } +void loadFromConfigUnlocked(const Poco::Util::AbstractConfiguration & config, std::unique_lock &) +{ + auto named_collections = LoadFromConfig(config).getAll(); + LOG_TRACE( + &Poco::Logger::get("NamedCollectionsUtils"), + "Loaded {} collections from config", named_collections.size()); + + NamedCollectionFactory::instance().add(std::move(named_collections)); + is_loaded_from_config = true; +} + void loadFromConfig(const Poco::Util::AbstractConfiguration & config) { auto lock = lockNamedCollectionsTransaction(); - NamedCollectionFactory::instance().add(LoadFromConfig(config).getAll()); + loadFromConfigUnlocked(config, lock); } void reloadFromConfig(const Poco::Util::AbstractConfiguration & config) @@ -342,17 +356,47 @@ void reloadFromConfig(const Poco::Util::AbstractConfiguration & config) auto & instance = NamedCollectionFactory::instance(); instance.removeById(SourceId::CONFIG); instance.add(collections); + is_loaded_from_config = true; +} + +void loadFromSQLUnlocked(ContextPtr context, std::unique_lock &) +{ + auto named_collections = LoadFromSQL(context).getAll(); + LOG_TRACE( + &Poco::Logger::get("NamedCollectionsUtils"), + "Loaded {} collections from SQL", named_collections.size()); + + NamedCollectionFactory::instance().add(std::move(named_collections)); + is_loaded_from_sql = true; } void loadFromSQL(ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); - NamedCollectionFactory::instance().add(LoadFromSQL(context).getAll()); + loadFromSQLUnlocked(context, lock); +} + +void loadIfNotUnlocked(std::unique_lock & lock) +{ + auto global_context = Context::getGlobalContextInstance(); + if (!is_loaded_from_config) + loadFromConfigUnlocked(global_context->getConfigRef(), lock); + if (!is_loaded_from_sql) + loadFromSQLUnlocked(global_context, lock); +} + +void loadIfNot() +{ + if (is_loaded_from_sql && is_loaded_from_config) + return; + auto lock = lockNamedCollectionsTransaction(); + return loadIfNotUnlocked(lock); } void removeFromSQL(const std::string & collection_name, ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); + loadIfNotUnlocked(lock); LoadFromSQL(context).remove(collection_name); NamedCollectionFactory::instance().remove(collection_name); } @@ -360,6 +404,7 @@ void removeFromSQL(const std::string & collection_name, ContextPtr context) void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); + loadIfNotUnlocked(lock); LoadFromSQL(context).removeIfExists(collection_name); NamedCollectionFactory::instance().removeIfExists(collection_name); } @@ -367,12 +412,14 @@ void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr conte void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); + loadIfNotUnlocked(lock); NamedCollectionFactory::instance().add(query.collection_name, LoadFromSQL(context).create(query)); } void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); + loadIfNotUnlocked(lock); LoadFromSQL(context).update(query); auto collection = NamedCollectionFactory::instance().getMutable(query.collection_name); diff --git a/src/Common/NamedCollections/NamedCollectionUtils.h b/src/Common/NamedCollections/NamedCollectionUtils.h index 8befc9cac3c..c929abb5d74 100644 --- a/src/Common/NamedCollections/NamedCollectionUtils.h +++ b/src/Common/NamedCollections/NamedCollectionUtils.h @@ -35,6 +35,8 @@ void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr conte /// Update definition of already existing collection from AST and update result in `context->getPath() / named_collections /`. void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context); +void loadIfNot(); + } } diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index cefed555781..6c783beaecb 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -58,6 +58,8 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) if (asts.empty()) return nullptr; + NamedCollectionUtils::loadIfNot(); + auto collection = tryGetNamedCollectionFromASTs(asts); if (!collection) return nullptr; diff --git a/src/Storages/System/StorageSystemNamedCollections.cpp b/src/Storages/System/StorageSystemNamedCollections.cpp index bc1e3a45e6b..aa095f48179 100644 --- a/src/Storages/System/StorageSystemNamedCollections.cpp +++ b/src/Storages/System/StorageSystemNamedCollections.cpp @@ -31,6 +31,8 @@ void StorageSystemNamedCollections::fillData(MutableColumns & res_columns, Conte { context->checkAccess(AccessType::SHOW_NAMED_COLLECTIONS); + NamedCollectionUtils::loadIfNot(); + auto collections = NamedCollectionFactory::instance().getAll(); for (const auto & [name, collection] : collections) {