mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Load named collections on first access
This commit is contained in:
parent
3ba96590ca
commit
9d16205c8a
@ -37,7 +37,6 @@
|
|||||||
#include <AggregateFunctions/registerAggregateFunctions.h>
|
#include <AggregateFunctions/registerAggregateFunctions.h>
|
||||||
#include <TableFunctions/registerTableFunctions.h>
|
#include <TableFunctions/registerTableFunctions.h>
|
||||||
#include <Storages/registerStorages.h>
|
#include <Storages/registerStorages.h>
|
||||||
#include <Common/NamedCollections/NamedCollectionUtils.h>
|
|
||||||
#include <Dictionaries/registerDictionaries.h>
|
#include <Dictionaries/registerDictionaries.h>
|
||||||
#include <Disks/registerDisks.h>
|
#include <Disks/registerDisks.h>
|
||||||
#include <Formats/registerFormats.h>
|
#include <Formats/registerFormats.h>
|
||||||
@ -131,8 +130,6 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
|||||||
config().getUInt("max_io_thread_pool_size", 100),
|
config().getUInt("max_io_thread_pool_size", 100),
|
||||||
config().getUInt("max_io_thread_pool_free_size", 0),
|
config().getUInt("max_io_thread_pool_free_size", 0),
|
||||||
config().getUInt("io_thread_pool_queue_size", 10000));
|
config().getUInt("io_thread_pool_queue_size", 10000));
|
||||||
|
|
||||||
NamedCollectionUtils::loadFromConfig(config());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -224,8 +221,6 @@ void LocalServer::tryInitPath()
|
|||||||
|
|
||||||
global_context->setUserFilesPath(""); // user's files are everywhere
|
global_context->setUserFilesPath(""); // user's files are everywhere
|
||||||
|
|
||||||
NamedCollectionUtils::loadFromSQL(global_context);
|
|
||||||
|
|
||||||
/// top_level_domains_lists
|
/// top_level_domains_lists
|
||||||
const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/");
|
const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/");
|
||||||
if (!top_level_domains_path.empty())
|
if (!top_level_domains_path.empty())
|
||||||
|
@ -770,8 +770,6 @@ try
|
|||||||
config().getUInt("max_io_thread_pool_free_size", 0),
|
config().getUInt("max_io_thread_pool_free_size", 0),
|
||||||
config().getUInt("io_thread_pool_queue_size", 10000));
|
config().getUInt("io_thread_pool_queue_size", 10000));
|
||||||
|
|
||||||
NamedCollectionUtils::loadFromConfig(config());
|
|
||||||
|
|
||||||
/// Initialize global local cache for remote filesystem.
|
/// Initialize global local cache for remote filesystem.
|
||||||
if (config().has("local_cache_for_remote_fs"))
|
if (config().has("local_cache_for_remote_fs"))
|
||||||
{
|
{
|
||||||
@ -1177,8 +1175,6 @@ try
|
|||||||
SensitiveDataMasker::setInstance(std::make_unique<SensitiveDataMasker>(config(), "query_masking_rules"));
|
SensitiveDataMasker::setInstance(std::make_unique<SensitiveDataMasker>(config(), "query_masking_rules"));
|
||||||
}
|
}
|
||||||
|
|
||||||
NamedCollectionUtils::loadFromSQL(global_context);
|
|
||||||
|
|
||||||
auto main_config_reloader = std::make_unique<ConfigReloader>(
|
auto main_config_reloader = std::make_unique<ConfigReloader>(
|
||||||
config_path,
|
config_path,
|
||||||
include_from_path,
|
include_from_path,
|
||||||
|
@ -32,6 +32,9 @@ namespace ErrorCodes
|
|||||||
namespace NamedCollectionUtils
|
namespace NamedCollectionUtils
|
||||||
{
|
{
|
||||||
|
|
||||||
|
static std::atomic<bool> is_loaded_from_config = false;
|
||||||
|
static std::atomic<bool> is_loaded_from_sql = false;
|
||||||
|
|
||||||
class LoadFromConfig
|
class LoadFromConfig
|
||||||
{
|
{
|
||||||
private:
|
private:
|
||||||
@ -329,10 +332,21 @@ std::unique_lock<std::mutex> lockNamedCollectionsTransaction()
|
|||||||
return std::unique_lock(transaction_lock);
|
return std::unique_lock(transaction_lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void loadFromConfigUnlocked(const Poco::Util::AbstractConfiguration & config, std::unique_lock<std::mutex> &)
|
||||||
|
{
|
||||||
|
auto named_collections = LoadFromConfig(config).getAll();
|
||||||
|
LOG_TRACE(
|
||||||
|
&Poco::Logger::get("NamedCollectionsUtils"),
|
||||||
|
"Loaded {} collections from config", named_collections.size());
|
||||||
|
|
||||||
|
NamedCollectionFactory::instance().add(std::move(named_collections));
|
||||||
|
is_loaded_from_config = true;
|
||||||
|
}
|
||||||
|
|
||||||
void loadFromConfig(const Poco::Util::AbstractConfiguration & config)
|
void loadFromConfig(const Poco::Util::AbstractConfiguration & config)
|
||||||
{
|
{
|
||||||
auto lock = lockNamedCollectionsTransaction();
|
auto lock = lockNamedCollectionsTransaction();
|
||||||
NamedCollectionFactory::instance().add(LoadFromConfig(config).getAll());
|
loadFromConfigUnlocked(config, lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
void reloadFromConfig(const Poco::Util::AbstractConfiguration & config)
|
void reloadFromConfig(const Poco::Util::AbstractConfiguration & config)
|
||||||
@ -342,17 +356,47 @@ void reloadFromConfig(const Poco::Util::AbstractConfiguration & config)
|
|||||||
auto & instance = NamedCollectionFactory::instance();
|
auto & instance = NamedCollectionFactory::instance();
|
||||||
instance.removeById(SourceId::CONFIG);
|
instance.removeById(SourceId::CONFIG);
|
||||||
instance.add(collections);
|
instance.add(collections);
|
||||||
|
is_loaded_from_config = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void loadFromSQLUnlocked(ContextPtr context, std::unique_lock<std::mutex> &)
|
||||||
|
{
|
||||||
|
auto named_collections = LoadFromSQL(context).getAll();
|
||||||
|
LOG_TRACE(
|
||||||
|
&Poco::Logger::get("NamedCollectionsUtils"),
|
||||||
|
"Loaded {} collections from SQL", named_collections.size());
|
||||||
|
|
||||||
|
NamedCollectionFactory::instance().add(std::move(named_collections));
|
||||||
|
is_loaded_from_sql = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void loadFromSQL(ContextPtr context)
|
void loadFromSQL(ContextPtr context)
|
||||||
{
|
{
|
||||||
auto lock = lockNamedCollectionsTransaction();
|
auto lock = lockNamedCollectionsTransaction();
|
||||||
NamedCollectionFactory::instance().add(LoadFromSQL(context).getAll());
|
loadFromSQLUnlocked(context, lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
void loadIfNotUnlocked(std::unique_lock<std::mutex> & lock)
|
||||||
|
{
|
||||||
|
auto global_context = Context::getGlobalContextInstance();
|
||||||
|
if (!is_loaded_from_config)
|
||||||
|
loadFromConfigUnlocked(global_context->getConfigRef(), lock);
|
||||||
|
if (!is_loaded_from_sql)
|
||||||
|
loadFromSQLUnlocked(global_context, lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
void loadIfNot()
|
||||||
|
{
|
||||||
|
if (is_loaded_from_sql && is_loaded_from_config)
|
||||||
|
return;
|
||||||
|
auto lock = lockNamedCollectionsTransaction();
|
||||||
|
return loadIfNotUnlocked(lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
void removeFromSQL(const std::string & collection_name, ContextPtr context)
|
void removeFromSQL(const std::string & collection_name, ContextPtr context)
|
||||||
{
|
{
|
||||||
auto lock = lockNamedCollectionsTransaction();
|
auto lock = lockNamedCollectionsTransaction();
|
||||||
|
loadIfNotUnlocked(lock);
|
||||||
LoadFromSQL(context).remove(collection_name);
|
LoadFromSQL(context).remove(collection_name);
|
||||||
NamedCollectionFactory::instance().remove(collection_name);
|
NamedCollectionFactory::instance().remove(collection_name);
|
||||||
}
|
}
|
||||||
@ -360,6 +404,7 @@ void removeFromSQL(const std::string & collection_name, ContextPtr context)
|
|||||||
void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr context)
|
void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr context)
|
||||||
{
|
{
|
||||||
auto lock = lockNamedCollectionsTransaction();
|
auto lock = lockNamedCollectionsTransaction();
|
||||||
|
loadIfNotUnlocked(lock);
|
||||||
LoadFromSQL(context).removeIfExists(collection_name);
|
LoadFromSQL(context).removeIfExists(collection_name);
|
||||||
NamedCollectionFactory::instance().removeIfExists(collection_name);
|
NamedCollectionFactory::instance().removeIfExists(collection_name);
|
||||||
}
|
}
|
||||||
@ -367,12 +412,14 @@ void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr conte
|
|||||||
void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr context)
|
void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr context)
|
||||||
{
|
{
|
||||||
auto lock = lockNamedCollectionsTransaction();
|
auto lock = lockNamedCollectionsTransaction();
|
||||||
|
loadIfNotUnlocked(lock);
|
||||||
NamedCollectionFactory::instance().add(query.collection_name, LoadFromSQL(context).create(query));
|
NamedCollectionFactory::instance().add(query.collection_name, LoadFromSQL(context).create(query));
|
||||||
}
|
}
|
||||||
|
|
||||||
void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context)
|
void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context)
|
||||||
{
|
{
|
||||||
auto lock = lockNamedCollectionsTransaction();
|
auto lock = lockNamedCollectionsTransaction();
|
||||||
|
loadIfNotUnlocked(lock);
|
||||||
LoadFromSQL(context).update(query);
|
LoadFromSQL(context).update(query);
|
||||||
|
|
||||||
auto collection = NamedCollectionFactory::instance().getMutable(query.collection_name);
|
auto collection = NamedCollectionFactory::instance().getMutable(query.collection_name);
|
||||||
|
@ -35,6 +35,8 @@ void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr conte
|
|||||||
/// Update definition of already existing collection from AST and update result in `context->getPath() / named_collections /`.
|
/// Update definition of already existing collection from AST and update result in `context->getPath() / named_collections /`.
|
||||||
void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context);
|
void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context);
|
||||||
|
|
||||||
|
void loadIfNot();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -58,6 +58,8 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts)
|
|||||||
if (asts.empty())
|
if (asts.empty())
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
|
||||||
|
NamedCollectionUtils::loadIfNot();
|
||||||
|
|
||||||
auto collection = tryGetNamedCollectionFromASTs(asts);
|
auto collection = tryGetNamedCollectionFromASTs(asts);
|
||||||
if (!collection)
|
if (!collection)
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
@ -31,6 +31,8 @@ void StorageSystemNamedCollections::fillData(MutableColumns & res_columns, Conte
|
|||||||
{
|
{
|
||||||
context->checkAccess(AccessType::SHOW_NAMED_COLLECTIONS);
|
context->checkAccess(AccessType::SHOW_NAMED_COLLECTIONS);
|
||||||
|
|
||||||
|
NamedCollectionUtils::loadIfNot();
|
||||||
|
|
||||||
auto collections = NamedCollectionFactory::instance().getAll();
|
auto collections = NamedCollectionFactory::instance().getAll();
|
||||||
for (const auto & [name, collection] : collections)
|
for (const auto & [name, collection] : collections)
|
||||||
{
|
{
|
||||||
|
Loading…
Reference in New Issue
Block a user