thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 16:50:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8458 from vitlibar/separate-variables-for-database-and-name

Browse Source 
Separate variables for database and name in dictionaries.
This commit is contained in:
alexey-milovidov 2019-12-31 07:39:01 +03:00 committed by GitHub
commit 17b413fd2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 494 additions and 365 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dbms/programs/server/Server.cpp
View File

@ -947,6 +947,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
});
/// try to load dictionaries immediately, throw on error and die
ext::scope_guard dictionaries_xmls, models_xmls;
try
{
if (!config().getBool("dictionaries_lazy_load", true))
@ -954,12 +955,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->tryCreateEmbeddedDictionaries();
global_context->getExternalDictionariesLoader().enableAlwaysLoadEverything(true);
}
auto dictionaries_repository = std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "dictionaries_config");
global_context->getExternalDictionariesLoader().addConfigRepository("", std::move(dictionaries_repository));
auto models_repository = std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "models_config");
global_context->getExternalModelsLoader().addConfigRepository("", std::move(models_repository));
dictionaries_xmls = global_context->getExternalDictionariesLoader().addConfigRepository(
std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "dictionaries_config"));
models_xmls = global_context->getExternalModelsLoader().addConfigRepository(
std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "models_config"));
}
catch (...)
{

8
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -10,8 +10,6 @@
#include <IO/WriteHelpers.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Storages/StorageFactory.h>
@ -181,12 +179,8 @@ void DatabaseOrdinary::loadStoredObjects(
/// After all tables was basically initialized, startup them.
startupTables(pool);
/// Add database as repository
auto dictionaries_repository = std::make_unique<ExternalLoaderDatabaseConfigRepository>(shared_from_this(), context);
auto & external_loader = context.getExternalDictionariesLoader();
external_loader.addConfigRepository(getDatabaseName(), std::move(dictionaries_repository));
/// Attach dictionaries.
attachToExternalDictionariesLoader(context);
for (const auto & name_with_query : file_names)
{
auto create_query = name_with_query.second->as<const ASTCreateQuery &>();

37
dbms/src/Databases/DatabaseWithDictionaries.cpp
View File

@ -1,6 +1,7 @@
#include <Databases/DatabaseWithDictionaries.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderPresetConfigRepository.h>
#include <Interpreters/ExternalLoaderTempConfigRepository.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Interpreters/Context.h>
#include <Storages/StorageDictionary.h>
@ -74,7 +75,7 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S
/// A dictionary with the same full name could be defined in *.xml config files.
String full_name = getDatabaseName() + "." + dictionary_name;
auto & external_loader = const_cast<ExternalDictionariesLoader &>(context.getExternalDictionariesLoader());
const auto & external_loader = context.getExternalDictionariesLoader();
if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST)
throw Exception(
"Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.",
@ -106,15 +107,10 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S
/// Add a temporary repository containing the dictionary.
/// We need this temp repository to try loading the dictionary before actually attaching it to the database.
static std::atomic<size_t> counter = 0;
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
+ std::to_string(++counter);
external_loader.addConfigRepository(
temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path,
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), getDatabaseName())}}));
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
auto temp_repository
= const_cast<ExternalDictionariesLoader &>(external_loader) /// the change of ExternalDictionariesLoader is temporary
.addConfigRepository(std::make_unique<ExternalLoaderTempConfigRepository>(
getDatabaseName(), dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>())));
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
if (!lazy_load)
@ -253,4 +249,23 @@ ASTPtr DatabaseWithDictionaries::getCreateDictionaryQueryImpl(
return ast;
}
void DatabaseWithDictionaries::shutdown()
{
detachFromExternalDictionariesLoader();
DatabaseOnDisk::shutdown();
}
DatabaseWithDictionaries::~DatabaseWithDictionaries() = default;
void DatabaseWithDictionaries::attachToExternalDictionariesLoader(Context & context)
{
database_as_config_repo_for_external_loader = context.getExternalDictionariesLoader().addConfigRepository(
std::make_unique<ExternalLoaderDatabaseConfigRepository>(*this, context));
}
void DatabaseWithDictionaries::detachFromExternalDictionariesLoader()
{
database_as_config_repo_for_external_loader = {};
}
}

11
dbms/src/Databases/DatabaseWithDictionaries.h
View File

@ -1,4 +1,5 @@
#include <Databases/DatabaseOnDisk.h>
#include <ext/scope_guard.h>
namespace DB
{
@ -25,15 +26,25 @@ public:
bool isDictionaryExist(const Context & context, const String & dictionary_name) const override;
void shutdown() override;
~DatabaseWithDictionaries() override;
protected:
DatabaseWithDictionaries(const String & name, const String & metadata_path_, const String & logger)
: DatabaseOnDisk(name, metadata_path_, logger) {}
void attachToExternalDictionariesLoader(Context & context);
void detachFromExternalDictionariesLoader();
StoragePtr getDictionaryStorage(const Context & context, const String & table_name) const;
ASTPtr getCreateDictionaryQueryImpl(const Context & context,
const String & dictionary_name,
bool throw_on_error) const override;
private:
ext::scope_guard database_as_config_repo_for_external_loader;
};
}

29
dbms/src/Dictionaries/CacheDictionary.cpp
View File

@ -57,12 +57,15 @@ inline size_t CacheDictionary::getCellIdx(const Key id) const
CacheDictionary::CacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -73,7 +76,7 @@ CacheDictionary::CacheDictionary(
, rnd_engine(randomSeed())
{
if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
throw Exception{full_name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes();
}
@ -204,7 +207,7 @@ void CacheDictionary::isInConstantVector(const Key child_id, const PaddedPODArra
void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -215,7 +218,7 @@ void CacheDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, ids, out, [&](const size_t row) { return def->getDataAt(row); });
}
@ -224,7 +227,7 @@ void CacheDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, ids, out, [&](const size_t) { return StringRef{def}; });
}
@ -352,7 +355,7 @@ void CacheDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -539,7 +542,7 @@ CacheDictionary::Attribute & CacheDictionary::getAttribute(const std::string & a
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -580,7 +583,7 @@ std::exception_ptr CacheDictionary::getLastException() const
void registerDictionaryCache(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -590,22 +593,24 @@ void registerDictionaryCache(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const auto & layout_prefix = config_prefix + ".layout";
const auto size = config.getInt(layout_prefix + ".cache.size_in_cells");
if (size == 0)
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
if (require_nonempty)
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
return std::make_unique<CacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
return std::make_unique<CacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
};
factory.registerLayout("cache", create_layout, false);
}

9
dbms/src/Dictionaries/CacheDictionary.h
View File

@ -25,13 +25,16 @@ class CacheDictionary final : public IDictionary
{
public:
CacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Cache"; }
@ -52,7 +55,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<CacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size);
return std::make_shared<CacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -254,7 +257,9 @@ private:
template <typename AncestorType>
void isInImpl(const PaddedPODArray<Key> & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
mutable DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

2
dbms/src/Dictionaries/CacheDictionary.inc.h
View File

@ -333,7 +333,7 @@ void CacheDictionary::update(
last_exception = std::current_exception();
backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count));
tryLogException(last_exception, log, "Could not update cache dictionary '" + getName() +
tryLogException(last_exception, log, "Could not update cache dictionary '" + getFullName() +
"', next update is scheduled at " + ext::to_string(backoff_end_time));
}
}

27
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
View File

@ -51,12 +51,15 @@ inline UInt64 ComplexKeyCacheDictionary::getCellIdx(const StringRef key) const
ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -65,7 +68,7 @@ ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
, rnd_engine(randomSeed())
{
if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
throw Exception{full_name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes();
}
@ -77,7 +80,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -94,7 +97,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, key_columns, out, [&](const size_t row) { return def->getDataAt(row); });
}
@ -109,7 +112,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, key_columns, out, [&](const size_t) { return StringRef{def}; });
}
@ -249,7 +252,7 @@ void ComplexKeyCacheDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
}
}
@ -258,7 +261,7 @@ ComplexKeyCacheDictionary::Attribute & ComplexKeyCacheDictionary::getAttribute(c
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -394,7 +397,7 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -405,15 +408,17 @@ void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
const auto & layout_prefix = config_prefix + ".layout";
const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells");
if (size == 0)
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
if (require_nonempty)
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
return std::make_unique<ComplexKeyCacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
return std::make_unique<ComplexKeyCacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
};
factory.registerLayout("complex_key_cache", create_layout, true);
}

9
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
View File

@ -42,6 +42,7 @@ class ComplexKeyCacheDictionary final : public IDictionaryBase
{
public:
ComplexKeyCacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -50,7 +51,9 @@ public:
std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "ComplexKeyCache"; }
@ -75,7 +78,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<ComplexKeyCacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size);
return std::make_shared<ComplexKeyCacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -668,7 +671,9 @@ private:
bool isEmptyCell(const UInt64 idx) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

30
dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
View File

@ -15,13 +15,16 @@ namespace ErrorCodes
}
ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -40,7 +43,7 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -72,7 +75,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -94,7 +97,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, \
@ -128,7 +131,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -148,7 +151,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -179,7 +182,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -256,7 +259,7 @@ void ComplexKeyHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
}
}
@ -397,7 +400,7 @@ void ComplexKeyHashedDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -630,7 +633,7 @@ const ComplexKeyHashedDictionary::Attribute & ComplexKeyHashedDictionary::getAtt
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -742,7 +745,7 @@ BlockInputStreamPtr ComplexKeyHashedDictionary::getBlockInputStream(const Names
void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string &,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -751,12 +754,13 @@ void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
if (!dict_struct.key)
throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<ComplexKeyHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<ComplexKeyHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("complex_key_hashed", create_layout, true);
}
}

9
dbms/src/Dictionaries/ComplexKeyHashedDictionary.h
View File

@ -23,6 +23,7 @@ class ComplexKeyHashedDictionary final : public IDictionaryBase
{
public:
ComplexKeyHashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -32,7 +33,9 @@ public:
std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "ComplexKeyHashed"; }
@ -48,7 +51,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<ComplexKeyHashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
return std::make_shared<ComplexKeyHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -233,7 +236,9 @@ private:
template <typename T>
std::vector<StringRef> getKeys(const Attribute & attribute) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

34
dbms/src/Dictionaries/FlatDictionary.cpp
View File

@ -21,13 +21,16 @@ static const auto max_array_size = 500000;
FlatDictionary::FlatDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -107,7 +110,7 @@ void FlatDictionary::isInConstantVector(const Key child_id, const PaddedPODArray
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -133,7 +136,7 @@ DECLARE(Decimal128)
void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = std::get<StringRef>(attribute.null_values);
@ -152,7 +155,7 @@ void FlatDictionary::getString(const std::string & attribute_name, const PaddedP
ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
@ -177,7 +180,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -191,7 +194,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -216,7 +219,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
FlatDictionary::getItemsImpl<StringRef, StringRef>(
attribute,
@ -297,7 +300,7 @@ void FlatDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -404,7 +407,7 @@ void FlatDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
@ -578,7 +581,7 @@ template <typename T>
void FlatDictionary::resize(Attribute & attribute, const Key id)
{
if (id >= max_array_size)
throw Exception{name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
throw Exception{full_name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
auto & array = std::get<ContainerType<T>>(attribute.arrays);
if (id >= array.size())
@ -666,7 +669,7 @@ const FlatDictionary::Attribute & FlatDictionary::getAttribute(const std::string
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -706,7 +709,7 @@ BlockInputStreamPtr FlatDictionary::getBlockInputStream(const Names & column_nam
void registerDictionaryFlat(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -716,13 +719,16 @@ void registerDictionaryFlat(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'flat'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<FlatDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<FlatDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("flat", create_layout, false);
}

9
dbms/src/Dictionaries/FlatDictionary.h
View File

@ -22,6 +22,7 @@ class FlatDictionary final : public IDictionary
{
public:
FlatDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -29,7 +30,9 @@ public:
bool require_nonempty_,
BlockPtr saved_block_ = nullptr);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Flat"; }
@ -45,7 +48,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<FlatDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
return std::make_shared<FlatDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -222,7 +225,9 @@ private:
PaddedPODArray<Key> getIds() const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

41
dbms/src/Dictionaries/HashedDictionary.cpp
View File

@ -31,6 +31,7 @@ namespace ErrorCodes
HashedDictionary::HashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -38,7 +39,9 @@ HashedDictionary::HashedDictionary(
bool require_nonempty_,
bool sparse_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -129,7 +132,7 @@ void HashedDictionary::isInConstantVector(const Key child_id, const PaddedPODArr
const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -155,7 +158,7 @@ DECLARE(Decimal128)
void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -174,7 +177,7 @@ void HashedDictionary::getString(const std::string & attribute_name, const Padde
ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
@ -199,7 +202,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -213,7 +216,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -238,7 +241,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -317,7 +320,7 @@ void HashedDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -424,7 +427,7 @@ void HashedDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -684,7 +687,7 @@ const HashedDictionary::Attribute & HashedDictionary::getAttribute(const std::st
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -768,27 +771,31 @@ BlockInputStreamPtr HashedDictionary::getBlockInputStream(const Names & column_n
void registerDictionaryHashed(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
DictionarySourcePtr source_ptr) -> DictionaryPtr
DictionarySourcePtr source_ptr,
bool sparse) -> DictionaryPtr
{
if (dict_struct.key)
throw Exception{"'key' is not supported for dictionary of layout 'hashed'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
const bool sparse = name == "sparse_hashed";
return std::make_unique<HashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
return std::make_unique<HashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
};
factory.registerLayout("hashed", create_layout, false);
factory.registerLayout("sparse_hashed", create_layout, false);
using namespace std::placeholders;
factory.registerLayout("hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ false), false);
factory.registerLayout("sparse_hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ true), false);
}
}

9
dbms/src/Dictionaries/HashedDictionary.h
View File

@ -26,6 +26,7 @@ class HashedDictionary final : public IDictionary
{
public:
HashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -34,7 +35,9 @@ public:
bool sparse_,
BlockPtr saved_block_ = nullptr);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return sparse ? "SparseHashed" : "Hashed"; }
@ -50,7 +53,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<HashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
return std::make_shared<HashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -262,7 +265,9 @@ private:
template <typename ChildType, typename AncestorType>
void isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

5
dbms/src/Dictionaries/IDictionary.h
View File

@ -25,6 +25,11 @@ struct IDictionaryBase : public IExternalLoadable
{
using Key = UInt64;
virtual const std::string & getDatabase() const = 0;
virtual const std::string & getName() const = 0;
virtual const std::string & getFullName() const = 0;
const std::string & getLoadableName() const override { return getFullName(); }
virtual std::string getTypeName() const = 0;
virtual size_t getBytesAllocated() const = 0;

21
dbms/src/Dictionaries/RangeHashedDictionary.cpp
View File

@ -68,12 +68,15 @@ static bool operator<(const RangeHashedDictionary::Range & left, const RangeHash
RangeHashedDictionary::RangeHashedDictionary(
const std::string & dictionary_name_,
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_)
: dictionary_name{dictionary_name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -156,7 +159,7 @@ void RangeHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{dictionary_name + ": hierarchical attributes not supported by " + getName() + " dictionary.",
throw Exception{full_name + ": hierarchical attributes not supported by " + getName() + " dictionary.",
ErrorCodes::BAD_ARGUMENTS};
}
}
@ -207,7 +210,7 @@ void RangeHashedDictionary::loadData()
stream->readSuffix();
if (require_nonempty && 0 == element_count)
throw Exception{dictionary_name + ": dictionary source is empty and 'require_nonempty' property is set.",
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.",
ErrorCodes::DICTIONARY_IS_EMPTY};
}
@ -520,7 +523,7 @@ const RangeHashedDictionary::Attribute & RangeHashedDictionary::getAttribute(con
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{dictionary_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -674,7 +677,7 @@ BlockInputStreamPtr RangeHashedDictionary::getBlockInputStream(const Names & col
void registerDictionaryRangeHashed(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -684,12 +687,14 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'range_hashed'", ErrorCodes::UNSUPPORTED_METHOD};
if (!dict_struct.range_min || !dict_struct.range_max)
throw Exception{name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max",
throw Exception{full_name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<RangeHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<RangeHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("range_hashed", create_layout, false);
}

13
dbms/src/Dictionaries/RangeHashedDictionary.h
View File

@ -18,13 +18,16 @@ class RangeHashedDictionary final : public IDictionaryBase
{
public:
RangeHashedDictionary(
const std::string & dictionary_name_,
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_);
std::string getName() const override { return dictionary_name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "RangeHashed"; }
@ -40,7 +43,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<RangeHashedDictionary>(dictionary_name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
return std::make_shared<RangeHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -208,7 +211,9 @@ private:
friend struct RangeHashedDIctionaryCallGetBlockInputStreamImpl;
const std::string dictionary_name;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

29
dbms/src/Dictionaries/TrieDictionary.cpp
View File

@ -35,12 +35,15 @@ namespace ErrorCodes
}
TrieDictionary::TrieDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -75,7 +78,7 @@ TrieDictionary::~TrieDictionary()
validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -107,7 +110,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -129,7 +132,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, \
@ -163,7 +166,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -183,7 +186,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -214,7 +217,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -291,7 +294,7 @@ void TrieDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
}
}
@ -337,7 +340,7 @@ void TrieDictionary::loadData()
stream->readSuffix();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -627,7 +630,7 @@ const TrieDictionary::Attribute & TrieDictionary::getAttribute(const std::string
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -767,7 +770,7 @@ BlockInputStreamPtr TrieDictionary::getBlockInputStream(const Names & column_nam
void registerDictionaryTrie(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string &,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -776,10 +779,12 @@ void registerDictionaryTrie(DictionaryFactory & factory)
if (!dict_struct.key)
throw Exception{"'key' is required for dictionary of layout 'ip_trie'", ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
// This is specialised trie for storing IPv4 and IPv6 prefixes.
return std::make_unique<TrieDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<TrieDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("ip_trie", create_layout, true);
}

9
dbms/src/Dictionaries/TrieDictionary.h
View File

@ -23,6 +23,7 @@ class TrieDictionary final : public IDictionaryBase
{
public:
TrieDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -33,7 +34,9 @@ public:
std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Trie"; }
@ -49,7 +52,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<TrieDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
return std::make_shared<TrieDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -232,7 +235,9 @@ private:
Columns getKeyColumns() const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

10
dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
View File

@ -421,7 +421,7 @@ void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs
}
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name)
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query)
{
checkAST(query);
@ -434,10 +434,14 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer
AutoPtr<Poco::XML::Element> name_element(xml_document->createElement("name"));
current_dictionary->appendChild(name_element);
String full_name = (!database_name.empty() ? database_name : query.database) + "." + query.table;
AutoPtr<Text> name(xml_document->createTextNode(full_name));
AutoPtr<Text> name(xml_document->createTextNode(query.table));
name_element->appendChild(name);
AutoPtr<Poco::XML::Element> database_element(xml_document->createElement("database"));
current_dictionary->appendChild(database_element);
AutoPtr<Text> database(xml_document->createTextNode(query.database));
database_element->appendChild(database);
AutoPtr<Element> structure_element(xml_document->createElement("structure"));
current_dictionary->appendChild(structure_element);
Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key);

2
dbms/src/Dictionaries/getDictionaryConfigurationFromAST.h
View File

@ -10,6 +10,6 @@ using DictionaryConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfigurati
/// Convert dictionary AST to Poco::AbstractConfiguration
/// This function is necessary because all loadable objects configuration are Poco::AbstractConfiguration
/// Can throw exception if query is ill-formed
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name = {});
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query);
}

3
dbms/src/Dictionaries/tests/gtest_dictionary_configuration.cpp
View File

@ -57,7 +57,8 @@ TEST(ConvertDictionaryAST, SimpleDictConfiguration)
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create);
/// name
EXPECT_EQ(config->getString("dictionary.name"), "test.dict1");
EXPECT_EQ(config->getString("dictionary.database"), "test");
EXPECT_EQ(config->getString("dictionary.name"), "dict1");
/// lifetime
EXPECT_EQ(config->getInt("dictionary.lifetime.min"), 1);

12
dbms/src/Functions/FunctionsExternalDictionaries.h
View File

@ -127,10 +127,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
if (!executeDispatchSimple<FlatDictionary>(block, arguments, result, dict_ptr) &&
@ -302,10 +302,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
@ -488,10 +488,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&

4
dbms/src/Interpreters/CatBoostModel.h
View File

@ -60,7 +60,7 @@ public:
const ExternalLoadableLifetime & getLifetime() const override;
std::string getName() const override { return name; }
const std::string & getLoadableName() const override { return name; }
bool supportUpdates() const override { return true; }
@ -69,7 +69,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override;
private:
std::string name;
const std::string name;
std::string model_path;
std::string lib_path;
ExternalLoadableLifetime lifetime;

3
dbms/src/Interpreters/Context.cpp
View File

@ -33,8 +33,6 @@
#include <Interpreters/UsersManager.h>
#include <Dictionaries/Embedded/GeoDictionariesLoader.h>
#include <Interpreters/EmbeddedDictionaries.h>
#include <Interpreters/ExternalLoaderXMLConfigRepository.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalModelsLoader.h>
#include <Interpreters/ExpressionActions.h>
@ -1088,7 +1086,6 @@ DatabasePtr Context::detachDatabase(const String & database_name)
{
auto lock = getLock();
auto res = getDatabase(database_name);
getExternalDictionariesLoader().removeConfigRepository(database_name);
shared->databases.erase(database_name);
return res;

8
dbms/src/Interpreters/ExternalDictionariesLoader.cpp
View File

@ -9,6 +9,7 @@ ExternalDictionariesLoader::ExternalDictionariesLoader(Context & context_)
: ExternalLoader("external dictionary", &Logger::get("ExternalDictionariesLoader"))
, context(context_)
{
setConfigSettings({"dictionary", "name", "database"});
enableAsyncLoading(true);
enablePeriodicUpdates(true);
}
@ -23,11 +24,4 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create(
bool dictionary_from_database = !repository_name.empty();
return DictionaryFactory::instance().create(name, config, key_in_config, context, dictionary_from_database);
}
void ExternalDictionariesLoader::addConfigRepository(
const std::string & repository_name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
{
ExternalLoader::addConfigRepository(repository_name, std::move(config_repository), {"dictionary", "name"});
}
}

4
dbms/src/Interpreters/ExternalDictionariesLoader.h
View File

@ -29,10 +29,6 @@ public:
return std::static_pointer_cast<const IDictionaryBase>(tryLoad(name));
}
void addConfigRepository(
const std::string & repository_name,
std::unique_ptr<IExternalLoaderConfigRepository> config_repository);
protected:
LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config,
const std::string & key_in_config, const std::string & repository_name) const override;

97
dbms/src/Interpreters/ExternalLoader.cpp
View File

@ -92,6 +92,7 @@ struct ExternalLoader::ObjectConfig
Poco::AutoPtr<Poco::Util::AbstractConfiguration> config;
String key_in_config;
String repository_name;
bool from_temp_repository = false;
String path;
};
@ -107,26 +108,30 @@ public:
}
~LoadablesConfigReader() = default;
using RepositoryPtr = std::unique_ptr<IExternalLoaderConfigRepository>;
using Repository = IExternalLoaderConfigRepository;
void addConfigRepository(const String & repository_name, RepositoryPtr repository, const ExternalLoaderConfigSettings & settings)
void addConfigRepository(std::unique_ptr<Repository> repository)
{
std::lock_guard lock{mutex};
RepositoryInfo repository_info{std::move(repository), settings, {}};
repositories.emplace(repository_name, std::move(repository_info));
auto * ptr = repository.get();
repositories.emplace(ptr, RepositoryInfo{std::move(repository), {}});
need_collect_object_configs = true;
}
RepositoryPtr removeConfigRepository(const String & repository_name)
void removeConfigRepository(Repository * repository)
{
std::lock_guard lock{mutex};
auto it = repositories.find(repository_name);
auto it = repositories.find(repository);
if (it == repositories.end())
return nullptr;
auto repository = std::move(it->second.repository);
return;
repositories.erase(it);
need_collect_object_configs = true;
return repository;
}
void setConfigSettings(const ExternalLoaderConfigSettings & settings_)
{
std::lock_guard lock{mutex};
settings = settings_;
}
using ObjectConfigsPtr = std::shared_ptr<const std::unordered_map<String /* object's name */, ObjectConfig>>;
@ -170,8 +175,7 @@ private:
struct RepositoryInfo
{
RepositoryPtr repository;
ExternalLoaderConfigSettings settings;
std::unique_ptr<Repository> repository;
std::unordered_map<String /* path */, FileInfo> files;
};
@ -179,18 +183,10 @@ private:
/// Checks last modification times of files and read those files which are new or changed.
void readRepositories(const std::optional<String> & only_repository_name = {}, const std::optional<String> & only_path = {})
{
Strings repository_names;
if (only_repository_name)
for (auto & [repository, repository_info] : repositories)
{
if (repositories.count(*only_repository_name))
repository_names.push_back(*only_repository_name);
}
else
boost::copy(repositories | boost::adaptors::map_keys, std::back_inserter(repository_names));
for (const auto & repository_name : repository_names)
{
auto & repository_info = repositories[repository_name];
if (only_repository_name && (repository->getName() != *only_repository_name))
continue;
for (auto & file_info : repository_info.files | boost::adaptors::map_values)
file_info.in_use = false;
@ -198,11 +194,11 @@ private:
Strings existing_paths;
if (only_path)
{
if (repository_info.repository->exists(*only_path))
if (repository->exists(*only_path))
existing_paths.push_back(*only_path);
}
else
boost::copy(repository_info.repository->getAllLoadablesDefinitionNames(), std::back_inserter(existing_paths));
boost::copy(repository->getAllLoadablesDefinitionNames(), std::back_inserter(existing_paths));
for (const auto & path : existing_paths)
{
@ -210,13 +206,13 @@ private:
if (it != repository_info.files.end())
{
FileInfo & file_info = it->second;
if (readFileInfo(file_info, *repository_info.repository, path, repository_info.settings))
if (readFileInfo(file_info, *repository, path))
need_collect_object_configs = true;
}
else
{
FileInfo file_info;
if (readFileInfo(file_info, *repository_info.repository, path, repository_info.settings))
if (readFileInfo(file_info, *repository, path))
{
repository_info.files.emplace(path, std::move(file_info));
need_collect_object_configs = true;
@ -249,8 +245,7 @@ private:
bool readFileInfo(
FileInfo & file_info,
IExternalLoaderConfigRepository & repository,
const String & path,
const ExternalLoaderConfigSettings & settings) const
const String & path) const
{
try
{
@ -293,7 +288,13 @@ private:
continue;
}
object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}});
String database;
if (!settings.external_database.empty())
database = file_contents->getString(key + "." + settings.external_database, "");
if (!database.empty())
object_name = database + "." + object_name;
object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}, {}});
}
file_info.objects = std::move(object_configs_from_file);
@ -318,7 +319,7 @@ private:
// Generate new result.
auto new_configs = std::make_shared<std::unordered_map<String /* object's name */, ObjectConfig>>();
for (const auto & [repository_name, repository_info] : repositories)
for (const auto & [repository, repository_info] : repositories)
{
for (const auto & [path, file_info] : repository_info.files)
{
@ -328,19 +329,19 @@ private:
if (already_added_it == new_configs->end())
{
auto & new_config = new_configs->emplace(object_name, object_config).first->second;
new_config.repository_name = repository_name;
new_config.from_temp_repository = repository->isTemporary();
new_config.repository_name = repository->getName();
new_config.path = path;
}
else
{
const auto & already_added = already_added_it->second;
if (!startsWith(repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) &&
!startsWith(already_added.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX))
if (!already_added.from_temp_repository && !repository->isTemporary())
{
LOG_WARNING(
log,
type_name << " '" << object_name << "' is found "
<< (((path == already_added.path) && repository_name == already_added.repository_name)
<< (((path == already_added.path) && (repository->getName() == already_added.repository_name))
? ("twice in the same file '" + path + "'")
: ("both in file '" + already_added.path + "' and '" + path + "'")));
}
@ -356,7 +357,8 @@ private:
Logger * log;
std::mutex mutex;
std::unordered_map<String, RepositoryInfo> repositories;
ExternalLoaderConfigSettings settings;
std::unordered_map<Repository *, RepositoryInfo> repositories;
ObjectConfigsPtr object_configs;
bool need_collect_object_configs = false;
};
@ -613,7 +615,7 @@ public:
}
catch (...)
{
tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getName() + "' was modified");
tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getLoadableName() + "' was modified");
/// Cannot check isModified, so update
should_update_flag = true;
}
@ -1151,20 +1153,23 @@ ExternalLoader::ExternalLoader(const String & type_name_, Logger * log_)
ExternalLoader::~ExternalLoader() = default;
void ExternalLoader::addConfigRepository(
const std::string & repository_name,
std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
const ExternalLoaderConfigSettings & config_settings)
ext::scope_guard ExternalLoader::addConfigRepository(std::unique_ptr<IExternalLoaderConfigRepository> repository)
{
config_files_reader->addConfigRepository(repository_name, std::move(config_repository), config_settings);
reloadConfig(repository_name);
auto * ptr = repository.get();
String name = ptr->getName();
config_files_reader->addConfigRepository(std::move(repository));
reloadConfig(name);
return [this, ptr, name]()
{
config_files_reader->removeConfigRepository(ptr);
reloadConfig(name);
};
}
std::unique_ptr<IExternalLoaderConfigRepository> ExternalLoader::removeConfigRepository(const std::string & repository_name)
void ExternalLoader::setConfigSettings(const ExternalLoaderConfigSettings & settings)
{
auto repository = config_files_reader->removeConfigRepository(repository_name);
reloadConfig(repository_name);
return repository;
config_files_reader->setConfigSettings(settings);
}
void ExternalLoader::enableAlwaysLoadEverything(bool enable)

11
dbms/src/Interpreters/ExternalLoader.h
View File

@ -7,6 +7,7 @@
#include <Interpreters/IExternalLoadable.h>
#include <Interpreters/IExternalLoaderConfigRepository.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
namespace DB
@ -24,9 +25,9 @@ struct ExternalLoaderConfigSettings
{
std::string external_config;
std::string external_name;
std::string external_database;
};
/** Interface for manage user-defined objects.
* Monitors configuration file and automatically reloads objects in separate threads.
* The monitoring thread wakes up every 'check_period_sec' seconds and checks
@ -87,13 +88,9 @@ public:
virtual ~ExternalLoader();
/// Adds a repository which will be used to read configurations from.
void addConfigRepository(
const std::string & repository_name,
std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
const ExternalLoaderConfigSettings & config_settings);
ext::scope_guard addConfigRepository(std::unique_ptr<IExternalLoaderConfigRepository> config_repository);
/// Removes a repository which were used to read configurations.
std::unique_ptr<IExternalLoaderConfigRepository> removeConfigRepository(const std::string & repository_name);
void setConfigSettings(const ExternalLoaderConfigSettings & settings_);
/// Sets whether all the objects from the configuration should be always loaded (even those which are never used).
void enableAlwaysLoadEverything(bool enable);

33
dbms/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp
View File

@ -1,4 +1,5 @@
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
namespace DB
@ -11,40 +12,46 @@ namespace ErrorCodes
namespace
{
String trimDatabaseName(const std::string & loadable_definition_name, const DatabasePtr database)
String trimDatabaseName(const std::string & loadable_definition_name, const IDatabase & database)
{
const auto & dbname = database->getDatabaseName();
const auto & dbname = database.getDatabaseName();
if (!startsWith(loadable_definition_name, dbname))
throw Exception(
"Loadable '" + loadable_definition_name + "' is not from database '" + database->getDatabaseName(), ErrorCodes::UNKNOWN_DICTIONARY);
"Loadable '" + loadable_definition_name + "' is not from database '" + database.getDatabaseName(), ErrorCodes::UNKNOWN_DICTIONARY);
/// dbname.loadable_name
///--> remove <---
return loadable_definition_name.substr(dbname.length() + 1);
}
}
LoadablesConfigurationPtr ExternalLoaderDatabaseConfigRepository::load(const std::string & loadable_definition_name) const
ExternalLoaderDatabaseConfigRepository::ExternalLoaderDatabaseConfigRepository(IDatabase & database_, const Context & context_)
: name(database_.getDatabaseName())
, database(database_)
, context(context_)
{
String dictname = trimDatabaseName(loadable_definition_name, database);
return getDictionaryConfigurationFromAST(database->getCreateDictionaryQuery(context, dictname)->as<const ASTCreateQuery &>());
}
bool ExternalLoaderDatabaseConfigRepository::exists(const std::string & loadable_definition_name) const
LoadablesConfigurationPtr ExternalLoaderDatabaseConfigRepository::load(const std::string & loadable_definition_name)
{
return database->isDictionaryExist(
context, trimDatabaseName(loadable_definition_name, database));
String dictname = trimDatabaseName(loadable_definition_name, database);
return getDictionaryConfigurationFromAST(database.getCreateDictionaryQuery(context, dictname)->as<const ASTCreateQuery &>());
}
bool ExternalLoaderDatabaseConfigRepository::exists(const std::string & loadable_definition_name)
{
return database.isDictionaryExist(context, trimDatabaseName(loadable_definition_name, database));
}
Poco::Timestamp ExternalLoaderDatabaseConfigRepository::getUpdateTime(const std::string & loadable_definition_name)
{
return database->getObjectMetadataModificationTime(trimDatabaseName(loadable_definition_name, database));
return database.getObjectMetadataModificationTime(trimDatabaseName(loadable_definition_name, database));
}
std::set<std::string> ExternalLoaderDatabaseConfigRepository::getAllLoadablesDefinitionNames() const
std::set<std::string> ExternalLoaderDatabaseConfigRepository::getAllLoadablesDefinitionNames()
{
std::set<std::string> result;
const auto & dbname = database->getDatabaseName();
auto itr = database->getDictionariesIterator(context);
const auto & dbname = database.getDatabaseName();
auto itr = database.getDictionariesIterator(context);
while (itr && itr->isValid())
{
result.insert(dbname + "." + itr->name());

17
dbms/src/Interpreters/ExternalLoaderDatabaseConfigRepository.h
View File

@ -12,22 +12,21 @@ namespace DB
class ExternalLoaderDatabaseConfigRepository : public IExternalLoaderConfigRepository
{
public:
ExternalLoaderDatabaseConfigRepository(const DatabasePtr & database_, const Context & context_)
: database(database_)
, context(context_)
{
}
ExternalLoaderDatabaseConfigRepository(IDatabase & database_, const Context & context_);
std::set<std::string> getAllLoadablesDefinitionNames() const override;
const std::string & getName() const override { return name; }
bool exists(const std::string & loadable_definition_name) const override;
std::set<std::string> getAllLoadablesDefinitionNames() override;
bool exists(const std::string & loadable_definition_name) override;
Poco::Timestamp getUpdateTime(const std::string & loadable_definition_name) override;
LoadablesConfigurationPtr load(const std::string & loadable_definition_name) const override;
LoadablesConfigurationPtr load(const std::string & loadable_definition_name) override;
private:
DatabasePtr database;
const String name;
IDatabase & database;
Context context;
};

49
dbms/src/Interpreters/ExternalLoaderPresetConfigRepository.cpp
View File

@ -1,49 +0,0 @@
#include <Interpreters/ExternalLoaderPresetConfigRepository.h>
#include <Common/Exception.h>
#include <boost/range/algorithm/copy.hpp>
#include <boost/range/adaptor/map.hpp>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
ExternalLoaderPresetConfigRepository::ExternalLoaderPresetConfigRepository(const std::vector<std::pair<String, LoadablesConfigurationPtr>> & preset_)
{
boost::range::copy(preset_, std::inserter(preset, preset.end()));
}
ExternalLoaderPresetConfigRepository::~ExternalLoaderPresetConfigRepository() = default;
std::set<String> ExternalLoaderPresetConfigRepository::getAllLoadablesDefinitionNames() const
{
std::set<String> paths;
boost::range::copy(preset | boost::adaptors::map_keys, std::inserter(paths, paths.end()));
return paths;
}
bool ExternalLoaderPresetConfigRepository::exists(const String& path) const
{
return preset.count(path);
}
Poco::Timestamp ExternalLoaderPresetConfigRepository::getUpdateTime(const String & path)
{
if (!exists(path))
throw Exception("Loadable " + path + " not found", ErrorCodes::BAD_ARGUMENTS);
return creation_time;
}
/// May contain definition about several entities (several dictionaries in one .xml file)
LoadablesConfigurationPtr ExternalLoaderPresetConfigRepository::load(const String & path) const
{
auto it = preset.find(path);
if (it == preset.end())
throw Exception("Loadable " + path + " not found", ErrorCodes::BAD_ARGUMENTS);
return it->second;
}
}

28
dbms/src/Interpreters/ExternalLoaderPresetConfigRepository.h
View File

@ -1,28 +0,0 @@
#pragma once
#include <Core/Types.h>
#include <unordered_map>
#include <Interpreters/IExternalLoaderConfigRepository.h>
#include <Poco/Timestamp.h>
namespace DB
{
/// A config repository filled with preset loadables used by ExternalLoader.
class ExternalLoaderPresetConfigRepository : public IExternalLoaderConfigRepository
{
public:
ExternalLoaderPresetConfigRepository(const std::vector<std::pair<String, LoadablesConfigurationPtr>> & preset_);
~ExternalLoaderPresetConfigRepository() override;
std::set<String> getAllLoadablesDefinitionNames() const override;
bool exists(const String & path) const override;
Poco::Timestamp getUpdateTime(const String & path) override;
LoadablesConfigurationPtr load(const String & path) const override;
private:
std::unordered_map<String, LoadablesConfigurationPtr> preset;
Poco::Timestamp creation_time;
};
}

46
dbms/src/Interpreters/ExternalLoaderTempConfigRepository.cpp Normal file
View File

@ -0,0 +1,46 @@
#include <Interpreters/ExternalLoaderTempConfigRepository.h>
#include <Common/Exception.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
ExternalLoaderTempConfigRepository::ExternalLoaderTempConfigRepository(const String & repository_name_, const String & path_, const LoadablesConfigurationPtr & config_)
: name(repository_name_), path(path_), config(config_) {}
std::set<String> ExternalLoaderTempConfigRepository::getAllLoadablesDefinitionNames()
{
std::set<String> paths;
paths.insert(path);
return paths;
}
bool ExternalLoaderTempConfigRepository::exists(const String & path_)
{
return path == path_;
}
Poco::Timestamp ExternalLoaderTempConfigRepository::getUpdateTime(const String & path_)
{
if (!exists(path_))
throw Exception("Loadable " + path_ + " not found", ErrorCodes::BAD_ARGUMENTS);
return creation_time;
}
LoadablesConfigurationPtr ExternalLoaderTempConfigRepository::load(const String & path_)
{
if (!exists(path_))
throw Exception("Loadable " + path_ + " not found", ErrorCodes::BAD_ARGUMENTS);
return config;
}
}

31
dbms/src/Interpreters/ExternalLoaderTempConfigRepository.h Normal file
View File

@ -0,0 +1,31 @@
#pragma once
#include <Core/Types.h>
#include <Interpreters/IExternalLoaderConfigRepository.h>
#include <Poco/Timestamp.h>
namespace DB
{
/// A config repository filled with preset loadables used by ExternalLoader.
class ExternalLoaderTempConfigRepository : public IExternalLoaderConfigRepository
{
public:
ExternalLoaderTempConfigRepository(const String & repository_name_, const String & path_, const LoadablesConfigurationPtr & config_);
const String & getName() const override { return name; }
bool isTemporary() const override { return true; }
std::set<String> getAllLoadablesDefinitionNames() override;
bool exists(const String & path) override;
Poco::Timestamp getUpdateTime(const String & path) override;
LoadablesConfigurationPtr load(const String & path) override;
private:
String name;
String path;
LoadablesConfigurationPtr config;
Poco::Timestamp creation_time;
};
}

11
dbms/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp
View File

@ -11,13 +11,18 @@
namespace DB
{
ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository(
const Poco::Util::AbstractConfiguration & main_config_, const std::string & config_key_)
: main_config(main_config_), config_key(config_key_)
{
}
Poco::Timestamp ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name)
{
return Poco::File(definition_entity_name).getLastModified();
}
std::set<std::string> ExternalLoaderXMLConfigRepository::getAllLoadablesDefinitionNames() const
std::set<std::string> ExternalLoaderXMLConfigRepository::getAllLoadablesDefinitionNames()
{
std::set<std::string> files;
@ -52,13 +57,13 @@ std::set<std::string> ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti
return files;
}
bool ExternalLoaderXMLConfigRepository::exists(const std::string & definition_entity_name) const
bool ExternalLoaderXMLConfigRepository::exists(const std::string & definition_entity_name)
{
return Poco::File(definition_entity_name).exists();
}
Poco::AutoPtr<Poco::Util::AbstractConfiguration> ExternalLoaderXMLConfigRepository::load(
const std::string & config_file) const
const std::string & config_file)
{
ConfigProcessor config_processor{config_file};
ConfigProcessor::LoadedConfig preprocessed = config_processor.loadConfig();

15
dbms/src/Interpreters/ExternalLoaderXMLConfigRepository.h
View File

@ -13,26 +13,25 @@ namespace DB
class ExternalLoaderXMLConfigRepository : public IExternalLoaderConfigRepository
{
public:
ExternalLoaderXMLConfigRepository(const Poco::Util::AbstractConfiguration & main_config_, const std::string & config_key_);
ExternalLoaderXMLConfigRepository(const Poco::Util::AbstractConfiguration & main_config_, const std::string & config_key_)
: main_config(main_config_)
, config_key(config_key_)
{
}
const String & getName() const override { return name; }
/// Return set of .xml files from path in main_config (config_key)
std::set<std::string> getAllLoadablesDefinitionNames() const override;
std::set<std::string> getAllLoadablesDefinitionNames() override;
/// Checks that file with name exists on filesystem
bool exists(const std::string & definition_entity_name) const override;
bool exists(const std::string & definition_entity_name) override;
/// Return xml-file modification time via stat call
Poco::Timestamp getUpdateTime(const std::string & definition_entity_name) override;
/// May contain definition about several entities (several dictionaries in one .xml file)
LoadablesConfigurationPtr load(const std::string & definition_entity_name) const override;
LoadablesConfigurationPtr load(const std::string & definition_entity_name) override;
private:
const String name;
/// Main server config (config.xml).
const Poco::Util::AbstractConfiguration & main_config;

6
dbms/src/Interpreters/ExternalModelsLoader.cpp
View File

@ -14,6 +14,7 @@ ExternalModelsLoader::ExternalModelsLoader(Context & context_)
: ExternalLoader("external model", &Logger::get("ExternalModelsLoader"))
, context(context_)
{
setConfigSettings({"models", "name", {}});
enablePeriodicUpdates(true);
}
@ -38,9 +39,4 @@ std::shared_ptr<const IExternalLoadable> ExternalModelsLoader::create(
throw Exception("Unknown model type: " + type, ErrorCodes::INVALID_CONFIG_PARAMETER);
}
}
void ExternalModelsLoader::addConfigRepository(const String & name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
{
ExternalLoader::addConfigRepository(name, std::move(config_repository), {"models", "name"});
}
}

4
dbms/src/Interpreters/ExternalModelsLoader.h
View File

@ -25,10 +25,6 @@ public:
return std::static_pointer_cast<const IModel>(load(name));
}
void addConfigRepository(const String & name,
std::unique_ptr<IExternalLoaderConfigRepository> config_repository);
protected:
LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config,
const std::string & key_in_config, const std::string & repository_name) const override;

2
dbms/src/Interpreters/IExternalLoadable.h
View File

@ -36,7 +36,7 @@ public:
virtual const ExternalLoadableLifetime & getLifetime() const = 0;
virtual std::string getName() const = 0;
virtual const std::string & getLoadableName() const = 0;
/// True if object can be updated when lifetime exceeded.
virtual bool supportUpdates() const = 0;
/// If lifetime exceeded and isModified(), ExternalLoader replace current object with the result of clone().

7
dbms/src/Interpreters/IExternalLoaderConfigRepository.cpp
View File

@ -1,7 +0,0 @@
#include <Interpreters/IExternalLoaderConfigRepository.h>
namespace DB
{
const char * IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX = "\xFF internal repo ";
}

23
dbms/src/Interpreters/IExternalLoaderConfigRepository.h
View File

@ -4,13 +4,13 @@
#include <Poco/Util/AbstractConfiguration.h>
#include <Poco/Timestamp.h>
#include <atomic>
#include <memory>
#include <string>
#include <set>
namespace DB
{
using LoadablesConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
/// Base interface for configurations source for Loadble objects, which can be
@ -22,24 +22,27 @@ using LoadablesConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguratio
class IExternalLoaderConfigRepository
{
public:
/// Returns the name of the repository.
virtual const std::string & getName() const = 0;
/// Whether this repository is temporary:
/// it's created and destroyed while executing the same query.
virtual bool isTemporary() const { return false; }
/// Return all available sources of loadables structure
/// (all .xml files from fs, all entities from database, etc)
virtual std::set<std::string> getAllLoadablesDefinitionNames() const = 0;
virtual std::set<std::string> getAllLoadablesDefinitionNames() = 0;
/// Checks that source of loadables configuration exist.
virtual bool exists(const std::string & loadable_definition_name) const = 0;
virtual bool exists(const std::string & path) = 0;
/// Returns entity last update time
virtual Poco::Timestamp getUpdateTime(const std::string & loadable_definition_name) = 0;
virtual Poco::Timestamp getUpdateTime(const std::string & path) = 0;
/// Load configuration from some concrete source to AbstractConfiguration
virtual LoadablesConfigurationPtr load(const std::string & loadable_definition_name) const = 0;
virtual LoadablesConfigurationPtr load(const std::string & path) = 0;
virtual ~IExternalLoaderConfigRepository() = default;
static const char * INTERNAL_REPOSITORY_NAME_PREFIX;
virtual ~IExternalLoaderConfigRepository() {}
};
using ExternalLoaderConfigRepositoryPtr = std::unique_ptr<IExternalLoaderConfigRepository>;
}

4
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -691,7 +691,9 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create)
String dictionary_name = create.table;
String database_name = !create.database.empty() ? create.database : context.getCurrentDatabase();
if (create.database.empty())
create.database = context.getCurrentDatabase();
const String & database_name = create.database;
auto guard = context.getDDLGuard(database_name, dictionary_name);
DatabasePtr database = context.getDatabase(database_name);

25
dbms/src/Storages/System/StorageSystemDictionaries.cpp
View File

@ -50,19 +50,25 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
const auto & external_dictionaries = context.getExternalDictionariesLoader();
for (const auto & load_result : external_dictionaries.getCurrentLoadResults())
{
if (startsWith(load_result.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX))
continue;
const auto dict_ptr = std::dynamic_pointer_cast<const IDictionaryBase>(load_result.object);
size_t i = 0;
String database;
String short_name = load_result.name;
if (!load_result.repository_name.empty() && startsWith(load_result.name, load_result.repository_name + "."))
String database, short_name;
if (dict_ptr)
{
database = load_result.repository_name;
short_name = load_result.name.substr(load_result.repository_name.length() + 1);
database = dict_ptr->getDatabase();
short_name = dict_ptr->getName();
}
else
{
short_name = load_result.name;
if (!load_result.repository_name.empty() && startsWith(short_name, load_result.repository_name + "."))
{
database = load_result.repository_name;
short_name = short_name.substr(database.length() + 1);
}
}
size_t i = 0;
res_columns[i++]->insert(database);
res_columns[i++]->insert(short_name);
res_columns[i++]->insert(static_cast<Int8>(load_result.status));
@ -70,7 +76,6 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
std::exception_ptr last_exception = load_result.exception;
const auto dict_ptr = std::dynamic_pointer_cast<const IDictionaryBase>(load_result.object);
if (dict_ptr)
{
res_columns[i++]->insert(dict_ptr->getTypeName());

52
libs/libcommon/include/ext/scope_guard.h
View File

@ -1,22 +1,56 @@
#pragma once
#include <utility>
#include <functional>
namespace ext
{
template <class F> class scope_guard {
const F function;
template <class F>
class [[nodiscard]] basic_scope_guard
{
public:
constexpr scope_guard(const F & function_) : function{function_} {}
constexpr scope_guard(F && function_) : function{std::move(function_)} {}
~scope_guard() { function(); }
constexpr basic_scope_guard() = default;
constexpr basic_scope_guard(basic_scope_guard && src) : function{std::exchange(src.function, F{})} {}
constexpr basic_scope_guard & operator=(basic_scope_guard && src)
{
if (this != &src)
{
invoke();
function = std::exchange(src.function, F{});
}
return *this;
}
template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
constexpr basic_scope_guard(const G & function_) : function{function_} {}
template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
constexpr basic_scope_guard(G && function_) : function{std::move(function_)} {}
~basic_scope_guard() { invoke(); }
private:
void invoke()
{
if constexpr (std::is_constructible_v<bool, F>)
{
if (!function)
return;
}
function();
}
F function = F{};
};
template <class F>
inline scope_guard<F> make_scope_guard(F && function_) { return std::forward<F>(function_); }
using scope_guard = basic_scope_guard<std::function<void(void)>>;
template <class F>
inline basic_scope_guard<F> make_scope_guard(F && function_) { return std::forward<F>(function_); }
}
#define SCOPE_EXIT_CONCAT(n, ...) \