thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use separate variables for the database and name in dictionaries.

Browse Source
This commit is contained in:
Vitaly Baranov 2019-12-26 06:12:12 +07:00
parent 4b4efa3b0b
commit 1fa64a2a86
29 changed files with 231 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dbms/src/Databases/DatabaseWithDictionaries.cpp
View File

@ -110,10 +110,9 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
+ std::to_string(++counter);
external_loader.addConfigRepository(
temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path,
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), getDatabaseName())}}));
temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>())}}));
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);

29
dbms/src/Dictionaries/CacheDictionary.cpp
View File

@ -57,12 +57,15 @@ inline size_t CacheDictionary::getCellIdx(const Key id) const
CacheDictionary::CacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -73,7 +76,7 @@ CacheDictionary::CacheDictionary(
, rnd_engine(randomSeed())
{
if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
throw Exception{full_name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes();
}
@ -204,7 +207,7 @@ void CacheDictionary::isInConstantVector(const Key child_id, const PaddedPODArra
void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -215,7 +218,7 @@ void CacheDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, ids, out, [&](const size_t row) { return def->getDataAt(row); });
}
@ -224,7 +227,7 @@ void CacheDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, ids, out, [&](const size_t) { return StringRef{def}; });
}
@ -352,7 +355,7 @@ void CacheDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -539,7 +542,7 @@ CacheDictionary::Attribute & CacheDictionary::getAttribute(const std::string & a
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -580,7 +583,7 @@ std::exception_ptr CacheDictionary::getLastException() const
void registerDictionaryCache(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -590,22 +593,24 @@ void registerDictionaryCache(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const auto & layout_prefix = config_prefix + ".layout";
const auto size = config.getInt(layout_prefix + ".cache.size_in_cells");
if (size == 0)
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
if (require_nonempty)
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
return std::make_unique<CacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
return std::make_unique<CacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
};
factory.registerLayout("cache", create_layout, false);
}

9
dbms/src/Dictionaries/CacheDictionary.h
View File

@ -25,13 +25,16 @@ class CacheDictionary final : public IDictionary
{
public:
CacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Cache"; }
@ -52,7 +55,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<CacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size);
return std::make_shared<CacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -254,7 +257,9 @@ private:
template <typename AncestorType>
void isInImpl(const PaddedPODArray<Key> & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
mutable DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

2
dbms/src/Dictionaries/CacheDictionary.inc.h
View File

@ -333,7 +333,7 @@ void CacheDictionary::update(
last_exception = std::current_exception();
backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count));
tryLogException(last_exception, log, "Could not update cache dictionary '" + getName() +
tryLogException(last_exception, log, "Could not update cache dictionary '" + getFullName() +
"', next update is scheduled at " + ext::to_string(backoff_end_time));
}
}

27
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
View File

@ -51,12 +51,15 @@ inline UInt64 ComplexKeyCacheDictionary::getCellIdx(const StringRef key) const
ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
const size_t size_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -65,7 +68,7 @@ ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
, rnd_engine(randomSeed())
{
if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
throw Exception{full_name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes();
}
@ -77,7 +80,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -94,7 +97,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, key_columns, out, [&](const size_t row) { return def->getDataAt(row); });
}
@ -109,7 +112,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, key_columns, out, [&](const size_t) { return StringRef{def}; });
}
@ -249,7 +252,7 @@ void ComplexKeyCacheDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
}
}
@ -258,7 +261,7 @@ ComplexKeyCacheDictionary::Attribute & ComplexKeyCacheDictionary::getAttribute(c
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -394,7 +397,7 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -405,15 +408,17 @@ void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
const auto & layout_prefix = config_prefix + ".layout";
const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells");
if (size == 0)
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
if (require_nonempty)
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
return std::make_unique<ComplexKeyCacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
return std::make_unique<ComplexKeyCacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
};
factory.registerLayout("complex_key_cache", create_layout, true);
}

9
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
View File

@ -42,6 +42,7 @@ class ComplexKeyCacheDictionary final : public IDictionaryBase
{
public:
ComplexKeyCacheDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -50,7 +51,9 @@ public:
std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "ComplexKeyCache"; }
@ -75,7 +78,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<ComplexKeyCacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size);
return std::make_shared<ComplexKeyCacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -668,7 +671,9 @@ private:
bool isEmptyCell(const UInt64 idx) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

30
dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
View File

@ -15,13 +15,16 @@ namespace ErrorCodes
}
ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -40,7 +43,7 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -72,7 +75,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -94,7 +97,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, \
@ -128,7 +131,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -148,7 +151,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -179,7 +182,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -256,7 +259,7 @@ void ComplexKeyHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
}
}
@ -397,7 +400,7 @@ void ComplexKeyHashedDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -630,7 +633,7 @@ const ComplexKeyHashedDictionary::Attribute & ComplexKeyHashedDictionary::getAtt
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -742,7 +745,7 @@ BlockInputStreamPtr ComplexKeyHashedDictionary::getBlockInputStream(const Names
void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string &,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -751,12 +754,13 @@ void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
if (!dict_struct.key)
throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<ComplexKeyHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<ComplexKeyHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("complex_key_hashed", create_layout, true);
}
}

9
dbms/src/Dictionaries/ComplexKeyHashedDictionary.h
View File

@ -23,6 +23,7 @@ class ComplexKeyHashedDictionary final : public IDictionaryBase
{
public:
ComplexKeyHashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -32,7 +33,9 @@ public:
std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "ComplexKeyHashed"; }
@ -48,7 +51,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<ComplexKeyHashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
return std::make_shared<ComplexKeyHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -233,7 +236,9 @@ private:
template <typename T>
std::vector<StringRef> getKeys(const Attribute & attribute) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

34
dbms/src/Dictionaries/FlatDictionary.cpp
View File

@ -21,13 +21,16 @@ static const auto max_array_size = 500000;
FlatDictionary::FlatDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -107,7 +110,7 @@ void FlatDictionary::isInConstantVector(const Key child_id, const PaddedPODArray
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -133,7 +136,7 @@ DECLARE(Decimal128)
void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = std::get<StringRef>(attribute.null_values);
@ -152,7 +155,7 @@ void FlatDictionary::getString(const std::string & attribute_name, const PaddedP
ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
@ -177,7 +180,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -191,7 +194,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -216,7 +219,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
FlatDictionary::getItemsImpl<StringRef, StringRef>(
attribute,
@ -297,7 +300,7 @@ void FlatDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -404,7 +407,7 @@ void FlatDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
@ -578,7 +581,7 @@ template <typename T>
void FlatDictionary::resize(Attribute & attribute, const Key id)
{
if (id >= max_array_size)
throw Exception{name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
throw Exception{full_name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
auto & array = std::get<ContainerType<T>>(attribute.arrays);
if (id >= array.size())
@ -666,7 +669,7 @@ const FlatDictionary::Attribute & FlatDictionary::getAttribute(const std::string
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -706,7 +709,7 @@ BlockInputStreamPtr FlatDictionary::getBlockInputStream(const Names & column_nam
void registerDictionaryFlat(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -716,13 +719,16 @@ void registerDictionaryFlat(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'flat'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<FlatDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<FlatDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("flat", create_layout, false);
}

9
dbms/src/Dictionaries/FlatDictionary.h
View File

@ -22,6 +22,7 @@ class FlatDictionary final : public IDictionary
{
public:
FlatDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -29,7 +30,9 @@ public:
bool require_nonempty_,
BlockPtr saved_block_ = nullptr);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Flat"; }
@ -45,7 +48,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<FlatDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
return std::make_shared<FlatDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -222,7 +225,9 @@ private:
PaddedPODArray<Key> getIds() const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

41
dbms/src/Dictionaries/HashedDictionary.cpp
View File

@ -31,6 +31,7 @@ namespace ErrorCodes
HashedDictionary::HashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -38,7 +39,9 @@ HashedDictionary::HashedDictionary(
bool require_nonempty_,
bool sparse_,
BlockPtr saved_block_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -129,7 +132,7 @@ void HashedDictionary::isInConstantVector(const Key child_id, const PaddedPODArr
const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -155,7 +158,7 @@ DECLARE(Decimal128)
void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -174,7 +177,7 @@ void HashedDictionary::getString(const std::string & attribute_name, const Padde
ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
@ -199,7 +202,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -213,7 +216,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const \
{ \
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -238,7 +241,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -317,7 +320,7 @@ void HashedDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -424,7 +427,7 @@ void HashedDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -684,7 +687,7 @@ const HashedDictionary::Attribute & HashedDictionary::getAttribute(const std::st
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -768,27 +771,31 @@ BlockInputStreamPtr HashedDictionary::getBlockInputStream(const Names & column_n
void registerDictionaryHashed(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
DictionarySourcePtr source_ptr) -> DictionaryPtr
DictionarySourcePtr source_ptr,
bool sparse) -> DictionaryPtr
{
if (dict_struct.key)
throw Exception{"'key' is not supported for dictionary of layout 'hashed'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name
throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
const bool sparse = name == "sparse_hashed";
return std::make_unique<HashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
return std::make_unique<HashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
};
factory.registerLayout("hashed", create_layout, false);
factory.registerLayout("sparse_hashed", create_layout, false);
using namespace std::placeholders;
factory.registerLayout("hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ false), false);
factory.registerLayout("sparse_hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ true), false);
}
}

9
dbms/src/Dictionaries/HashedDictionary.h
View File

@ -26,6 +26,7 @@ class HashedDictionary final : public IDictionary
{
public:
HashedDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -34,7 +35,9 @@ public:
bool sparse_,
BlockPtr saved_block_ = nullptr);
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return sparse ? "SparseHashed" : "Hashed"; }
@ -50,7 +53,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<HashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
return std::make_shared<HashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -262,7 +265,9 @@ private:
template <typename ChildType, typename AncestorType>
void isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

5
dbms/src/Dictionaries/IDictionary.h
View File

@ -25,6 +25,11 @@ struct IDictionaryBase : public IExternalLoadable
{
using Key = UInt64;
virtual const std::string & getDatabase() const = 0;
virtual const std::string & getName() const = 0;
virtual const std::string & getFullName() const = 0;
const std::string & getLoadableName() const override { return getFullName(); }
virtual std::string getTypeName() const = 0;
virtual size_t getBytesAllocated() const = 0;

21
dbms/src/Dictionaries/RangeHashedDictionary.cpp
View File

@ -68,12 +68,15 @@ static bool operator<(const RangeHashedDictionary::Range & left, const RangeHash
RangeHashedDictionary::RangeHashedDictionary(
const std::string & dictionary_name_,
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_)
: dictionary_name{dictionary_name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -156,7 +159,7 @@ void RangeHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{dictionary_name + ": hierarchical attributes not supported by " + getName() + " dictionary.",
throw Exception{full_name + ": hierarchical attributes not supported by " + getName() + " dictionary.",
ErrorCodes::BAD_ARGUMENTS};
}
}
@ -207,7 +210,7 @@ void RangeHashedDictionary::loadData()
stream->readSuffix();
if (require_nonempty && 0 == element_count)
throw Exception{dictionary_name + ": dictionary source is empty and 'require_nonempty' property is set.",
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.",
ErrorCodes::DICTIONARY_IS_EMPTY};
}
@ -520,7 +523,7 @@ const RangeHashedDictionary::Attribute & RangeHashedDictionary::getAttribute(con
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{dictionary_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -674,7 +677,7 @@ BlockInputStreamPtr RangeHashedDictionary::getBlockInputStream(const Names & col
void registerDictionaryRangeHashed(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -684,12 +687,14 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'range_hashed'", ErrorCodes::UNSUPPORTED_METHOD};
if (!dict_struct.range_min || !dict_struct.range_max)
throw Exception{name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max",
throw Exception{full_name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max",
ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<RangeHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<RangeHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("range_hashed", create_layout, false);
}

13
dbms/src/Dictionaries/RangeHashedDictionary.h
View File

@ -18,13 +18,16 @@ class RangeHashedDictionary final : public IDictionaryBase
{
public:
RangeHashedDictionary(
const std::string & dictionary_name_,
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_);
std::string getName() const override { return dictionary_name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "RangeHashed"; }
@ -40,7 +43,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<RangeHashedDictionary>(dictionary_name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
return std::make_shared<RangeHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -208,7 +211,9 @@ private:
friend struct RangeHashedDIctionaryCallGetBlockInputStreamImpl;
const std::string dictionary_name;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

29
dbms/src/Dictionaries/TrieDictionary.cpp
View File

@ -35,12 +35,15 @@ namespace ErrorCodes
}
TrieDictionary::TrieDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_,
bool require_nonempty_)
: name{name_}
: database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_)
@ -75,7 +78,7 @@ TrieDictionary::~TrieDictionary()
validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
const auto null_value = std::get<TYPE>(attribute.null_values); \
\
@ -107,7 +110,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -129,7 +132,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, \
@ -163,7 +166,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -183,7 +186,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); \
\
const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\
getItemsImpl<TYPE, TYPE>( \
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -214,7 +217,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>(
attribute,
@ -291,7 +294,7 @@ void TrieDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
}
}
@ -337,7 +340,7 @@ void TrieDictionary::loadData()
stream->readSuffix();
if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -627,7 +630,7 @@ const TrieDictionary::Attribute & TrieDictionary::getAttribute(const std::string
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -767,7 +770,7 @@ BlockInputStreamPtr TrieDictionary::getBlockInputStream(const Names & column_nam
void registerDictionaryTrie(DictionaryFactory & factory)
{
auto create_layout = [=](const std::string & name,
auto create_layout = [=](const std::string &,
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
@ -776,10 +779,12 @@ void registerDictionaryTrie(DictionaryFactory & factory)
if (!dict_struct.key)
throw Exception{"'key' is required for dictionary of layout 'ip_trie'", ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
// This is specialised trie for storing IPv4 and IPv6 prefixes.
return std::make_unique<TrieDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
return std::make_unique<TrieDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
};
factory.registerLayout("ip_trie", create_layout, true);
}

9
dbms/src/Dictionaries/TrieDictionary.h
View File

@ -23,6 +23,7 @@ class TrieDictionary final : public IDictionaryBase
{
public:
TrieDictionary(
const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_,
@ -33,7 +34,9 @@ public:
std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; }
const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Trie"; }
@ -49,7 +52,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override
{
return std::make_shared<TrieDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
return std::make_shared<TrieDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -232,7 +235,9 @@ private:
Columns getKeyColumns() const;
const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime;

10
dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
View File

@ -421,7 +421,7 @@ void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs
}
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name)
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query)
{
checkAST(query);
@ -434,10 +434,14 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer
AutoPtr<Poco::XML::Element> name_element(xml_document->createElement("name"));
current_dictionary->appendChild(name_element);
String full_name = (!database_name.empty() ? database_name : query.database) + "." + query.table;
AutoPtr<Text> name(xml_document->createTextNode(full_name));
AutoPtr<Text> name(xml_document->createTextNode(query.table));
name_element->appendChild(name);
AutoPtr<Poco::XML::Element> database_element(xml_document->createElement("database"));
current_dictionary->appendChild(database_element);
AutoPtr<Text> database(xml_document->createTextNode(query.database));
database_element->appendChild(database);
AutoPtr<Element> structure_element(xml_document->createElement("structure"));
current_dictionary->appendChild(structure_element);
Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key);

2
dbms/src/Dictionaries/getDictionaryConfigurationFromAST.h
View File

@ -10,6 +10,6 @@ using DictionaryConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfigurati
/// Convert dictionary AST to Poco::AbstractConfiguration
/// This function is necessary because all loadable objects configuration are Poco::AbstractConfiguration
/// Can throw exception if query is ill-formed
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name = {});
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query);
}

3
dbms/src/Dictionaries/tests/gtest_dictionary_configuration.cpp
View File

@ -57,7 +57,8 @@ TEST(ConvertDictionaryAST, SimpleDictConfiguration)
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create);
/// name
EXPECT_EQ(config->getString("dictionary.name"), "test.dict1");
EXPECT_EQ(config->getString("dictionary.database"), "test");
EXPECT_EQ(config->getString("dictionary.name"), "dict1");
/// lifetime
EXPECT_EQ(config->getInt("dictionary.lifetime.min"), 1);

12
dbms/src/Functions/FunctionsExternalDictionaries.h
View File

@ -127,10 +127,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
if (!executeDispatchSimple<FlatDictionary>(block, arguments, result, dict_ptr) &&
@ -302,10 +302,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
@ -488,10 +488,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&

4
dbms/src/Interpreters/CatBoostModel.h
View File

@ -60,7 +60,7 @@ public:
const ExternalLoadableLifetime & getLifetime() const override;
std::string getName() const override { return name; }
const std::string & getLoadableName() const override { return name; }
bool supportUpdates() const override { return true; }
@ -69,7 +69,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override;
private:
std::string name;
const std::string name;
std::string model_path;
std::string lib_path;
ExternalLoadableLifetime lifetime;

2
dbms/src/Interpreters/ExternalDictionariesLoader.cpp
View File

@ -27,7 +27,7 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create(
void ExternalDictionariesLoader::addConfigRepository(
const std::string & repository_name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
{
ExternalLoader::addConfigRepository(repository_name, std::move(config_repository), {"dictionary", "name"});
ExternalLoader::addConfigRepository(repository_name, std::move(config_repository), {"dictionary", "name", "database"});
}
}

8
dbms/src/Interpreters/ExternalLoader.cpp
View File

@ -293,6 +293,12 @@ private:
continue;
}
String database;
if (!settings.external_database.empty())
database = file_contents->getString(key + "." + settings.external_database, "");
if (!database.empty())
object_name = database + "." + object_name;
object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}});
}
@ -613,7 +619,7 @@ public:
}
catch (...)
{
tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getName() + "' was modified");
tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getLoadableName() + "' was modified");
/// Cannot check isModified, so update
should_update_flag = true;
}

1
dbms/src/Interpreters/ExternalLoader.h
View File

@ -24,6 +24,7 @@ struct ExternalLoaderConfigSettings
{
std::string external_config;
std::string external_name;
std::string external_database;
};

2
dbms/src/Interpreters/ExternalModelsLoader.cpp
View File

@ -41,6 +41,6 @@ std::shared_ptr<const IExternalLoadable> ExternalModelsLoader::create(
void ExternalModelsLoader::addConfigRepository(const String & name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
{
ExternalLoader::addConfigRepository(name, std::move(config_repository), {"models", "name"});
ExternalLoader::addConfigRepository(name, std::move(config_repository), {"models", "name", {}});
}
}

2
dbms/src/Interpreters/IExternalLoadable.h
View File

@ -36,7 +36,7 @@ public:
virtual const ExternalLoadableLifetime & getLifetime() const = 0;
virtual std::string getName() const = 0;
virtual const std::string & getLoadableName() const = 0;
/// True if object can be updated when lifetime exceeded.
virtual bool supportUpdates() const = 0;
/// If lifetime exceeded and isModified(), ExternalLoader replace current object with the result of clone().

4
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -691,7 +691,9 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create)
String dictionary_name = create.table;
String database_name = !create.database.empty() ? create.database : context.getCurrentDatabase();
if (create.database.empty())
create.database = context.getCurrentDatabase();
const String & database_name = create.database;
auto guard = context.getDDLGuard(database_name, dictionary_name);
DatabasePtr database = context.getDatabase(database_name);

22
dbms/src/Storages/System/StorageSystemDictionaries.cpp
View File

@ -53,16 +53,25 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
if (startsWith(load_result.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX))
continue;
size_t i = 0;
String database;
String short_name = load_result.name;
const auto dict_ptr = std::dynamic_pointer_cast<const IDictionaryBase>(load_result.object);
if (!load_result.repository_name.empty() && startsWith(load_result.name, load_result.repository_name + "."))
String database, short_name;
if (dict_ptr)
{
database = load_result.repository_name;
short_name = load_result.name.substr(load_result.repository_name.length() + 1);
database = dict_ptr->getDatabase();
short_name = dict_ptr->getName();
}
else
{
short_name = load_result.name;
if (!load_result.repository_name.empty() && startsWith(short_name, load_result.repository_name + "."))
{
database = load_result.repository_name;
short_name = short_name.substr(database.length() + 1);
}
}
size_t i = 0;
res_columns[i++]->insert(database);
res_columns[i++]->insert(short_name);
res_columns[i++]->insert(static_cast<Int8>(load_result.status));
@ -70,7 +79,6 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
std::exception_ptr last_exception = load_result.exception;
const auto dict_ptr = std::dynamic_pointer_cast<const IDictionaryBase>(load_result.object);
if (dict_ptr)
{
res_columns[i++]->insert(dict_ptr->getTypeName());