Use separate variables for the database and name in dictionaries.

This commit is contained in:
Vitaly Baranov 2019-12-26 06:12:12 +07:00
parent 4b4efa3b0b
commit 1fa64a2a86
29 changed files with 231 additions and 133 deletions

View File

@ -110,10 +110,9 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " " String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
+ std::to_string(++counter); + std::to_string(++counter);
external_loader.addConfigRepository( external_loader.addConfigRepository(
temp_repository_name, temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>( std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path, std::vector{std::pair{dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>())}}));
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), getDatabaseName())}}));
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); }); SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true); bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);

View File

@ -57,12 +57,15 @@ inline size_t CacheDictionary::getCellIdx(const Key id) const
CacheDictionary::CacheDictionary( CacheDictionary::CacheDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_, const DictionaryLifetime dict_lifetime_,
const size_t size_) const size_t size_)
: name{name_} : database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_) , dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)} , source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_) , dict_lifetime(dict_lifetime_)
@ -73,7 +76,7 @@ CacheDictionary::CacheDictionary(
, rnd_engine(randomSeed()) , rnd_engine(randomSeed())
{ {
if (!this->source_ptr->supportsSelectiveLoad()) if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD}; throw Exception{full_name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes(); createAttributes();
} }
@ -204,7 +207,7 @@ void CacheDictionary::isInConstantVector(const Key child_id, const PaddedPODArra
void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{ {
auto & attribute = getAttribute(attribute_name); auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto null_value = StringRef{std::get<String>(attribute.null_values)}; const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -215,7 +218,7 @@ void CacheDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{ {
auto & attribute = getAttribute(attribute_name); auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, ids, out, [&](const size_t row) { return def->getDataAt(row); }); getItemsString(attribute, ids, out, [&](const size_t row) { return def->getDataAt(row); });
} }
@ -224,7 +227,7 @@ void CacheDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{ {
auto & attribute = getAttribute(attribute_name); auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, ids, out, [&](const size_t) { return StringRef{def}; }); getItemsString(attribute, ids, out, [&](const size_t) { return StringRef{def}; });
} }
@ -352,7 +355,7 @@ void CacheDictionary::createAttributes()
hierarchical_attribute = &attributes.back(); hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64) if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH}; throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
} }
} }
} }
@ -539,7 +542,7 @@ CacheDictionary::Attribute & CacheDictionary::getAttribute(const std::string & a
{ {
const auto it = attribute_index_by_name.find(attribute_name); const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name)) if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second]; return attributes[it->second];
} }
@ -580,7 +583,7 @@ std::exception_ptr CacheDictionary::getLastException() const
void registerDictionaryCache(DictionaryFactory & factory) void registerDictionaryCache(DictionaryFactory & factory)
{ {
auto create_layout = [=](const std::string & name, auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct, const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const std::string & config_prefix,
@ -590,22 +593,24 @@ void registerDictionaryCache(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD}; throw Exception{"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max) if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only " + ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'", "for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS}; ErrorCodes::BAD_ARGUMENTS};
const auto & layout_prefix = config_prefix + ".layout"; const auto & layout_prefix = config_prefix + ".layout";
const auto size = config.getInt(layout_prefix + ".cache.size_in_cells"); const auto size = config.getInt(layout_prefix + ".cache.size_in_cells");
if (size == 0) if (size == 0)
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE}; throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
if (require_nonempty) if (require_nonempty)
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set", throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
ErrorCodes::BAD_ARGUMENTS}; ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
return std::make_unique<CacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size); return std::make_unique<CacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
}; };
factory.registerLayout("cache", create_layout, false); factory.registerLayout("cache", create_layout, false);
} }

View File

@ -25,13 +25,16 @@ class CacheDictionary final : public IDictionary
{ {
public: public:
CacheDictionary( CacheDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_, const DictionaryLifetime dict_lifetime_,
const size_t size_); const size_t size_);
std::string getName() const override { return name; } const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Cache"; } std::string getTypeName() const override { return "Cache"; }
@ -52,7 +55,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override std::shared_ptr<const IExternalLoadable> clone() const override
{ {
return std::make_shared<CacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size); return std::make_shared<CacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
} }
const IDictionarySource * getSource() const override { return source_ptr.get(); } const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -254,7 +257,9 @@ private:
template <typename AncestorType> template <typename AncestorType>
void isInImpl(const PaddedPODArray<Key> & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const; void isInImpl(const PaddedPODArray<Key> & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
const std::string database;
const std::string name; const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct; const DictionaryStructure dict_struct;
mutable DictionarySourcePtr source_ptr; mutable DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime; const DictionaryLifetime dict_lifetime;

View File

@ -333,7 +333,7 @@ void CacheDictionary::update(
last_exception = std::current_exception(); last_exception = std::current_exception();
backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count));
tryLogException(last_exception, log, "Could not update cache dictionary '" + getName() + tryLogException(last_exception, log, "Could not update cache dictionary '" + getFullName() +
"', next update is scheduled at " + ext::to_string(backoff_end_time)); "', next update is scheduled at " + ext::to_string(backoff_end_time));
} }
} }

View File

@ -51,12 +51,15 @@ inline UInt64 ComplexKeyCacheDictionary::getCellIdx(const StringRef key) const
ComplexKeyCacheDictionary::ComplexKeyCacheDictionary( ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_, const DictionaryLifetime dict_lifetime_,
const size_t size_) const size_t size_)
: name{name_} : database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_) , dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)} , source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_) , dict_lifetime(dict_lifetime_)
@ -65,7 +68,7 @@ ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
, rnd_engine(randomSeed()) , rnd_engine(randomSeed())
{ {
if (!this->source_ptr->supportsSelectiveLoad()) if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD}; throw Exception{full_name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes(); createAttributes();
} }
@ -77,7 +80,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types); dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name); auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto null_value = StringRef{std::get<String>(attribute.null_values)}; const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -94,7 +97,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types); dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name); auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, key_columns, out, [&](const size_t row) { return def->getDataAt(row); }); getItemsString(attribute, key_columns, out, [&](const size_t row) { return def->getDataAt(row); });
} }
@ -109,7 +112,7 @@ void ComplexKeyCacheDictionary::getString(
dict_struct.validateKeyTypes(key_types); dict_struct.validateKeyTypes(key_types);
auto & attribute = getAttribute(attribute_name); auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsString(attribute, key_columns, out, [&](const size_t) { return StringRef{def}; }); getItemsString(attribute, key_columns, out, [&](const size_t) { return StringRef{def}; });
} }
@ -249,7 +252,7 @@ void ComplexKeyCacheDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value)); attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical) if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(), throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH}; ErrorCodes::TYPE_MISMATCH};
} }
} }
@ -258,7 +261,7 @@ ComplexKeyCacheDictionary::Attribute & ComplexKeyCacheDictionary::getAttribute(c
{ {
const auto it = attribute_index_by_name.find(attribute_name); const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name)) if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second]; return attributes[it->second];
} }
@ -394,7 +397,7 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
void registerDictionaryComplexKeyCache(DictionaryFactory & factory) void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
{ {
auto create_layout = [=](const std::string & name, auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct, const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const std::string & config_prefix,
@ -405,15 +408,17 @@ void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
const auto & layout_prefix = config_prefix + ".layout"; const auto & layout_prefix = config_prefix + ".layout";
const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells"); const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells");
if (size == 0) if (size == 0)
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE}; throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
if (require_nonempty) if (require_nonempty)
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set", throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
ErrorCodes::BAD_ARGUMENTS}; ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
return std::make_unique<ComplexKeyCacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size); return std::make_unique<ComplexKeyCacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
}; };
factory.registerLayout("complex_key_cache", create_layout, true); factory.registerLayout("complex_key_cache", create_layout, true);
} }

View File

@ -42,6 +42,7 @@ class ComplexKeyCacheDictionary final : public IDictionaryBase
{ {
public: public:
ComplexKeyCacheDictionary( ComplexKeyCacheDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
@ -50,7 +51,9 @@ public:
std::string getKeyDescription() const { return key_description; } std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; } const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "ComplexKeyCache"; } std::string getTypeName() const override { return "ComplexKeyCache"; }
@ -75,7 +78,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override std::shared_ptr<const IExternalLoadable> clone() const override
{ {
return std::make_shared<ComplexKeyCacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size); return std::make_shared<ComplexKeyCacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
} }
const IDictionarySource * getSource() const override { return source_ptr.get(); } const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -668,7 +671,9 @@ private:
bool isEmptyCell(const UInt64 idx) const; bool isEmptyCell(const UInt64 idx) const;
const std::string database;
const std::string name; const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct; const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr; const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime; const DictionaryLifetime dict_lifetime;

View File

@ -15,13 +15,16 @@ namespace ErrorCodes
} }
ComplexKeyHashedDictionary::ComplexKeyHashedDictionary( ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_, const DictionaryLifetime dict_lifetime_,
bool require_nonempty_, bool require_nonempty_,
BlockPtr saved_block_) BlockPtr saved_block_)
: name{name_} : database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_) , dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)} , source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_) , dict_lifetime(dict_lifetime_)
@ -40,7 +43,7 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
dict_struct.validateKeyTypes(key_types); \ dict_struct.validateKeyTypes(key_types); \
\ \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
const auto null_value = std::get<TYPE>(attribute.null_values); \ const auto null_value = std::get<TYPE>(attribute.null_values); \
\ \
@ -72,7 +75,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)}; const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -94,7 +97,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); \ dict_struct.validateKeyTypes(key_types); \
\ \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
getItemsImpl<TYPE, TYPE>( \ getItemsImpl<TYPE, TYPE>( \
attribute, \ attribute, \
@ -128,7 +131,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>( getItemsImpl<StringRef, StringRef>(
attribute, attribute,
@ -148,7 +151,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); \ dict_struct.validateKeyTypes(key_types); \
\ \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
getItemsImpl<TYPE, TYPE>( \ getItemsImpl<TYPE, TYPE>( \
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \ attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -179,7 +182,7 @@ void ComplexKeyHashedDictionary::getString(
dict_struct.validateKeyTypes(key_types); dict_struct.validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>( getItemsImpl<StringRef, StringRef>(
attribute, attribute,
@ -256,7 +259,7 @@ void ComplexKeyHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value)); attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical) if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(), throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH}; ErrorCodes::TYPE_MISMATCH};
} }
} }
@ -397,7 +400,7 @@ void ComplexKeyHashedDictionary::loadData()
updateData(); updateData();
if (require_nonempty && 0 == element_count) if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY}; throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
} }
template <typename T> template <typename T>
@ -630,7 +633,7 @@ const ComplexKeyHashedDictionary::Attribute & ComplexKeyHashedDictionary::getAtt
{ {
const auto it = attribute_index_by_name.find(attribute_name); const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name)) if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second]; return attributes[it->second];
} }
@ -742,7 +745,7 @@ BlockInputStreamPtr ComplexKeyHashedDictionary::getBlockInputStream(const Names
void registerDictionaryComplexKeyHashed(DictionaryFactory & factory) void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
{ {
auto create_layout = [=](const std::string & name, auto create_layout = [=](const std::string &,
const DictionaryStructure & dict_struct, const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const std::string & config_prefix,
@ -751,12 +754,13 @@ void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
if (!dict_struct.key) if (!dict_struct.key)
throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<ComplexKeyHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); return std::make_unique<ComplexKeyHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
}; };
factory.registerLayout("complex_key_hashed", create_layout, true); factory.registerLayout("complex_key_hashed", create_layout, true);
} }
} }

View File

@ -23,6 +23,7 @@ class ComplexKeyHashedDictionary final : public IDictionaryBase
{ {
public: public:
ComplexKeyHashedDictionary( ComplexKeyHashedDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
@ -32,7 +33,9 @@ public:
std::string getKeyDescription() const { return key_description; } std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; } const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "ComplexKeyHashed"; } std::string getTypeName() const override { return "ComplexKeyHashed"; }
@ -48,7 +51,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override std::shared_ptr<const IExternalLoadable> clone() const override
{ {
return std::make_shared<ComplexKeyHashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block); return std::make_shared<ComplexKeyHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
} }
const IDictionarySource * getSource() const override { return source_ptr.get(); } const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -233,7 +236,9 @@ private:
template <typename T> template <typename T>
std::vector<StringRef> getKeys(const Attribute & attribute) const; std::vector<StringRef> getKeys(const Attribute & attribute) const;
const std::string database;
const std::string name; const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct; const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr; const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime; const DictionaryLifetime dict_lifetime;

View File

@ -21,13 +21,16 @@ static const auto max_array_size = 500000;
FlatDictionary::FlatDictionary( FlatDictionary::FlatDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_, const DictionaryLifetime dict_lifetime_,
bool require_nonempty_, bool require_nonempty_,
BlockPtr saved_block_) BlockPtr saved_block_)
: name{name_} : database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_) , dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)} , source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_) , dict_lifetime(dict_lifetime_)
@ -107,7 +110,7 @@ void FlatDictionary::isInConstantVector(const Key child_id, const PaddedPODArray
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const \ void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const \
{ \ { \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
const auto null_value = std::get<TYPE>(attribute.null_values); \ const auto null_value = std::get<TYPE>(attribute.null_values); \
\ \
@ -133,7 +136,7 @@ DECLARE(Decimal128)
void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{ {
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = std::get<StringRef>(attribute.null_values); const auto & null_value = std::get<StringRef>(attribute.null_values);
@ -152,7 +155,7 @@ void FlatDictionary::getString(const std::string & attribute_name, const PaddedP
ResultArrayType<TYPE> & out) const \ ResultArrayType<TYPE> & out) const \
{ \ { \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
getItemsImpl<TYPE, TYPE>( \ getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \ attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
@ -177,7 +180,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{ {
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>( getItemsImpl<StringRef, StringRef>(
attribute, attribute,
@ -191,7 +194,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const \ const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const \
{ \ { \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
getItemsImpl<TYPE, TYPE>( \ getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \ attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -216,7 +219,7 @@ void FlatDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{ {
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
FlatDictionary::getItemsImpl<StringRef, StringRef>( FlatDictionary::getItemsImpl<StringRef, StringRef>(
attribute, attribute,
@ -297,7 +300,7 @@ void FlatDictionary::createAttributes()
hierarchical_attribute = &attributes.back(); hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64) if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH}; throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
} }
} }
} }
@ -404,7 +407,7 @@ void FlatDictionary::loadData()
updateData(); updateData();
if (require_nonempty && 0 == element_count) if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY}; throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
} }
@ -578,7 +581,7 @@ template <typename T>
void FlatDictionary::resize(Attribute & attribute, const Key id) void FlatDictionary::resize(Attribute & attribute, const Key id)
{ {
if (id >= max_array_size) if (id >= max_array_size)
throw Exception{name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND}; throw Exception{full_name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
auto & array = std::get<ContainerType<T>>(attribute.arrays); auto & array = std::get<ContainerType<T>>(attribute.arrays);
if (id >= array.size()) if (id >= array.size())
@ -666,7 +669,7 @@ const FlatDictionary::Attribute & FlatDictionary::getAttribute(const std::string
{ {
const auto it = attribute_index_by_name.find(attribute_name); const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name)) if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second]; return attributes[it->second];
} }
@ -706,7 +709,7 @@ BlockInputStreamPtr FlatDictionary::getBlockInputStream(const Names & column_nam
void registerDictionaryFlat(DictionaryFactory & factory) void registerDictionaryFlat(DictionaryFactory & factory)
{ {
auto create_layout = [=](const std::string & name, auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct, const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const std::string & config_prefix,
@ -716,13 +719,16 @@ void registerDictionaryFlat(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'flat'", ErrorCodes::UNSUPPORTED_METHOD}; throw Exception{"'key' is not supported for dictionary of layout 'flat'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max) if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only " + ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'", "for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS}; ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<FlatDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); return std::make_unique<FlatDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
}; };
factory.registerLayout("flat", create_layout, false); factory.registerLayout("flat", create_layout, false);
} }

View File

@ -22,6 +22,7 @@ class FlatDictionary final : public IDictionary
{ {
public: public:
FlatDictionary( FlatDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
@ -29,7 +30,9 @@ public:
bool require_nonempty_, bool require_nonempty_,
BlockPtr saved_block_ = nullptr); BlockPtr saved_block_ = nullptr);
std::string getName() const override { return name; } const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Flat"; } std::string getTypeName() const override { return "Flat"; }
@ -45,7 +48,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override std::shared_ptr<const IExternalLoadable> clone() const override
{ {
return std::make_shared<FlatDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block); return std::make_shared<FlatDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
} }
const IDictionarySource * getSource() const override { return source_ptr.get(); } const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -222,7 +225,9 @@ private:
PaddedPODArray<Key> getIds() const; PaddedPODArray<Key> getIds() const;
const std::string database;
const std::string name; const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct; const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr; const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime; const DictionaryLifetime dict_lifetime;

View File

@ -31,6 +31,7 @@ namespace ErrorCodes
HashedDictionary::HashedDictionary( HashedDictionary::HashedDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
@ -38,7 +39,9 @@ HashedDictionary::HashedDictionary(
bool require_nonempty_, bool require_nonempty_,
bool sparse_, bool sparse_,
BlockPtr saved_block_) BlockPtr saved_block_)
: name{name_} : database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_) , dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)} , source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_) , dict_lifetime(dict_lifetime_)
@ -129,7 +132,7 @@ void HashedDictionary::isInConstantVector(const Key child_id, const PaddedPODArr
const \ const \
{ \ { \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
const auto null_value = std::get<TYPE>(attribute.null_values); \ const auto null_value = std::get<TYPE>(attribute.null_values); \
\ \
@ -155,7 +158,7 @@ DECLARE(Decimal128)
void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
{ {
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)}; const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -174,7 +177,7 @@ void HashedDictionary::getString(const std::string & attribute_name, const Padde
ResultArrayType<TYPE> & out) const \ ResultArrayType<TYPE> & out) const \
{ \ { \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
getItemsImpl<TYPE, TYPE>( \ getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \ attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
@ -199,7 +202,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
{ {
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>( getItemsImpl<StringRef, StringRef>(
attribute, attribute,
@ -213,7 +216,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const \ const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const \
{ \ { \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
getItemsImpl<TYPE, TYPE>( \ getItemsImpl<TYPE, TYPE>( \
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \ attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -238,7 +241,7 @@ void HashedDictionary::getString(
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
{ {
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>( getItemsImpl<StringRef, StringRef>(
attribute, attribute,
@ -317,7 +320,7 @@ void HashedDictionary::createAttributes()
hierarchical_attribute = &attributes.back(); hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64) if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH}; throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
} }
} }
} }
@ -424,7 +427,7 @@ void HashedDictionary::loadData()
updateData(); updateData();
if (require_nonempty && 0 == element_count) if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY}; throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
} }
template <typename T> template <typename T>
@ -684,7 +687,7 @@ const HashedDictionary::Attribute & HashedDictionary::getAttribute(const std::st
{ {
const auto it = attribute_index_by_name.find(attribute_name); const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name)) if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second]; return attributes[it->second];
} }
@ -768,27 +771,31 @@ BlockInputStreamPtr HashedDictionary::getBlockInputStream(const Names & column_n
void registerDictionaryHashed(DictionaryFactory & factory) void registerDictionaryHashed(DictionaryFactory & factory)
{ {
auto create_layout = [=](const std::string & name, auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct, const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const std::string & config_prefix,
DictionarySourcePtr source_ptr) -> DictionaryPtr DictionarySourcePtr source_ptr,
bool sparse) -> DictionaryPtr
{ {
if (dict_struct.key) if (dict_struct.key)
throw Exception{"'key' is not supported for dictionary of layout 'hashed'", ErrorCodes::UNSUPPORTED_METHOD}; throw Exception{"'key' is not supported for dictionary of layout 'hashed'", ErrorCodes::UNSUPPORTED_METHOD};
if (dict_struct.range_min || dict_struct.range_max) if (dict_struct.range_min || dict_struct.range_max)
throw Exception{name throw Exception{full_name
+ ": elements .structure.range_min and .structure.range_max should be defined only " + ": elements .structure.range_min and .structure.range_max should be defined only "
"for a dictionary of layout 'range_hashed'", "for a dictionary of layout 'range_hashed'",
ErrorCodes::BAD_ARGUMENTS}; ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
const bool sparse = name == "sparse_hashed"; return std::make_unique<HashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
return std::make_unique<HashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
}; };
factory.registerLayout("hashed", create_layout, false); using namespace std::placeholders;
factory.registerLayout("sparse_hashed", create_layout, false); factory.registerLayout("hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ false), false);
factory.registerLayout("sparse_hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ true), false);
} }
} }

View File

@ -26,6 +26,7 @@ class HashedDictionary final : public IDictionary
{ {
public: public:
HashedDictionary( HashedDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
@ -34,7 +35,9 @@ public:
bool sparse_, bool sparse_,
BlockPtr saved_block_ = nullptr); BlockPtr saved_block_ = nullptr);
std::string getName() const override { return name; } const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return sparse ? "SparseHashed" : "Hashed"; } std::string getTypeName() const override { return sparse ? "SparseHashed" : "Hashed"; }
@ -50,7 +53,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override std::shared_ptr<const IExternalLoadable> clone() const override
{ {
return std::make_shared<HashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block); return std::make_shared<HashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
} }
const IDictionarySource * getSource() const override { return source_ptr.get(); } const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -262,7 +265,9 @@ private:
template <typename ChildType, typename AncestorType> template <typename ChildType, typename AncestorType>
void isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const; void isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
const std::string database;
const std::string name; const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct; const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr; const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime; const DictionaryLifetime dict_lifetime;

View File

@ -25,6 +25,11 @@ struct IDictionaryBase : public IExternalLoadable
{ {
using Key = UInt64; using Key = UInt64;
virtual const std::string & getDatabase() const = 0;
virtual const std::string & getName() const = 0;
virtual const std::string & getFullName() const = 0;
const std::string & getLoadableName() const override { return getFullName(); }
virtual std::string getTypeName() const = 0; virtual std::string getTypeName() const = 0;
virtual size_t getBytesAllocated() const = 0; virtual size_t getBytesAllocated() const = 0;

View File

@ -68,12 +68,15 @@ static bool operator<(const RangeHashedDictionary::Range & left, const RangeHash
RangeHashedDictionary::RangeHashedDictionary( RangeHashedDictionary::RangeHashedDictionary(
const std::string & dictionary_name_, const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_, const DictionaryLifetime dict_lifetime_,
bool require_nonempty_) bool require_nonempty_)
: dictionary_name{dictionary_name_} : database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_) , dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)} , source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_) , dict_lifetime(dict_lifetime_)
@ -156,7 +159,7 @@ void RangeHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value)); attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical) if (attribute.hierarchical)
throw Exception{dictionary_name + ": hierarchical attributes not supported by " + getName() + " dictionary.", throw Exception{full_name + ": hierarchical attributes not supported by " + getName() + " dictionary.",
ErrorCodes::BAD_ARGUMENTS}; ErrorCodes::BAD_ARGUMENTS};
} }
} }
@ -207,7 +210,7 @@ void RangeHashedDictionary::loadData()
stream->readSuffix(); stream->readSuffix();
if (require_nonempty && 0 == element_count) if (require_nonempty && 0 == element_count)
throw Exception{dictionary_name + ": dictionary source is empty and 'require_nonempty' property is set.", throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.",
ErrorCodes::DICTIONARY_IS_EMPTY}; ErrorCodes::DICTIONARY_IS_EMPTY};
} }
@ -520,7 +523,7 @@ const RangeHashedDictionary::Attribute & RangeHashedDictionary::getAttribute(con
{ {
const auto it = attribute_index_by_name.find(attribute_name); const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name)) if (it == std::end(attribute_index_by_name))
throw Exception{dictionary_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second]; return attributes[it->second];
} }
@ -674,7 +677,7 @@ BlockInputStreamPtr RangeHashedDictionary::getBlockInputStream(const Names & col
void registerDictionaryRangeHashed(DictionaryFactory & factory) void registerDictionaryRangeHashed(DictionaryFactory & factory)
{ {
auto create_layout = [=](const std::string & name, auto create_layout = [=](const std::string & full_name,
const DictionaryStructure & dict_struct, const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const std::string & config_prefix,
@ -684,12 +687,14 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory)
throw Exception{"'key' is not supported for dictionary of layout 'range_hashed'", ErrorCodes::UNSUPPORTED_METHOD}; throw Exception{"'key' is not supported for dictionary of layout 'range_hashed'", ErrorCodes::UNSUPPORTED_METHOD};
if (!dict_struct.range_min || !dict_struct.range_max) if (!dict_struct.range_min || !dict_struct.range_max)
throw Exception{name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max", throw Exception{full_name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max",
ErrorCodes::BAD_ARGUMENTS}; ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
return std::make_unique<RangeHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); return std::make_unique<RangeHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
}; };
factory.registerLayout("range_hashed", create_layout, false); factory.registerLayout("range_hashed", create_layout, false);
} }

View File

@ -18,13 +18,16 @@ class RangeHashedDictionary final : public IDictionaryBase
{ {
public: public:
RangeHashedDictionary( RangeHashedDictionary(
const std::string & dictionary_name_, const std::string & database_,
const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_, const DictionaryLifetime dict_lifetime_,
bool require_nonempty_); bool require_nonempty_);
std::string getName() const override { return dictionary_name; } const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "RangeHashed"; } std::string getTypeName() const override { return "RangeHashed"; }
@ -40,7 +43,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override std::shared_ptr<const IExternalLoadable> clone() const override
{ {
return std::make_shared<RangeHashedDictionary>(dictionary_name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty); return std::make_shared<RangeHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
} }
const IDictionarySource * getSource() const override { return source_ptr.get(); } const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -208,7 +211,9 @@ private:
friend struct RangeHashedDIctionaryCallGetBlockInputStreamImpl; friend struct RangeHashedDIctionaryCallGetBlockInputStreamImpl;
const std::string dictionary_name; const std::string database;
const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct; const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr; const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime; const DictionaryLifetime dict_lifetime;

View File

@ -35,12 +35,15 @@ namespace ErrorCodes
} }
TrieDictionary::TrieDictionary( TrieDictionary::TrieDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
const DictionaryLifetime dict_lifetime_, const DictionaryLifetime dict_lifetime_,
bool require_nonempty_) bool require_nonempty_)
: name{name_} : database(database_)
, name(name_)
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
, dict_struct(dict_struct_) , dict_struct(dict_struct_)
, source_ptr{std::move(source_ptr_)} , source_ptr{std::move(source_ptr_)}
, dict_lifetime(dict_lifetime_) , dict_lifetime(dict_lifetime_)
@ -75,7 +78,7 @@ TrieDictionary::~TrieDictionary()
validateKeyTypes(key_types); \ validateKeyTypes(key_types); \
\ \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
const auto null_value = std::get<TYPE>(attribute.null_values); \ const auto null_value = std::get<TYPE>(attribute.null_values); \
\ \
@ -107,7 +110,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
const auto & null_value = StringRef{std::get<String>(attribute.null_values)}; const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -129,7 +132,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); \ validateKeyTypes(key_types); \
\ \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
getItemsImpl<TYPE, TYPE>( \ getItemsImpl<TYPE, TYPE>( \
attribute, \ attribute, \
@ -163,7 +166,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>( getItemsImpl<StringRef, StringRef>(
attribute, attribute,
@ -183,7 +186,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); \ validateKeyTypes(key_types); \
\ \
const auto & attribute = getAttribute(attribute_name); \ const auto & attribute = getAttribute(attribute_name); \
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \ checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
\ \
getItemsImpl<TYPE, TYPE>( \ getItemsImpl<TYPE, TYPE>( \
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \ attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
@ -214,7 +217,7 @@ void TrieDictionary::getString(
validateKeyTypes(key_types); validateKeyTypes(key_types);
const auto & attribute = getAttribute(attribute_name); const auto & attribute = getAttribute(attribute_name);
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString); checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
getItemsImpl<StringRef, StringRef>( getItemsImpl<StringRef, StringRef>(
attribute, attribute,
@ -291,7 +294,7 @@ void TrieDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value)); attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical) if (attribute.hierarchical)
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(), throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH}; ErrorCodes::TYPE_MISMATCH};
} }
} }
@ -337,7 +340,7 @@ void TrieDictionary::loadData()
stream->readSuffix(); stream->readSuffix();
if (require_nonempty && 0 == element_count) if (require_nonempty && 0 == element_count)
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY}; throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
} }
template <typename T> template <typename T>
@ -627,7 +630,7 @@ const TrieDictionary::Attribute & TrieDictionary::getAttribute(const std::string
{ {
const auto it = attribute_index_by_name.find(attribute_name); const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name)) if (it == std::end(attribute_index_by_name))
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second]; return attributes[it->second];
} }
@ -767,7 +770,7 @@ BlockInputStreamPtr TrieDictionary::getBlockInputStream(const Names & column_nam
void registerDictionaryTrie(DictionaryFactory & factory) void registerDictionaryTrie(DictionaryFactory & factory)
{ {
auto create_layout = [=](const std::string & name, auto create_layout = [=](const std::string &,
const DictionaryStructure & dict_struct, const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const std::string & config_prefix,
@ -776,10 +779,12 @@ void registerDictionaryTrie(DictionaryFactory & factory)
if (!dict_struct.key) if (!dict_struct.key)
throw Exception{"'key' is required for dictionary of layout 'ip_trie'", ErrorCodes::BAD_ARGUMENTS}; throw Exception{"'key' is required for dictionary of layout 'ip_trie'", ErrorCodes::BAD_ARGUMENTS};
const String database = config.getString(config_prefix + ".database", "");
const String name = config.getString(config_prefix + ".name");
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
// This is specialised trie for storing IPv4 and IPv6 prefixes. // This is specialised trie for storing IPv4 and IPv6 prefixes.
return std::make_unique<TrieDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); return std::make_unique<TrieDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
}; };
factory.registerLayout("ip_trie", create_layout, true); factory.registerLayout("ip_trie", create_layout, true);
} }

View File

@ -23,6 +23,7 @@ class TrieDictionary final : public IDictionaryBase
{ {
public: public:
TrieDictionary( TrieDictionary(
const std::string & database_,
const std::string & name_, const std::string & name_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
DictionarySourcePtr source_ptr_, DictionarySourcePtr source_ptr_,
@ -33,7 +34,9 @@ public:
std::string getKeyDescription() const { return key_description; } std::string getKeyDescription() const { return key_description; }
std::string getName() const override { return name; } const std::string & getDatabase() const override { return database; }
const std::string & getName() const override { return name; }
const std::string & getFullName() const override { return full_name; }
std::string getTypeName() const override { return "Trie"; } std::string getTypeName() const override { return "Trie"; }
@ -49,7 +52,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override std::shared_ptr<const IExternalLoadable> clone() const override
{ {
return std::make_shared<TrieDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty); return std::make_shared<TrieDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
} }
const IDictionarySource * getSource() const override { return source_ptr.get(); } const IDictionarySource * getSource() const override { return source_ptr.get(); }
@ -232,7 +235,9 @@ private:
Columns getKeyColumns() const; Columns getKeyColumns() const;
const std::string database;
const std::string name; const std::string name;
const std::string full_name;
const DictionaryStructure dict_struct; const DictionaryStructure dict_struct;
const DictionarySourcePtr source_ptr; const DictionarySourcePtr source_ptr;
const DictionaryLifetime dict_lifetime; const DictionaryLifetime dict_lifetime;

View File

@ -421,7 +421,7 @@ void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs
} }
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name) DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query)
{ {
checkAST(query); checkAST(query);
@ -434,10 +434,14 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer
AutoPtr<Poco::XML::Element> name_element(xml_document->createElement("name")); AutoPtr<Poco::XML::Element> name_element(xml_document->createElement("name"));
current_dictionary->appendChild(name_element); current_dictionary->appendChild(name_element);
String full_name = (!database_name.empty() ? database_name : query.database) + "." + query.table; AutoPtr<Text> name(xml_document->createTextNode(query.table));
AutoPtr<Text> name(xml_document->createTextNode(full_name));
name_element->appendChild(name); name_element->appendChild(name);
AutoPtr<Poco::XML::Element> database_element(xml_document->createElement("database"));
current_dictionary->appendChild(database_element);
AutoPtr<Text> database(xml_document->createTextNode(query.database));
database_element->appendChild(database);
AutoPtr<Element> structure_element(xml_document->createElement("structure")); AutoPtr<Element> structure_element(xml_document->createElement("structure"));
current_dictionary->appendChild(structure_element); current_dictionary->appendChild(structure_element);
Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key); Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key);

View File

@ -10,6 +10,6 @@ using DictionaryConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfigurati
/// Convert dictionary AST to Poco::AbstractConfiguration /// Convert dictionary AST to Poco::AbstractConfiguration
/// This function is necessary because all loadable objects configuration are Poco::AbstractConfiguration /// This function is necessary because all loadable objects configuration are Poco::AbstractConfiguration
/// Can throw exception if query is ill-formed /// Can throw exception if query is ill-formed
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name = {}); DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query);
} }

View File

@ -57,7 +57,8 @@ TEST(ConvertDictionaryAST, SimpleDictConfiguration)
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create);
/// name /// name
EXPECT_EQ(config->getString("dictionary.name"), "test.dict1"); EXPECT_EQ(config->getString("dictionary.database"), "test");
EXPECT_EQ(config->getString("dictionary.name"), "dict1");
/// lifetime /// lifetime
EXPECT_EQ(config->getInt("dictionary.lifetime.min"), 1); EXPECT_EQ(config->getInt("dictionary.lifetime.min"), 1);

View File

@ -127,10 +127,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>()); auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get(); const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName())) if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{ {
throw Exception{"For function " + getName() + ", cannot access dictionary " throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED}; + dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
} }
if (!executeDispatchSimple<FlatDictionary>(block, arguments, result, dict_ptr) && if (!executeDispatchSimple<FlatDictionary>(block, arguments, result, dict_ptr) &&
@ -302,10 +302,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>()); auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get(); const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName())) if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{ {
throw Exception{"For function " + getName() + ", cannot access dictionary " throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED}; + dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
} }
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) && if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
@ -488,10 +488,10 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>()); auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get(); const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName())) if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{ {
throw Exception{"For function " + getName() + ", cannot access dictionary " throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED}; + dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
} }
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) && if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&

View File

@ -60,7 +60,7 @@ public:
const ExternalLoadableLifetime & getLifetime() const override; const ExternalLoadableLifetime & getLifetime() const override;
std::string getName() const override { return name; } const std::string & getLoadableName() const override { return name; }
bool supportUpdates() const override { return true; } bool supportUpdates() const override { return true; }
@ -69,7 +69,7 @@ public:
std::shared_ptr<const IExternalLoadable> clone() const override; std::shared_ptr<const IExternalLoadable> clone() const override;
private: private:
std::string name; const std::string name;
std::string model_path; std::string model_path;
std::string lib_path; std::string lib_path;
ExternalLoadableLifetime lifetime; ExternalLoadableLifetime lifetime;

View File

@ -27,7 +27,7 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create(
void ExternalDictionariesLoader::addConfigRepository( void ExternalDictionariesLoader::addConfigRepository(
const std::string & repository_name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository) const std::string & repository_name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
{ {
ExternalLoader::addConfigRepository(repository_name, std::move(config_repository), {"dictionary", "name"}); ExternalLoader::addConfigRepository(repository_name, std::move(config_repository), {"dictionary", "name", "database"});
} }
} }

View File

@ -293,6 +293,12 @@ private:
continue; continue;
} }
String database;
if (!settings.external_database.empty())
database = file_contents->getString(key + "." + settings.external_database, "");
if (!database.empty())
object_name = database + "." + object_name;
object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}}); object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}});
} }
@ -613,7 +619,7 @@ public:
} }
catch (...) catch (...)
{ {
tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getName() + "' was modified"); tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getLoadableName() + "' was modified");
/// Cannot check isModified, so update /// Cannot check isModified, so update
should_update_flag = true; should_update_flag = true;
} }

View File

@ -24,6 +24,7 @@ struct ExternalLoaderConfigSettings
{ {
std::string external_config; std::string external_config;
std::string external_name; std::string external_name;
std::string external_database;
}; };

View File

@ -41,6 +41,6 @@ std::shared_ptr<const IExternalLoadable> ExternalModelsLoader::create(
void ExternalModelsLoader::addConfigRepository(const String & name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository) void ExternalModelsLoader::addConfigRepository(const String & name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
{ {
ExternalLoader::addConfigRepository(name, std::move(config_repository), {"models", "name"}); ExternalLoader::addConfigRepository(name, std::move(config_repository), {"models", "name", {}});
} }
} }

View File

@ -36,7 +36,7 @@ public:
virtual const ExternalLoadableLifetime & getLifetime() const = 0; virtual const ExternalLoadableLifetime & getLifetime() const = 0;
virtual std::string getName() const = 0; virtual const std::string & getLoadableName() const = 0;
/// True if object can be updated when lifetime exceeded. /// True if object can be updated when lifetime exceeded.
virtual bool supportUpdates() const = 0; virtual bool supportUpdates() const = 0;
/// If lifetime exceeded and isModified(), ExternalLoader replace current object with the result of clone(). /// If lifetime exceeded and isModified(), ExternalLoader replace current object with the result of clone().

View File

@ -691,7 +691,9 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create)
String dictionary_name = create.table; String dictionary_name = create.table;
String database_name = !create.database.empty() ? create.database : context.getCurrentDatabase(); if (create.database.empty())
create.database = context.getCurrentDatabase();
const String & database_name = create.database;
auto guard = context.getDDLGuard(database_name, dictionary_name); auto guard = context.getDDLGuard(database_name, dictionary_name);
DatabasePtr database = context.getDatabase(database_name); DatabasePtr database = context.getDatabase(database_name);

View File

@ -53,16 +53,25 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
if (startsWith(load_result.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX)) if (startsWith(load_result.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX))
continue; continue;
size_t i = 0; const auto dict_ptr = std::dynamic_pointer_cast<const IDictionaryBase>(load_result.object);
String database;
String short_name = load_result.name;
if (!load_result.repository_name.empty() && startsWith(load_result.name, load_result.repository_name + ".")) String database, short_name;
if (dict_ptr)
{ {
database = load_result.repository_name; database = dict_ptr->getDatabase();
short_name = load_result.name.substr(load_result.repository_name.length() + 1); short_name = dict_ptr->getName();
}
else
{
short_name = load_result.name;
if (!load_result.repository_name.empty() && startsWith(short_name, load_result.repository_name + "."))
{
database = load_result.repository_name;
short_name = short_name.substr(database.length() + 1);
}
} }
size_t i = 0;
res_columns[i++]->insert(database); res_columns[i++]->insert(database);
res_columns[i++]->insert(short_name); res_columns[i++]->insert(short_name);
res_columns[i++]->insert(static_cast<Int8>(load_result.status)); res_columns[i++]->insert(static_cast<Int8>(load_result.status));
@ -70,7 +79,6 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
std::exception_ptr last_exception = load_result.exception; std::exception_ptr last_exception = load_result.exception;
const auto dict_ptr = std::dynamic_pointer_cast<const IDictionaryBase>(load_result.object);
if (dict_ptr) if (dict_ptr)
{ {
res_columns[i++]->insert(dict_ptr->getTypeName()); res_columns[i++]->insert(dict_ptr->getTypeName());