mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 23:52:03 +00:00
Use separate variables for the database and name in dictionaries.
This commit is contained in:
parent
4b4efa3b0b
commit
1fa64a2a86
@ -110,10 +110,9 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S
|
|||||||
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
|
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
|
||||||
+ std::to_string(++counter);
|
+ std::to_string(++counter);
|
||||||
external_loader.addConfigRepository(
|
external_loader.addConfigRepository(
|
||||||
temp_repository_name,
|
temp_repository_name,
|
||||||
std::make_unique<ExternalLoaderPresetConfigRepository>(
|
std::make_unique<ExternalLoaderPresetConfigRepository>(
|
||||||
std::vector{std::pair{dictionary_metadata_tmp_path,
|
std::vector{std::pair{dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>())}}));
|
||||||
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), getDatabaseName())}}));
|
|
||||||
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
|
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
|
||||||
|
|
||||||
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
|
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
|
||||||
|
@ -57,12 +57,15 @@ inline size_t CacheDictionary::getCellIdx(const Key id) const
|
|||||||
|
|
||||||
|
|
||||||
CacheDictionary::CacheDictionary(
|
CacheDictionary::CacheDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
const DictionaryLifetime dict_lifetime_,
|
const DictionaryLifetime dict_lifetime_,
|
||||||
const size_t size_)
|
const size_t size_)
|
||||||
: name{name_}
|
: database(database_)
|
||||||
|
, name(name_)
|
||||||
|
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
|
||||||
, dict_struct(dict_struct_)
|
, dict_struct(dict_struct_)
|
||||||
, source_ptr{std::move(source_ptr_)}
|
, source_ptr{std::move(source_ptr_)}
|
||||||
, dict_lifetime(dict_lifetime_)
|
, dict_lifetime(dict_lifetime_)
|
||||||
@ -73,7 +76,7 @@ CacheDictionary::CacheDictionary(
|
|||||||
, rnd_engine(randomSeed())
|
, rnd_engine(randomSeed())
|
||||||
{
|
{
|
||||||
if (!this->source_ptr->supportsSelectiveLoad())
|
if (!this->source_ptr->supportsSelectiveLoad())
|
||||||
throw Exception{name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
|
throw Exception{full_name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
|
||||||
|
|
||||||
createAttributes();
|
createAttributes();
|
||||||
}
|
}
|
||||||
@ -204,7 +207,7 @@ void CacheDictionary::isInConstantVector(const Key child_id, const PaddedPODArra
|
|||||||
void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
||||||
{
|
{
|
||||||
auto & attribute = getAttribute(attribute_name);
|
auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
|
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||||
|
|
||||||
@ -215,7 +218,7 @@ void CacheDictionary::getString(
|
|||||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
||||||
{
|
{
|
||||||
auto & attribute = getAttribute(attribute_name);
|
auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsString(attribute, ids, out, [&](const size_t row) { return def->getDataAt(row); });
|
getItemsString(attribute, ids, out, [&](const size_t row) { return def->getDataAt(row); });
|
||||||
}
|
}
|
||||||
@ -224,7 +227,7 @@ void CacheDictionary::getString(
|
|||||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
||||||
{
|
{
|
||||||
auto & attribute = getAttribute(attribute_name);
|
auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsString(attribute, ids, out, [&](const size_t) { return StringRef{def}; });
|
getItemsString(attribute, ids, out, [&](const size_t) { return StringRef{def}; });
|
||||||
}
|
}
|
||||||
@ -352,7 +355,7 @@ void CacheDictionary::createAttributes()
|
|||||||
hierarchical_attribute = &attributes.back();
|
hierarchical_attribute = &attributes.back();
|
||||||
|
|
||||||
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
|
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
|
||||||
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
|
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -539,7 +542,7 @@ CacheDictionary::Attribute & CacheDictionary::getAttribute(const std::string & a
|
|||||||
{
|
{
|
||||||
const auto it = attribute_index_by_name.find(attribute_name);
|
const auto it = attribute_index_by_name.find(attribute_name);
|
||||||
if (it == std::end(attribute_index_by_name))
|
if (it == std::end(attribute_index_by_name))
|
||||||
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
return attributes[it->second];
|
return attributes[it->second];
|
||||||
}
|
}
|
||||||
@ -580,7 +583,7 @@ std::exception_ptr CacheDictionary::getLastException() const
|
|||||||
|
|
||||||
void registerDictionaryCache(DictionaryFactory & factory)
|
void registerDictionaryCache(DictionaryFactory & factory)
|
||||||
{
|
{
|
||||||
auto create_layout = [=](const std::string & name,
|
auto create_layout = [=](const std::string & full_name,
|
||||||
const DictionaryStructure & dict_struct,
|
const DictionaryStructure & dict_struct,
|
||||||
const Poco::Util::AbstractConfiguration & config,
|
const Poco::Util::AbstractConfiguration & config,
|
||||||
const std::string & config_prefix,
|
const std::string & config_prefix,
|
||||||
@ -590,22 +593,24 @@ void registerDictionaryCache(DictionaryFactory & factory)
|
|||||||
throw Exception{"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD};
|
throw Exception{"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD};
|
||||||
|
|
||||||
if (dict_struct.range_min || dict_struct.range_max)
|
if (dict_struct.range_min || dict_struct.range_max)
|
||||||
throw Exception{name
|
throw Exception{full_name
|
||||||
+ ": elements .structure.range_min and .structure.range_max should be defined only "
|
+ ": elements .structure.range_min and .structure.range_max should be defined only "
|
||||||
"for a dictionary of layout 'range_hashed'",
|
"for a dictionary of layout 'range_hashed'",
|
||||||
ErrorCodes::BAD_ARGUMENTS};
|
ErrorCodes::BAD_ARGUMENTS};
|
||||||
const auto & layout_prefix = config_prefix + ".layout";
|
const auto & layout_prefix = config_prefix + ".layout";
|
||||||
const auto size = config.getInt(layout_prefix + ".cache.size_in_cells");
|
const auto size = config.getInt(layout_prefix + ".cache.size_in_cells");
|
||||||
if (size == 0)
|
if (size == 0)
|
||||||
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
|
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
|
||||||
|
|
||||||
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
||||||
if (require_nonempty)
|
if (require_nonempty)
|
||||||
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
|
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
|
||||||
ErrorCodes::BAD_ARGUMENTS};
|
ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
|
const String database = config.getString(config_prefix + ".database", "");
|
||||||
|
const String name = config.getString(config_prefix + ".name");
|
||||||
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
||||||
return std::make_unique<CacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
|
return std::make_unique<CacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
|
||||||
};
|
};
|
||||||
factory.registerLayout("cache", create_layout, false);
|
factory.registerLayout("cache", create_layout, false);
|
||||||
}
|
}
|
||||||
|
@ -25,13 +25,16 @@ class CacheDictionary final : public IDictionary
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
CacheDictionary(
|
CacheDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
const DictionaryLifetime dict_lifetime_,
|
const DictionaryLifetime dict_lifetime_,
|
||||||
const size_t size_);
|
const size_t size_);
|
||||||
|
|
||||||
std::string getName() const override { return name; }
|
const std::string & getDatabase() const override { return database; }
|
||||||
|
const std::string & getName() const override { return name; }
|
||||||
|
const std::string & getFullName() const override { return full_name; }
|
||||||
|
|
||||||
std::string getTypeName() const override { return "Cache"; }
|
std::string getTypeName() const override { return "Cache"; }
|
||||||
|
|
||||||
@ -52,7 +55,7 @@ public:
|
|||||||
|
|
||||||
std::shared_ptr<const IExternalLoadable> clone() const override
|
std::shared_ptr<const IExternalLoadable> clone() const override
|
||||||
{
|
{
|
||||||
return std::make_shared<CacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size);
|
return std::make_shared<CacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
||||||
@ -254,7 +257,9 @@ private:
|
|||||||
template <typename AncestorType>
|
template <typename AncestorType>
|
||||||
void isInImpl(const PaddedPODArray<Key> & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
|
void isInImpl(const PaddedPODArray<Key> & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
|
||||||
|
|
||||||
|
const std::string database;
|
||||||
const std::string name;
|
const std::string name;
|
||||||
|
const std::string full_name;
|
||||||
const DictionaryStructure dict_struct;
|
const DictionaryStructure dict_struct;
|
||||||
mutable DictionarySourcePtr source_ptr;
|
mutable DictionarySourcePtr source_ptr;
|
||||||
const DictionaryLifetime dict_lifetime;
|
const DictionaryLifetime dict_lifetime;
|
||||||
|
@ -333,7 +333,7 @@ void CacheDictionary::update(
|
|||||||
last_exception = std::current_exception();
|
last_exception = std::current_exception();
|
||||||
backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count));
|
backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count));
|
||||||
|
|
||||||
tryLogException(last_exception, log, "Could not update cache dictionary '" + getName() +
|
tryLogException(last_exception, log, "Could not update cache dictionary '" + getFullName() +
|
||||||
"', next update is scheduled at " + ext::to_string(backoff_end_time));
|
"', next update is scheduled at " + ext::to_string(backoff_end_time));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -51,12 +51,15 @@ inline UInt64 ComplexKeyCacheDictionary::getCellIdx(const StringRef key) const
|
|||||||
|
|
||||||
|
|
||||||
ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
|
ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
const DictionaryLifetime dict_lifetime_,
|
const DictionaryLifetime dict_lifetime_,
|
||||||
const size_t size_)
|
const size_t size_)
|
||||||
: name{name_}
|
: database(database_)
|
||||||
|
, name(name_)
|
||||||
|
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
|
||||||
, dict_struct(dict_struct_)
|
, dict_struct(dict_struct_)
|
||||||
, source_ptr{std::move(source_ptr_)}
|
, source_ptr{std::move(source_ptr_)}
|
||||||
, dict_lifetime(dict_lifetime_)
|
, dict_lifetime(dict_lifetime_)
|
||||||
@ -65,7 +68,7 @@ ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(
|
|||||||
, rnd_engine(randomSeed())
|
, rnd_engine(randomSeed())
|
||||||
{
|
{
|
||||||
if (!this->source_ptr->supportsSelectiveLoad())
|
if (!this->source_ptr->supportsSelectiveLoad())
|
||||||
throw Exception{name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
|
throw Exception{full_name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
|
||||||
|
|
||||||
createAttributes();
|
createAttributes();
|
||||||
}
|
}
|
||||||
@ -77,7 +80,7 @@ void ComplexKeyCacheDictionary::getString(
|
|||||||
dict_struct.validateKeyTypes(key_types);
|
dict_struct.validateKeyTypes(key_types);
|
||||||
|
|
||||||
auto & attribute = getAttribute(attribute_name);
|
auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
|
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||||
|
|
||||||
@ -94,7 +97,7 @@ void ComplexKeyCacheDictionary::getString(
|
|||||||
dict_struct.validateKeyTypes(key_types);
|
dict_struct.validateKeyTypes(key_types);
|
||||||
|
|
||||||
auto & attribute = getAttribute(attribute_name);
|
auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsString(attribute, key_columns, out, [&](const size_t row) { return def->getDataAt(row); });
|
getItemsString(attribute, key_columns, out, [&](const size_t row) { return def->getDataAt(row); });
|
||||||
}
|
}
|
||||||
@ -109,7 +112,7 @@ void ComplexKeyCacheDictionary::getString(
|
|||||||
dict_struct.validateKeyTypes(key_types);
|
dict_struct.validateKeyTypes(key_types);
|
||||||
|
|
||||||
auto & attribute = getAttribute(attribute_name);
|
auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsString(attribute, key_columns, out, [&](const size_t) { return StringRef{def}; });
|
getItemsString(attribute, key_columns, out, [&](const size_t) { return StringRef{def}; });
|
||||||
}
|
}
|
||||||
@ -249,7 +252,7 @@ void ComplexKeyCacheDictionary::createAttributes()
|
|||||||
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
|
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
|
||||||
|
|
||||||
if (attribute.hierarchical)
|
if (attribute.hierarchical)
|
||||||
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
|
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
|
||||||
ErrorCodes::TYPE_MISMATCH};
|
ErrorCodes::TYPE_MISMATCH};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -258,7 +261,7 @@ ComplexKeyCacheDictionary::Attribute & ComplexKeyCacheDictionary::getAttribute(c
|
|||||||
{
|
{
|
||||||
const auto it = attribute_index_by_name.find(attribute_name);
|
const auto it = attribute_index_by_name.find(attribute_name);
|
||||||
if (it == std::end(attribute_index_by_name))
|
if (it == std::end(attribute_index_by_name))
|
||||||
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
return attributes[it->second];
|
return attributes[it->second];
|
||||||
}
|
}
|
||||||
@ -394,7 +397,7 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
|
|||||||
|
|
||||||
void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
|
void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
|
||||||
{
|
{
|
||||||
auto create_layout = [=](const std::string & name,
|
auto create_layout = [=](const std::string & full_name,
|
||||||
const DictionaryStructure & dict_struct,
|
const DictionaryStructure & dict_struct,
|
||||||
const Poco::Util::AbstractConfiguration & config,
|
const Poco::Util::AbstractConfiguration & config,
|
||||||
const std::string & config_prefix,
|
const std::string & config_prefix,
|
||||||
@ -405,15 +408,17 @@ void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
|
|||||||
const auto & layout_prefix = config_prefix + ".layout";
|
const auto & layout_prefix = config_prefix + ".layout";
|
||||||
const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells");
|
const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells");
|
||||||
if (size == 0)
|
if (size == 0)
|
||||||
throw Exception{name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
|
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
|
||||||
|
|
||||||
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
||||||
if (require_nonempty)
|
if (require_nonempty)
|
||||||
throw Exception{name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
|
throw Exception{full_name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
|
||||||
ErrorCodes::BAD_ARGUMENTS};
|
ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
|
const String database = config.getString(config_prefix + ".database", "");
|
||||||
|
const String name = config.getString(config_prefix + ".name");
|
||||||
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
||||||
return std::make_unique<ComplexKeyCacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
|
return std::make_unique<ComplexKeyCacheDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, size);
|
||||||
};
|
};
|
||||||
factory.registerLayout("complex_key_cache", create_layout, true);
|
factory.registerLayout("complex_key_cache", create_layout, true);
|
||||||
}
|
}
|
||||||
|
@ -42,6 +42,7 @@ class ComplexKeyCacheDictionary final : public IDictionaryBase
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
ComplexKeyCacheDictionary(
|
ComplexKeyCacheDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
@ -50,7 +51,9 @@ public:
|
|||||||
|
|
||||||
std::string getKeyDescription() const { return key_description; }
|
std::string getKeyDescription() const { return key_description; }
|
||||||
|
|
||||||
std::string getName() const override { return name; }
|
const std::string & getDatabase() const override { return database; }
|
||||||
|
const std::string & getName() const override { return name; }
|
||||||
|
const std::string & getFullName() const override { return full_name; }
|
||||||
|
|
||||||
std::string getTypeName() const override { return "ComplexKeyCache"; }
|
std::string getTypeName() const override { return "ComplexKeyCache"; }
|
||||||
|
|
||||||
@ -75,7 +78,7 @@ public:
|
|||||||
|
|
||||||
std::shared_ptr<const IExternalLoadable> clone() const override
|
std::shared_ptr<const IExternalLoadable> clone() const override
|
||||||
{
|
{
|
||||||
return std::make_shared<ComplexKeyCacheDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, size);
|
return std::make_shared<ComplexKeyCacheDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
||||||
@ -668,7 +671,9 @@ private:
|
|||||||
|
|
||||||
bool isEmptyCell(const UInt64 idx) const;
|
bool isEmptyCell(const UInt64 idx) const;
|
||||||
|
|
||||||
|
const std::string database;
|
||||||
const std::string name;
|
const std::string name;
|
||||||
|
const std::string full_name;
|
||||||
const DictionaryStructure dict_struct;
|
const DictionaryStructure dict_struct;
|
||||||
const DictionarySourcePtr source_ptr;
|
const DictionarySourcePtr source_ptr;
|
||||||
const DictionaryLifetime dict_lifetime;
|
const DictionaryLifetime dict_lifetime;
|
||||||
|
@ -15,13 +15,16 @@ namespace ErrorCodes
|
|||||||
}
|
}
|
||||||
|
|
||||||
ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
|
ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
const DictionaryLifetime dict_lifetime_,
|
const DictionaryLifetime dict_lifetime_,
|
||||||
bool require_nonempty_,
|
bool require_nonempty_,
|
||||||
BlockPtr saved_block_)
|
BlockPtr saved_block_)
|
||||||
: name{name_}
|
: database(database_)
|
||||||
|
, name(name_)
|
||||||
|
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
|
||||||
, dict_struct(dict_struct_)
|
, dict_struct(dict_struct_)
|
||||||
, source_ptr{std::move(source_ptr_)}
|
, source_ptr{std::move(source_ptr_)}
|
||||||
, dict_lifetime(dict_lifetime_)
|
, dict_lifetime(dict_lifetime_)
|
||||||
@ -40,7 +43,7 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
|
|||||||
dict_struct.validateKeyTypes(key_types); \
|
dict_struct.validateKeyTypes(key_types); \
|
||||||
\
|
\
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
||||||
\
|
\
|
||||||
@ -72,7 +75,7 @@ void ComplexKeyHashedDictionary::getString(
|
|||||||
dict_struct.validateKeyTypes(key_types);
|
dict_struct.validateKeyTypes(key_types);
|
||||||
|
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||||
|
|
||||||
@ -94,7 +97,7 @@ void ComplexKeyHashedDictionary::getString(
|
|||||||
dict_struct.validateKeyTypes(key_types); \
|
dict_struct.validateKeyTypes(key_types); \
|
||||||
\
|
\
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
getItemsImpl<TYPE, TYPE>( \
|
getItemsImpl<TYPE, TYPE>( \
|
||||||
attribute, \
|
attribute, \
|
||||||
@ -128,7 +131,7 @@ void ComplexKeyHashedDictionary::getString(
|
|||||||
dict_struct.validateKeyTypes(key_types);
|
dict_struct.validateKeyTypes(key_types);
|
||||||
|
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsImpl<StringRef, StringRef>(
|
getItemsImpl<StringRef, StringRef>(
|
||||||
attribute,
|
attribute,
|
||||||
@ -148,7 +151,7 @@ void ComplexKeyHashedDictionary::getString(
|
|||||||
dict_struct.validateKeyTypes(key_types); \
|
dict_struct.validateKeyTypes(key_types); \
|
||||||
\
|
\
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
getItemsImpl<TYPE, TYPE>( \
|
getItemsImpl<TYPE, TYPE>( \
|
||||||
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
||||||
@ -179,7 +182,7 @@ void ComplexKeyHashedDictionary::getString(
|
|||||||
dict_struct.validateKeyTypes(key_types);
|
dict_struct.validateKeyTypes(key_types);
|
||||||
|
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsImpl<StringRef, StringRef>(
|
getItemsImpl<StringRef, StringRef>(
|
||||||
attribute,
|
attribute,
|
||||||
@ -256,7 +259,7 @@ void ComplexKeyHashedDictionary::createAttributes()
|
|||||||
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
|
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
|
||||||
|
|
||||||
if (attribute.hierarchical)
|
if (attribute.hierarchical)
|
||||||
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
|
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
|
||||||
ErrorCodes::TYPE_MISMATCH};
|
ErrorCodes::TYPE_MISMATCH};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -397,7 +400,7 @@ void ComplexKeyHashedDictionary::loadData()
|
|||||||
updateData();
|
updateData();
|
||||||
|
|
||||||
if (require_nonempty && 0 == element_count)
|
if (require_nonempty && 0 == element_count)
|
||||||
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
|
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
@ -630,7 +633,7 @@ const ComplexKeyHashedDictionary::Attribute & ComplexKeyHashedDictionary::getAtt
|
|||||||
{
|
{
|
||||||
const auto it = attribute_index_by_name.find(attribute_name);
|
const auto it = attribute_index_by_name.find(attribute_name);
|
||||||
if (it == std::end(attribute_index_by_name))
|
if (it == std::end(attribute_index_by_name))
|
||||||
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
return attributes[it->second];
|
return attributes[it->second];
|
||||||
}
|
}
|
||||||
@ -742,7 +745,7 @@ BlockInputStreamPtr ComplexKeyHashedDictionary::getBlockInputStream(const Names
|
|||||||
|
|
||||||
void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
|
void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
|
||||||
{
|
{
|
||||||
auto create_layout = [=](const std::string & name,
|
auto create_layout = [=](const std::string &,
|
||||||
const DictionaryStructure & dict_struct,
|
const DictionaryStructure & dict_struct,
|
||||||
const Poco::Util::AbstractConfiguration & config,
|
const Poco::Util::AbstractConfiguration & config,
|
||||||
const std::string & config_prefix,
|
const std::string & config_prefix,
|
||||||
@ -751,12 +754,13 @@ void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
|
|||||||
if (!dict_struct.key)
|
if (!dict_struct.key)
|
||||||
throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
|
const String database = config.getString(config_prefix + ".database", "");
|
||||||
|
const String name = config.getString(config_prefix + ".name");
|
||||||
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
||||||
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
||||||
return std::make_unique<ComplexKeyHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
|
return std::make_unique<ComplexKeyHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
|
||||||
};
|
};
|
||||||
factory.registerLayout("complex_key_hashed", create_layout, true);
|
factory.registerLayout("complex_key_hashed", create_layout, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -23,6 +23,7 @@ class ComplexKeyHashedDictionary final : public IDictionaryBase
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
ComplexKeyHashedDictionary(
|
ComplexKeyHashedDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
@ -32,7 +33,9 @@ public:
|
|||||||
|
|
||||||
std::string getKeyDescription() const { return key_description; }
|
std::string getKeyDescription() const { return key_description; }
|
||||||
|
|
||||||
std::string getName() const override { return name; }
|
const std::string & getDatabase() const override { return database; }
|
||||||
|
const std::string & getName() const override { return name; }
|
||||||
|
const std::string & getFullName() const override { return full_name; }
|
||||||
|
|
||||||
std::string getTypeName() const override { return "ComplexKeyHashed"; }
|
std::string getTypeName() const override { return "ComplexKeyHashed"; }
|
||||||
|
|
||||||
@ -48,7 +51,7 @@ public:
|
|||||||
|
|
||||||
std::shared_ptr<const IExternalLoadable> clone() const override
|
std::shared_ptr<const IExternalLoadable> clone() const override
|
||||||
{
|
{
|
||||||
return std::make_shared<ComplexKeyHashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
|
return std::make_shared<ComplexKeyHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
|
||||||
}
|
}
|
||||||
|
|
||||||
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
||||||
@ -233,7 +236,9 @@ private:
|
|||||||
template <typename T>
|
template <typename T>
|
||||||
std::vector<StringRef> getKeys(const Attribute & attribute) const;
|
std::vector<StringRef> getKeys(const Attribute & attribute) const;
|
||||||
|
|
||||||
|
const std::string database;
|
||||||
const std::string name;
|
const std::string name;
|
||||||
|
const std::string full_name;
|
||||||
const DictionaryStructure dict_struct;
|
const DictionaryStructure dict_struct;
|
||||||
const DictionarySourcePtr source_ptr;
|
const DictionarySourcePtr source_ptr;
|
||||||
const DictionaryLifetime dict_lifetime;
|
const DictionaryLifetime dict_lifetime;
|
||||||
|
@ -21,13 +21,16 @@ static const auto max_array_size = 500000;
|
|||||||
|
|
||||||
|
|
||||||
FlatDictionary::FlatDictionary(
|
FlatDictionary::FlatDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
const DictionaryLifetime dict_lifetime_,
|
const DictionaryLifetime dict_lifetime_,
|
||||||
bool require_nonempty_,
|
bool require_nonempty_,
|
||||||
BlockPtr saved_block_)
|
BlockPtr saved_block_)
|
||||||
: name{name_}
|
: database(database_)
|
||||||
|
, name(name_)
|
||||||
|
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
|
||||||
, dict_struct(dict_struct_)
|
, dict_struct(dict_struct_)
|
||||||
, source_ptr{std::move(source_ptr_)}
|
, source_ptr{std::move(source_ptr_)}
|
||||||
, dict_lifetime(dict_lifetime_)
|
, dict_lifetime(dict_lifetime_)
|
||||||
@ -107,7 +110,7 @@ void FlatDictionary::isInConstantVector(const Key child_id, const PaddedPODArray
|
|||||||
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const \
|
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const \
|
||||||
{ \
|
{ \
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
||||||
\
|
\
|
||||||
@ -133,7 +136,7 @@ DECLARE(Decimal128)
|
|||||||
void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
||||||
{
|
{
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
const auto & null_value = std::get<StringRef>(attribute.null_values);
|
const auto & null_value = std::get<StringRef>(attribute.null_values);
|
||||||
|
|
||||||
@ -152,7 +155,7 @@ void FlatDictionary::getString(const std::string & attribute_name, const PaddedP
|
|||||||
ResultArrayType<TYPE> & out) const \
|
ResultArrayType<TYPE> & out) const \
|
||||||
{ \
|
{ \
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
getItemsImpl<TYPE, TYPE>( \
|
getItemsImpl<TYPE, TYPE>( \
|
||||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
|
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
|
||||||
@ -177,7 +180,7 @@ void FlatDictionary::getString(
|
|||||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
||||||
{
|
{
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsImpl<StringRef, StringRef>(
|
getItemsImpl<StringRef, StringRef>(
|
||||||
attribute,
|
attribute,
|
||||||
@ -191,7 +194,7 @@ void FlatDictionary::getString(
|
|||||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const \
|
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const \
|
||||||
{ \
|
{ \
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
getItemsImpl<TYPE, TYPE>( \
|
getItemsImpl<TYPE, TYPE>( \
|
||||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
||||||
@ -216,7 +219,7 @@ void FlatDictionary::getString(
|
|||||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
||||||
{
|
{
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
FlatDictionary::getItemsImpl<StringRef, StringRef>(
|
FlatDictionary::getItemsImpl<StringRef, StringRef>(
|
||||||
attribute,
|
attribute,
|
||||||
@ -297,7 +300,7 @@ void FlatDictionary::createAttributes()
|
|||||||
hierarchical_attribute = &attributes.back();
|
hierarchical_attribute = &attributes.back();
|
||||||
|
|
||||||
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
|
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
|
||||||
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
|
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -404,7 +407,7 @@ void FlatDictionary::loadData()
|
|||||||
updateData();
|
updateData();
|
||||||
|
|
||||||
if (require_nonempty && 0 == element_count)
|
if (require_nonempty && 0 == element_count)
|
||||||
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
|
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -578,7 +581,7 @@ template <typename T>
|
|||||||
void FlatDictionary::resize(Attribute & attribute, const Key id)
|
void FlatDictionary::resize(Attribute & attribute, const Key id)
|
||||||
{
|
{
|
||||||
if (id >= max_array_size)
|
if (id >= max_array_size)
|
||||||
throw Exception{name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
|
throw Exception{full_name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
|
||||||
|
|
||||||
auto & array = std::get<ContainerType<T>>(attribute.arrays);
|
auto & array = std::get<ContainerType<T>>(attribute.arrays);
|
||||||
if (id >= array.size())
|
if (id >= array.size())
|
||||||
@ -666,7 +669,7 @@ const FlatDictionary::Attribute & FlatDictionary::getAttribute(const std::string
|
|||||||
{
|
{
|
||||||
const auto it = attribute_index_by_name.find(attribute_name);
|
const auto it = attribute_index_by_name.find(attribute_name);
|
||||||
if (it == std::end(attribute_index_by_name))
|
if (it == std::end(attribute_index_by_name))
|
||||||
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
return attributes[it->second];
|
return attributes[it->second];
|
||||||
}
|
}
|
||||||
@ -706,7 +709,7 @@ BlockInputStreamPtr FlatDictionary::getBlockInputStream(const Names & column_nam
|
|||||||
|
|
||||||
void registerDictionaryFlat(DictionaryFactory & factory)
|
void registerDictionaryFlat(DictionaryFactory & factory)
|
||||||
{
|
{
|
||||||
auto create_layout = [=](const std::string & name,
|
auto create_layout = [=](const std::string & full_name,
|
||||||
const DictionaryStructure & dict_struct,
|
const DictionaryStructure & dict_struct,
|
||||||
const Poco::Util::AbstractConfiguration & config,
|
const Poco::Util::AbstractConfiguration & config,
|
||||||
const std::string & config_prefix,
|
const std::string & config_prefix,
|
||||||
@ -716,13 +719,16 @@ void registerDictionaryFlat(DictionaryFactory & factory)
|
|||||||
throw Exception{"'key' is not supported for dictionary of layout 'flat'", ErrorCodes::UNSUPPORTED_METHOD};
|
throw Exception{"'key' is not supported for dictionary of layout 'flat'", ErrorCodes::UNSUPPORTED_METHOD};
|
||||||
|
|
||||||
if (dict_struct.range_min || dict_struct.range_max)
|
if (dict_struct.range_min || dict_struct.range_max)
|
||||||
throw Exception{name
|
throw Exception{full_name
|
||||||
+ ": elements .structure.range_min and .structure.range_max should be defined only "
|
+ ": elements .structure.range_min and .structure.range_max should be defined only "
|
||||||
"for a dictionary of layout 'range_hashed'",
|
"for a dictionary of layout 'range_hashed'",
|
||||||
ErrorCodes::BAD_ARGUMENTS};
|
ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
|
const String database = config.getString(config_prefix + ".database", "");
|
||||||
|
const String name = config.getString(config_prefix + ".name");
|
||||||
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
||||||
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
||||||
return std::make_unique<FlatDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
|
return std::make_unique<FlatDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
|
||||||
};
|
};
|
||||||
factory.registerLayout("flat", create_layout, false);
|
factory.registerLayout("flat", create_layout, false);
|
||||||
}
|
}
|
||||||
|
@ -22,6 +22,7 @@ class FlatDictionary final : public IDictionary
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
FlatDictionary(
|
FlatDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
@ -29,7 +30,9 @@ public:
|
|||||||
bool require_nonempty_,
|
bool require_nonempty_,
|
||||||
BlockPtr saved_block_ = nullptr);
|
BlockPtr saved_block_ = nullptr);
|
||||||
|
|
||||||
std::string getName() const override { return name; }
|
const std::string & getDatabase() const override { return database; }
|
||||||
|
const std::string & getName() const override { return name; }
|
||||||
|
const std::string & getFullName() const override { return full_name; }
|
||||||
|
|
||||||
std::string getTypeName() const override { return "Flat"; }
|
std::string getTypeName() const override { return "Flat"; }
|
||||||
|
|
||||||
@ -45,7 +48,7 @@ public:
|
|||||||
|
|
||||||
std::shared_ptr<const IExternalLoadable> clone() const override
|
std::shared_ptr<const IExternalLoadable> clone() const override
|
||||||
{
|
{
|
||||||
return std::make_shared<FlatDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
|
return std::make_shared<FlatDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block);
|
||||||
}
|
}
|
||||||
|
|
||||||
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
||||||
@ -222,7 +225,9 @@ private:
|
|||||||
|
|
||||||
PaddedPODArray<Key> getIds() const;
|
PaddedPODArray<Key> getIds() const;
|
||||||
|
|
||||||
|
const std::string database;
|
||||||
const std::string name;
|
const std::string name;
|
||||||
|
const std::string full_name;
|
||||||
const DictionaryStructure dict_struct;
|
const DictionaryStructure dict_struct;
|
||||||
const DictionarySourcePtr source_ptr;
|
const DictionarySourcePtr source_ptr;
|
||||||
const DictionaryLifetime dict_lifetime;
|
const DictionaryLifetime dict_lifetime;
|
||||||
|
@ -31,6 +31,7 @@ namespace ErrorCodes
|
|||||||
|
|
||||||
|
|
||||||
HashedDictionary::HashedDictionary(
|
HashedDictionary::HashedDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
@ -38,7 +39,9 @@ HashedDictionary::HashedDictionary(
|
|||||||
bool require_nonempty_,
|
bool require_nonempty_,
|
||||||
bool sparse_,
|
bool sparse_,
|
||||||
BlockPtr saved_block_)
|
BlockPtr saved_block_)
|
||||||
: name{name_}
|
: database(database_)
|
||||||
|
, name(name_)
|
||||||
|
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
|
||||||
, dict_struct(dict_struct_)
|
, dict_struct(dict_struct_)
|
||||||
, source_ptr{std::move(source_ptr_)}
|
, source_ptr{std::move(source_ptr_)}
|
||||||
, dict_lifetime(dict_lifetime_)
|
, dict_lifetime(dict_lifetime_)
|
||||||
@ -129,7 +132,7 @@ void HashedDictionary::isInConstantVector(const Key child_id, const PaddedPODArr
|
|||||||
const \
|
const \
|
||||||
{ \
|
{ \
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
||||||
\
|
\
|
||||||
@ -155,7 +158,7 @@ DECLARE(Decimal128)
|
|||||||
void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
||||||
{
|
{
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||||
|
|
||||||
@ -174,7 +177,7 @@ void HashedDictionary::getString(const std::string & attribute_name, const Padde
|
|||||||
ResultArrayType<TYPE> & out) const \
|
ResultArrayType<TYPE> & out) const \
|
||||||
{ \
|
{ \
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
getItemsImpl<TYPE, TYPE>( \
|
getItemsImpl<TYPE, TYPE>( \
|
||||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
|
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
|
||||||
@ -199,7 +202,7 @@ void HashedDictionary::getString(
|
|||||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
||||||
{
|
{
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsImpl<StringRef, StringRef>(
|
getItemsImpl<StringRef, StringRef>(
|
||||||
attribute,
|
attribute,
|
||||||
@ -213,7 +216,7 @@ void HashedDictionary::getString(
|
|||||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const \
|
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const \
|
||||||
{ \
|
{ \
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
getItemsImpl<TYPE, TYPE>( \
|
getItemsImpl<TYPE, TYPE>( \
|
||||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
||||||
@ -238,7 +241,7 @@ void HashedDictionary::getString(
|
|||||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
||||||
{
|
{
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsImpl<StringRef, StringRef>(
|
getItemsImpl<StringRef, StringRef>(
|
||||||
attribute,
|
attribute,
|
||||||
@ -317,7 +320,7 @@ void HashedDictionary::createAttributes()
|
|||||||
hierarchical_attribute = &attributes.back();
|
hierarchical_attribute = &attributes.back();
|
||||||
|
|
||||||
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
|
if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64)
|
||||||
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
|
throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -424,7 +427,7 @@ void HashedDictionary::loadData()
|
|||||||
updateData();
|
updateData();
|
||||||
|
|
||||||
if (require_nonempty && 0 == element_count)
|
if (require_nonempty && 0 == element_count)
|
||||||
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
|
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
@ -684,7 +687,7 @@ const HashedDictionary::Attribute & HashedDictionary::getAttribute(const std::st
|
|||||||
{
|
{
|
||||||
const auto it = attribute_index_by_name.find(attribute_name);
|
const auto it = attribute_index_by_name.find(attribute_name);
|
||||||
if (it == std::end(attribute_index_by_name))
|
if (it == std::end(attribute_index_by_name))
|
||||||
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
return attributes[it->second];
|
return attributes[it->second];
|
||||||
}
|
}
|
||||||
@ -768,27 +771,31 @@ BlockInputStreamPtr HashedDictionary::getBlockInputStream(const Names & column_n
|
|||||||
|
|
||||||
void registerDictionaryHashed(DictionaryFactory & factory)
|
void registerDictionaryHashed(DictionaryFactory & factory)
|
||||||
{
|
{
|
||||||
auto create_layout = [=](const std::string & name,
|
auto create_layout = [=](const std::string & full_name,
|
||||||
const DictionaryStructure & dict_struct,
|
const DictionaryStructure & dict_struct,
|
||||||
const Poco::Util::AbstractConfiguration & config,
|
const Poco::Util::AbstractConfiguration & config,
|
||||||
const std::string & config_prefix,
|
const std::string & config_prefix,
|
||||||
DictionarySourcePtr source_ptr) -> DictionaryPtr
|
DictionarySourcePtr source_ptr,
|
||||||
|
bool sparse) -> DictionaryPtr
|
||||||
{
|
{
|
||||||
if (dict_struct.key)
|
if (dict_struct.key)
|
||||||
throw Exception{"'key' is not supported for dictionary of layout 'hashed'", ErrorCodes::UNSUPPORTED_METHOD};
|
throw Exception{"'key' is not supported for dictionary of layout 'hashed'", ErrorCodes::UNSUPPORTED_METHOD};
|
||||||
|
|
||||||
if (dict_struct.range_min || dict_struct.range_max)
|
if (dict_struct.range_min || dict_struct.range_max)
|
||||||
throw Exception{name
|
throw Exception{full_name
|
||||||
+ ": elements .structure.range_min and .structure.range_max should be defined only "
|
+ ": elements .structure.range_min and .structure.range_max should be defined only "
|
||||||
"for a dictionary of layout 'range_hashed'",
|
"for a dictionary of layout 'range_hashed'",
|
||||||
ErrorCodes::BAD_ARGUMENTS};
|
ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
|
const String database = config.getString(config_prefix + ".database", "");
|
||||||
|
const String name = config.getString(config_prefix + ".name");
|
||||||
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
||||||
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
||||||
const bool sparse = name == "sparse_hashed";
|
return std::make_unique<HashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
|
||||||
return std::make_unique<HashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty, sparse);
|
|
||||||
};
|
};
|
||||||
factory.registerLayout("hashed", create_layout, false);
|
using namespace std::placeholders;
|
||||||
factory.registerLayout("sparse_hashed", create_layout, false);
|
factory.registerLayout("hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ false), false);
|
||||||
|
factory.registerLayout("sparse_hashed", std::bind(create_layout, _1, _2, _3, _4, _5, /* sparse = */ true), false);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -26,6 +26,7 @@ class HashedDictionary final : public IDictionary
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
HashedDictionary(
|
HashedDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
@ -34,7 +35,9 @@ public:
|
|||||||
bool sparse_,
|
bool sparse_,
|
||||||
BlockPtr saved_block_ = nullptr);
|
BlockPtr saved_block_ = nullptr);
|
||||||
|
|
||||||
std::string getName() const override { return name; }
|
const std::string & getDatabase() const override { return database; }
|
||||||
|
const std::string & getName() const override { return name; }
|
||||||
|
const std::string & getFullName() const override { return full_name; }
|
||||||
|
|
||||||
std::string getTypeName() const override { return sparse ? "SparseHashed" : "Hashed"; }
|
std::string getTypeName() const override { return sparse ? "SparseHashed" : "Hashed"; }
|
||||||
|
|
||||||
@ -50,7 +53,7 @@ public:
|
|||||||
|
|
||||||
std::shared_ptr<const IExternalLoadable> clone() const override
|
std::shared_ptr<const IExternalLoadable> clone() const override
|
||||||
{
|
{
|
||||||
return std::make_shared<HashedDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
|
return std::make_shared<HashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, sparse, saved_block);
|
||||||
}
|
}
|
||||||
|
|
||||||
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
||||||
@ -262,7 +265,9 @@ private:
|
|||||||
template <typename ChildType, typename AncestorType>
|
template <typename ChildType, typename AncestorType>
|
||||||
void isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
|
void isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray<UInt8> & out) const;
|
||||||
|
|
||||||
|
const std::string database;
|
||||||
const std::string name;
|
const std::string name;
|
||||||
|
const std::string full_name;
|
||||||
const DictionaryStructure dict_struct;
|
const DictionaryStructure dict_struct;
|
||||||
const DictionarySourcePtr source_ptr;
|
const DictionarySourcePtr source_ptr;
|
||||||
const DictionaryLifetime dict_lifetime;
|
const DictionaryLifetime dict_lifetime;
|
||||||
|
@ -25,6 +25,11 @@ struct IDictionaryBase : public IExternalLoadable
|
|||||||
{
|
{
|
||||||
using Key = UInt64;
|
using Key = UInt64;
|
||||||
|
|
||||||
|
virtual const std::string & getDatabase() const = 0;
|
||||||
|
virtual const std::string & getName() const = 0;
|
||||||
|
virtual const std::string & getFullName() const = 0;
|
||||||
|
const std::string & getLoadableName() const override { return getFullName(); }
|
||||||
|
|
||||||
virtual std::string getTypeName() const = 0;
|
virtual std::string getTypeName() const = 0;
|
||||||
|
|
||||||
virtual size_t getBytesAllocated() const = 0;
|
virtual size_t getBytesAllocated() const = 0;
|
||||||
|
@ -68,12 +68,15 @@ static bool operator<(const RangeHashedDictionary::Range & left, const RangeHash
|
|||||||
|
|
||||||
|
|
||||||
RangeHashedDictionary::RangeHashedDictionary(
|
RangeHashedDictionary::RangeHashedDictionary(
|
||||||
const std::string & dictionary_name_,
|
const std::string & database_,
|
||||||
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
const DictionaryLifetime dict_lifetime_,
|
const DictionaryLifetime dict_lifetime_,
|
||||||
bool require_nonempty_)
|
bool require_nonempty_)
|
||||||
: dictionary_name{dictionary_name_}
|
: database(database_)
|
||||||
|
, name(name_)
|
||||||
|
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
|
||||||
, dict_struct(dict_struct_)
|
, dict_struct(dict_struct_)
|
||||||
, source_ptr{std::move(source_ptr_)}
|
, source_ptr{std::move(source_ptr_)}
|
||||||
, dict_lifetime(dict_lifetime_)
|
, dict_lifetime(dict_lifetime_)
|
||||||
@ -156,7 +159,7 @@ void RangeHashedDictionary::createAttributes()
|
|||||||
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
|
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
|
||||||
|
|
||||||
if (attribute.hierarchical)
|
if (attribute.hierarchical)
|
||||||
throw Exception{dictionary_name + ": hierarchical attributes not supported by " + getName() + " dictionary.",
|
throw Exception{full_name + ": hierarchical attributes not supported by " + getName() + " dictionary.",
|
||||||
ErrorCodes::BAD_ARGUMENTS};
|
ErrorCodes::BAD_ARGUMENTS};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -207,7 +210,7 @@ void RangeHashedDictionary::loadData()
|
|||||||
stream->readSuffix();
|
stream->readSuffix();
|
||||||
|
|
||||||
if (require_nonempty && 0 == element_count)
|
if (require_nonempty && 0 == element_count)
|
||||||
throw Exception{dictionary_name + ": dictionary source is empty and 'require_nonempty' property is set.",
|
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.",
|
||||||
ErrorCodes::DICTIONARY_IS_EMPTY};
|
ErrorCodes::DICTIONARY_IS_EMPTY};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -520,7 +523,7 @@ const RangeHashedDictionary::Attribute & RangeHashedDictionary::getAttribute(con
|
|||||||
{
|
{
|
||||||
const auto it = attribute_index_by_name.find(attribute_name);
|
const auto it = attribute_index_by_name.find(attribute_name);
|
||||||
if (it == std::end(attribute_index_by_name))
|
if (it == std::end(attribute_index_by_name))
|
||||||
throw Exception{dictionary_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
return attributes[it->second];
|
return attributes[it->second];
|
||||||
}
|
}
|
||||||
@ -674,7 +677,7 @@ BlockInputStreamPtr RangeHashedDictionary::getBlockInputStream(const Names & col
|
|||||||
|
|
||||||
void registerDictionaryRangeHashed(DictionaryFactory & factory)
|
void registerDictionaryRangeHashed(DictionaryFactory & factory)
|
||||||
{
|
{
|
||||||
auto create_layout = [=](const std::string & name,
|
auto create_layout = [=](const std::string & full_name,
|
||||||
const DictionaryStructure & dict_struct,
|
const DictionaryStructure & dict_struct,
|
||||||
const Poco::Util::AbstractConfiguration & config,
|
const Poco::Util::AbstractConfiguration & config,
|
||||||
const std::string & config_prefix,
|
const std::string & config_prefix,
|
||||||
@ -684,12 +687,14 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory)
|
|||||||
throw Exception{"'key' is not supported for dictionary of layout 'range_hashed'", ErrorCodes::UNSUPPORTED_METHOD};
|
throw Exception{"'key' is not supported for dictionary of layout 'range_hashed'", ErrorCodes::UNSUPPORTED_METHOD};
|
||||||
|
|
||||||
if (!dict_struct.range_min || !dict_struct.range_max)
|
if (!dict_struct.range_min || !dict_struct.range_max)
|
||||||
throw Exception{name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max",
|
throw Exception{full_name + ": dictionary of layout 'range_hashed' requires .structure.range_min and .structure.range_max",
|
||||||
ErrorCodes::BAD_ARGUMENTS};
|
ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
|
const String database = config.getString(config_prefix + ".database", "");
|
||||||
|
const String name = config.getString(config_prefix + ".name");
|
||||||
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
||||||
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
||||||
return std::make_unique<RangeHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
|
return std::make_unique<RangeHashedDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
|
||||||
};
|
};
|
||||||
factory.registerLayout("range_hashed", create_layout, false);
|
factory.registerLayout("range_hashed", create_layout, false);
|
||||||
}
|
}
|
||||||
|
@ -18,13 +18,16 @@ class RangeHashedDictionary final : public IDictionaryBase
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
RangeHashedDictionary(
|
RangeHashedDictionary(
|
||||||
const std::string & dictionary_name_,
|
const std::string & database_,
|
||||||
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
const DictionaryLifetime dict_lifetime_,
|
const DictionaryLifetime dict_lifetime_,
|
||||||
bool require_nonempty_);
|
bool require_nonempty_);
|
||||||
|
|
||||||
std::string getName() const override { return dictionary_name; }
|
const std::string & getDatabase() const override { return database; }
|
||||||
|
const std::string & getName() const override { return name; }
|
||||||
|
const std::string & getFullName() const override { return full_name; }
|
||||||
|
|
||||||
std::string getTypeName() const override { return "RangeHashed"; }
|
std::string getTypeName() const override { return "RangeHashed"; }
|
||||||
|
|
||||||
@ -40,7 +43,7 @@ public:
|
|||||||
|
|
||||||
std::shared_ptr<const IExternalLoadable> clone() const override
|
std::shared_ptr<const IExternalLoadable> clone() const override
|
||||||
{
|
{
|
||||||
return std::make_shared<RangeHashedDictionary>(dictionary_name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
|
return std::make_shared<RangeHashedDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
|
||||||
}
|
}
|
||||||
|
|
||||||
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
||||||
@ -208,7 +211,9 @@ private:
|
|||||||
|
|
||||||
friend struct RangeHashedDIctionaryCallGetBlockInputStreamImpl;
|
friend struct RangeHashedDIctionaryCallGetBlockInputStreamImpl;
|
||||||
|
|
||||||
const std::string dictionary_name;
|
const std::string database;
|
||||||
|
const std::string name;
|
||||||
|
const std::string full_name;
|
||||||
const DictionaryStructure dict_struct;
|
const DictionaryStructure dict_struct;
|
||||||
const DictionarySourcePtr source_ptr;
|
const DictionarySourcePtr source_ptr;
|
||||||
const DictionaryLifetime dict_lifetime;
|
const DictionaryLifetime dict_lifetime;
|
||||||
|
@ -35,12 +35,15 @@ namespace ErrorCodes
|
|||||||
}
|
}
|
||||||
|
|
||||||
TrieDictionary::TrieDictionary(
|
TrieDictionary::TrieDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
const DictionaryLifetime dict_lifetime_,
|
const DictionaryLifetime dict_lifetime_,
|
||||||
bool require_nonempty_)
|
bool require_nonempty_)
|
||||||
: name{name_}
|
: database(database_)
|
||||||
|
, name(name_)
|
||||||
|
, full_name{database_.empty() ? name_ : (database_ + "." + name_)}
|
||||||
, dict_struct(dict_struct_)
|
, dict_struct(dict_struct_)
|
||||||
, source_ptr{std::move(source_ptr_)}
|
, source_ptr{std::move(source_ptr_)}
|
||||||
, dict_lifetime(dict_lifetime_)
|
, dict_lifetime(dict_lifetime_)
|
||||||
@ -75,7 +78,7 @@ TrieDictionary::~TrieDictionary()
|
|||||||
validateKeyTypes(key_types); \
|
validateKeyTypes(key_types); \
|
||||||
\
|
\
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
||||||
\
|
\
|
||||||
@ -107,7 +110,7 @@ void TrieDictionary::getString(
|
|||||||
validateKeyTypes(key_types);
|
validateKeyTypes(key_types);
|
||||||
|
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||||
|
|
||||||
@ -129,7 +132,7 @@ void TrieDictionary::getString(
|
|||||||
validateKeyTypes(key_types); \
|
validateKeyTypes(key_types); \
|
||||||
\
|
\
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
getItemsImpl<TYPE, TYPE>( \
|
getItemsImpl<TYPE, TYPE>( \
|
||||||
attribute, \
|
attribute, \
|
||||||
@ -163,7 +166,7 @@ void TrieDictionary::getString(
|
|||||||
validateKeyTypes(key_types);
|
validateKeyTypes(key_types);
|
||||||
|
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsImpl<StringRef, StringRef>(
|
getItemsImpl<StringRef, StringRef>(
|
||||||
attribute,
|
attribute,
|
||||||
@ -183,7 +186,7 @@ void TrieDictionary::getString(
|
|||||||
validateKeyTypes(key_types); \
|
validateKeyTypes(key_types); \
|
||||||
\
|
\
|
||||||
const auto & attribute = getAttribute(attribute_name); \
|
const auto & attribute = getAttribute(attribute_name); \
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::ut##TYPE); \
|
||||||
\
|
\
|
||||||
getItemsImpl<TYPE, TYPE>( \
|
getItemsImpl<TYPE, TYPE>( \
|
||||||
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
||||||
@ -214,7 +217,7 @@ void TrieDictionary::getString(
|
|||||||
validateKeyTypes(key_types);
|
validateKeyTypes(key_types);
|
||||||
|
|
||||||
const auto & attribute = getAttribute(attribute_name);
|
const auto & attribute = getAttribute(attribute_name);
|
||||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
checkAttributeType(full_name, attribute_name, attribute.type, AttributeUnderlyingType::utString);
|
||||||
|
|
||||||
getItemsImpl<StringRef, StringRef>(
|
getItemsImpl<StringRef, StringRef>(
|
||||||
attribute,
|
attribute,
|
||||||
@ -291,7 +294,7 @@ void TrieDictionary::createAttributes()
|
|||||||
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
|
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
|
||||||
|
|
||||||
if (attribute.hierarchical)
|
if (attribute.hierarchical)
|
||||||
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
|
throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
|
||||||
ErrorCodes::TYPE_MISMATCH};
|
ErrorCodes::TYPE_MISMATCH};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -337,7 +340,7 @@ void TrieDictionary::loadData()
|
|||||||
stream->readSuffix();
|
stream->readSuffix();
|
||||||
|
|
||||||
if (require_nonempty && 0 == element_count)
|
if (require_nonempty && 0 == element_count)
|
||||||
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
|
throw Exception{full_name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
@ -627,7 +630,7 @@ const TrieDictionary::Attribute & TrieDictionary::getAttribute(const std::string
|
|||||||
{
|
{
|
||||||
const auto it = attribute_index_by_name.find(attribute_name);
|
const auto it = attribute_index_by_name.find(attribute_name);
|
||||||
if (it == std::end(attribute_index_by_name))
|
if (it == std::end(attribute_index_by_name))
|
||||||
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
return attributes[it->second];
|
return attributes[it->second];
|
||||||
}
|
}
|
||||||
@ -767,7 +770,7 @@ BlockInputStreamPtr TrieDictionary::getBlockInputStream(const Names & column_nam
|
|||||||
|
|
||||||
void registerDictionaryTrie(DictionaryFactory & factory)
|
void registerDictionaryTrie(DictionaryFactory & factory)
|
||||||
{
|
{
|
||||||
auto create_layout = [=](const std::string & name,
|
auto create_layout = [=](const std::string &,
|
||||||
const DictionaryStructure & dict_struct,
|
const DictionaryStructure & dict_struct,
|
||||||
const Poco::Util::AbstractConfiguration & config,
|
const Poco::Util::AbstractConfiguration & config,
|
||||||
const std::string & config_prefix,
|
const std::string & config_prefix,
|
||||||
@ -776,10 +779,12 @@ void registerDictionaryTrie(DictionaryFactory & factory)
|
|||||||
if (!dict_struct.key)
|
if (!dict_struct.key)
|
||||||
throw Exception{"'key' is required for dictionary of layout 'ip_trie'", ErrorCodes::BAD_ARGUMENTS};
|
throw Exception{"'key' is required for dictionary of layout 'ip_trie'", ErrorCodes::BAD_ARGUMENTS};
|
||||||
|
|
||||||
|
const String database = config.getString(config_prefix + ".database", "");
|
||||||
|
const String name = config.getString(config_prefix + ".name");
|
||||||
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"};
|
||||||
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
|
||||||
// This is specialised trie for storing IPv4 and IPv6 prefixes.
|
// This is specialised trie for storing IPv4 and IPv6 prefixes.
|
||||||
return std::make_unique<TrieDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
|
return std::make_unique<TrieDictionary>(database, name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
|
||||||
};
|
};
|
||||||
factory.registerLayout("ip_trie", create_layout, true);
|
factory.registerLayout("ip_trie", create_layout, true);
|
||||||
}
|
}
|
||||||
|
@ -23,6 +23,7 @@ class TrieDictionary final : public IDictionaryBase
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
TrieDictionary(
|
TrieDictionary(
|
||||||
|
const std::string & database_,
|
||||||
const std::string & name_,
|
const std::string & name_,
|
||||||
const DictionaryStructure & dict_struct_,
|
const DictionaryStructure & dict_struct_,
|
||||||
DictionarySourcePtr source_ptr_,
|
DictionarySourcePtr source_ptr_,
|
||||||
@ -33,7 +34,9 @@ public:
|
|||||||
|
|
||||||
std::string getKeyDescription() const { return key_description; }
|
std::string getKeyDescription() const { return key_description; }
|
||||||
|
|
||||||
std::string getName() const override { return name; }
|
const std::string & getDatabase() const override { return database; }
|
||||||
|
const std::string & getName() const override { return name; }
|
||||||
|
const std::string & getFullName() const override { return full_name; }
|
||||||
|
|
||||||
std::string getTypeName() const override { return "Trie"; }
|
std::string getTypeName() const override { return "Trie"; }
|
||||||
|
|
||||||
@ -49,7 +52,7 @@ public:
|
|||||||
|
|
||||||
std::shared_ptr<const IExternalLoadable> clone() const override
|
std::shared_ptr<const IExternalLoadable> clone() const override
|
||||||
{
|
{
|
||||||
return std::make_shared<TrieDictionary>(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
|
return std::make_shared<TrieDictionary>(database, name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
|
||||||
}
|
}
|
||||||
|
|
||||||
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
||||||
@ -232,7 +235,9 @@ private:
|
|||||||
|
|
||||||
Columns getKeyColumns() const;
|
Columns getKeyColumns() const;
|
||||||
|
|
||||||
|
const std::string database;
|
||||||
const std::string name;
|
const std::string name;
|
||||||
|
const std::string full_name;
|
||||||
const DictionaryStructure dict_struct;
|
const DictionaryStructure dict_struct;
|
||||||
const DictionarySourcePtr source_ptr;
|
const DictionarySourcePtr source_ptr;
|
||||||
const DictionaryLifetime dict_lifetime;
|
const DictionaryLifetime dict_lifetime;
|
||||||
|
@ -421,7 +421,7 @@ void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name)
|
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query)
|
||||||
{
|
{
|
||||||
checkAST(query);
|
checkAST(query);
|
||||||
|
|
||||||
@ -434,10 +434,14 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer
|
|||||||
|
|
||||||
AutoPtr<Poco::XML::Element> name_element(xml_document->createElement("name"));
|
AutoPtr<Poco::XML::Element> name_element(xml_document->createElement("name"));
|
||||||
current_dictionary->appendChild(name_element);
|
current_dictionary->appendChild(name_element);
|
||||||
String full_name = (!database_name.empty() ? database_name : query.database) + "." + query.table;
|
AutoPtr<Text> name(xml_document->createTextNode(query.table));
|
||||||
AutoPtr<Text> name(xml_document->createTextNode(full_name));
|
|
||||||
name_element->appendChild(name);
|
name_element->appendChild(name);
|
||||||
|
|
||||||
|
AutoPtr<Poco::XML::Element> database_element(xml_document->createElement("database"));
|
||||||
|
current_dictionary->appendChild(database_element);
|
||||||
|
AutoPtr<Text> database(xml_document->createTextNode(query.database));
|
||||||
|
database_element->appendChild(database);
|
||||||
|
|
||||||
AutoPtr<Element> structure_element(xml_document->createElement("structure"));
|
AutoPtr<Element> structure_element(xml_document->createElement("structure"));
|
||||||
current_dictionary->appendChild(structure_element);
|
current_dictionary->appendChild(structure_element);
|
||||||
Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key);
|
Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key);
|
||||||
|
@ -10,6 +10,6 @@ using DictionaryConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfigurati
|
|||||||
/// Convert dictionary AST to Poco::AbstractConfiguration
|
/// Convert dictionary AST to Poco::AbstractConfiguration
|
||||||
/// This function is necessary because all loadable objects configuration are Poco::AbstractConfiguration
|
/// This function is necessary because all loadable objects configuration are Poco::AbstractConfiguration
|
||||||
/// Can throw exception if query is ill-formed
|
/// Can throw exception if query is ill-formed
|
||||||
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const String & database_name = {});
|
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -57,7 +57,8 @@ TEST(ConvertDictionaryAST, SimpleDictConfiguration)
|
|||||||
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create);
|
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create);
|
||||||
|
|
||||||
/// name
|
/// name
|
||||||
EXPECT_EQ(config->getString("dictionary.name"), "test.dict1");
|
EXPECT_EQ(config->getString("dictionary.database"), "test");
|
||||||
|
EXPECT_EQ(config->getString("dictionary.name"), "dict1");
|
||||||
|
|
||||||
/// lifetime
|
/// lifetime
|
||||||
EXPECT_EQ(config->getInt("dictionary.lifetime.min"), 1);
|
EXPECT_EQ(config->getInt("dictionary.lifetime.min"), 1);
|
||||||
|
@ -127,10 +127,10 @@ private:
|
|||||||
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
|
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
|
||||||
const auto dict_ptr = dict.get();
|
const auto dict_ptr = dict.get();
|
||||||
|
|
||||||
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
|
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
|
||||||
{
|
{
|
||||||
throw Exception{"For function " + getName() + ", cannot access dictionary "
|
throw Exception{"For function " + getName() + ", cannot access dictionary "
|
||||||
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
|
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!executeDispatchSimple<FlatDictionary>(block, arguments, result, dict_ptr) &&
|
if (!executeDispatchSimple<FlatDictionary>(block, arguments, result, dict_ptr) &&
|
||||||
@ -302,10 +302,10 @@ private:
|
|||||||
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
|
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
|
||||||
const auto dict_ptr = dict.get();
|
const auto dict_ptr = dict.get();
|
||||||
|
|
||||||
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
|
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
|
||||||
{
|
{
|
||||||
throw Exception{"For function " + getName() + ", cannot access dictionary "
|
throw Exception{"For function " + getName() + ", cannot access dictionary "
|
||||||
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
|
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
|
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
|
||||||
@ -488,10 +488,10 @@ private:
|
|||||||
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
|
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
|
||||||
const auto dict_ptr = dict.get();
|
const auto dict_ptr = dict.get();
|
||||||
|
|
||||||
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
|
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
|
||||||
{
|
{
|
||||||
throw Exception{"For function " + getName() + ", cannot access dictionary "
|
throw Exception{"For function " + getName() + ", cannot access dictionary "
|
||||||
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
|
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
|
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
|
||||||
|
@ -60,7 +60,7 @@ public:
|
|||||||
|
|
||||||
const ExternalLoadableLifetime & getLifetime() const override;
|
const ExternalLoadableLifetime & getLifetime() const override;
|
||||||
|
|
||||||
std::string getName() const override { return name; }
|
const std::string & getLoadableName() const override { return name; }
|
||||||
|
|
||||||
bool supportUpdates() const override { return true; }
|
bool supportUpdates() const override { return true; }
|
||||||
|
|
||||||
@ -69,7 +69,7 @@ public:
|
|||||||
std::shared_ptr<const IExternalLoadable> clone() const override;
|
std::shared_ptr<const IExternalLoadable> clone() const override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::string name;
|
const std::string name;
|
||||||
std::string model_path;
|
std::string model_path;
|
||||||
std::string lib_path;
|
std::string lib_path;
|
||||||
ExternalLoadableLifetime lifetime;
|
ExternalLoadableLifetime lifetime;
|
||||||
|
@ -27,7 +27,7 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create(
|
|||||||
void ExternalDictionariesLoader::addConfigRepository(
|
void ExternalDictionariesLoader::addConfigRepository(
|
||||||
const std::string & repository_name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
|
const std::string & repository_name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
|
||||||
{
|
{
|
||||||
ExternalLoader::addConfigRepository(repository_name, std::move(config_repository), {"dictionary", "name"});
|
ExternalLoader::addConfigRepository(repository_name, std::move(config_repository), {"dictionary", "name", "database"});
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -293,6 +293,12 @@ private:
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
String database;
|
||||||
|
if (!settings.external_database.empty())
|
||||||
|
database = file_contents->getString(key + "." + settings.external_database, "");
|
||||||
|
if (!database.empty())
|
||||||
|
object_name = database + "." + object_name;
|
||||||
|
|
||||||
object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}});
|
object_configs_from_file.emplace_back(object_name, ObjectConfig{file_contents, key, {}, {}});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -613,7 +619,7 @@ public:
|
|||||||
}
|
}
|
||||||
catch (...)
|
catch (...)
|
||||||
{
|
{
|
||||||
tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getName() + "' was modified");
|
tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getLoadableName() + "' was modified");
|
||||||
/// Cannot check isModified, so update
|
/// Cannot check isModified, so update
|
||||||
should_update_flag = true;
|
should_update_flag = true;
|
||||||
}
|
}
|
||||||
|
@ -24,6 +24,7 @@ struct ExternalLoaderConfigSettings
|
|||||||
{
|
{
|
||||||
std::string external_config;
|
std::string external_config;
|
||||||
std::string external_name;
|
std::string external_name;
|
||||||
|
std::string external_database;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -41,6 +41,6 @@ std::shared_ptr<const IExternalLoadable> ExternalModelsLoader::create(
|
|||||||
|
|
||||||
void ExternalModelsLoader::addConfigRepository(const String & name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
|
void ExternalModelsLoader::addConfigRepository(const String & name, std::unique_ptr<IExternalLoaderConfigRepository> config_repository)
|
||||||
{
|
{
|
||||||
ExternalLoader::addConfigRepository(name, std::move(config_repository), {"models", "name"});
|
ExternalLoader::addConfigRepository(name, std::move(config_repository), {"models", "name", {}});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -36,7 +36,7 @@ public:
|
|||||||
|
|
||||||
virtual const ExternalLoadableLifetime & getLifetime() const = 0;
|
virtual const ExternalLoadableLifetime & getLifetime() const = 0;
|
||||||
|
|
||||||
virtual std::string getName() const = 0;
|
virtual const std::string & getLoadableName() const = 0;
|
||||||
/// True if object can be updated when lifetime exceeded.
|
/// True if object can be updated when lifetime exceeded.
|
||||||
virtual bool supportUpdates() const = 0;
|
virtual bool supportUpdates() const = 0;
|
||||||
/// If lifetime exceeded and isModified(), ExternalLoader replace current object with the result of clone().
|
/// If lifetime exceeded and isModified(), ExternalLoader replace current object with the result of clone().
|
||||||
|
@ -691,7 +691,9 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create)
|
|||||||
|
|
||||||
String dictionary_name = create.table;
|
String dictionary_name = create.table;
|
||||||
|
|
||||||
String database_name = !create.database.empty() ? create.database : context.getCurrentDatabase();
|
if (create.database.empty())
|
||||||
|
create.database = context.getCurrentDatabase();
|
||||||
|
const String & database_name = create.database;
|
||||||
|
|
||||||
auto guard = context.getDDLGuard(database_name, dictionary_name);
|
auto guard = context.getDDLGuard(database_name, dictionary_name);
|
||||||
DatabasePtr database = context.getDatabase(database_name);
|
DatabasePtr database = context.getDatabase(database_name);
|
||||||
|
@ -53,16 +53,25 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
|
|||||||
if (startsWith(load_result.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX))
|
if (startsWith(load_result.repository_name, IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
size_t i = 0;
|
const auto dict_ptr = std::dynamic_pointer_cast<const IDictionaryBase>(load_result.object);
|
||||||
String database;
|
|
||||||
String short_name = load_result.name;
|
|
||||||
|
|
||||||
if (!load_result.repository_name.empty() && startsWith(load_result.name, load_result.repository_name + "."))
|
String database, short_name;
|
||||||
|
if (dict_ptr)
|
||||||
{
|
{
|
||||||
database = load_result.repository_name;
|
database = dict_ptr->getDatabase();
|
||||||
short_name = load_result.name.substr(load_result.repository_name.length() + 1);
|
short_name = dict_ptr->getName();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
short_name = load_result.name;
|
||||||
|
if (!load_result.repository_name.empty() && startsWith(short_name, load_result.repository_name + "."))
|
||||||
|
{
|
||||||
|
database = load_result.repository_name;
|
||||||
|
short_name = short_name.substr(database.length() + 1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t i = 0;
|
||||||
res_columns[i++]->insert(database);
|
res_columns[i++]->insert(database);
|
||||||
res_columns[i++]->insert(short_name);
|
res_columns[i++]->insert(short_name);
|
||||||
res_columns[i++]->insert(static_cast<Int8>(load_result.status));
|
res_columns[i++]->insert(static_cast<Int8>(load_result.status));
|
||||||
@ -70,7 +79,6 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
|
|||||||
|
|
||||||
std::exception_ptr last_exception = load_result.exception;
|
std::exception_ptr last_exception = load_result.exception;
|
||||||
|
|
||||||
const auto dict_ptr = std::dynamic_pointer_cast<const IDictionaryBase>(load_result.object);
|
|
||||||
if (dict_ptr)
|
if (dict_ptr)
|
||||||
{
|
{
|
||||||
res_columns[i++]->insert(dict_ptr->getTypeName());
|
res_columns[i++]->insert(dict_ptr->getTypeName());
|
||||||
|
Loading…
Reference in New Issue
Block a user