diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index 012750bde60..5624f9595d7 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -252,7 +252,7 @@ Strings DictionaryStructure::getKeysNames() const static void checkAttributeKeys(const Poco::Util::AbstractConfiguration::Keys & keys) { static const std::unordered_set valid_keys - = {"name", "type", "expression", "null_value", "hierarchical", "injective", "is_object_id"}; + = {"name", "type", "expression", "null_value", "hierarchical", "bidirectional", "injective", "is_object_id"}; for (const auto & key : keys) { @@ -350,6 +350,7 @@ std::vector DictionaryStructure::getAttributes( } const auto hierarchical = config.getBool(prefix + "hierarchical", false); + const auto bidirectional = config.getBool(prefix + "bidirectional", false); const auto injective = config.getBool(prefix + "injective", false); const auto is_object_id = config.getBool(prefix + "is_object_id", false); @@ -362,6 +363,9 @@ std::vector DictionaryStructure::getAttributes( if (has_hierarchy && hierarchical) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Only one hierarchical attribute supported"); + if (bidirectional && !hierarchical) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bidirectional can only be applied to hierarchical attributes"); + has_hierarchy = has_hierarchy || hierarchical; res_attributes.emplace_back(DictionaryAttribute{ @@ -372,6 +376,7 @@ std::vector DictionaryStructure::getAttributes( expression, null_value, hierarchical, + bidirectional, injective, is_object_id, is_nullable}); diff --git a/src/Dictionaries/DictionaryStructure.h b/src/Dictionaries/DictionaryStructure.h index 50cfba01894..bb4c306affa 100644 --- a/src/Dictionaries/DictionaryStructure.h +++ b/src/Dictionaries/DictionaryStructure.h @@ -67,6 +67,7 @@ struct DictionaryAttribute final const std::string expression; const Field null_value; const bool hierarchical; + const bool bidirectional; const bool injective; const bool is_object_id; const bool is_nullable; diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index ff13cbcdff9..6f7b210d212 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -43,6 +43,7 @@ FlatDictionary::FlatDictionary( { createAttributes(); loadData(); + buildHierarchyParentToChildIndexIfNeeded(); calculateBytesAllocated(); } @@ -246,6 +247,9 @@ ColumnUInt8::Ptr FlatDictionary::isInHierarchy( DictionaryHierarchyParentToChildIndexPtr FlatDictionary::getHierarchyParentToChildIndex() const { + if (hierarchy_parent_to_child_index) + return hierarchy_parent_to_child_index; + size_t hierarchical_attribute_index = *dict_struct.hierarchical_attribute_index; const auto & hierarchical_attribute = attributes[hierarchical_attribute_index]; const ContainerType & parent_keys = std::get>(hierarchical_attribute.container); @@ -406,6 +410,15 @@ void FlatDictionary::loadData() throw Exception(ErrorCodes::DICTIONARY_IS_EMPTY, "{}: dictionary source is empty and 'require_nonempty' property is set.", getFullName()); } +void FlatDictionary::buildHierarchyParentToChildIndexIfNeeded() +{ + if (!dict_struct.hierarchical_attribute_index) + return; + + if (dict_struct.attributes[*dict_struct.hierarchical_attribute_index].bidirectional) + hierarchy_parent_to_child_index = getHierarchyParentToChildIndex(); +} + void FlatDictionary::calculateBytesAllocated() { bytes_allocated += attributes.size() * sizeof(attributes.front()); @@ -445,6 +458,9 @@ void FlatDictionary::calculateBytesAllocated() if (update_field_loaded_block) bytes_allocated += update_field_loaded_block->allocatedBytes(); + if (hierarchy_parent_to_child_index) + bytes_allocated += hierarchy_parent_to_child_index->getSizeInBytes(); + bytes_allocated += string_arena.size(); } diff --git a/src/Dictionaries/FlatDictionary.h b/src/Dictionaries/FlatDictionary.h index dc97ec2a79e..254c7c85375 100644 --- a/src/Dictionaries/FlatDictionary.h +++ b/src/Dictionaries/FlatDictionary.h @@ -140,10 +140,15 @@ private: }; void createAttributes(); + void blockToAttributes(const Block & block); + void updateData(); + void loadData(); + void buildHierarchyParentToChildIndexIfNeeded(); + void calculateBytesAllocated(); Attribute createAttribute(const DictionaryAttribute & attribute); @@ -175,6 +180,7 @@ private: BlockPtr update_field_loaded_block; Arena string_arena; + DictionaryHierarchyParentToChildIndexPtr hierarchy_parent_to_child_index; }; } diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index 3fc06ba5960..7d25e99f9ad 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -37,6 +37,7 @@ HashedArrayDictionary::HashedArrayDictionary( { createAttributes(); loadData(); + buildHierarchyParentToChildIndexIfNeeded(); calculateBytesAllocated(); } @@ -286,6 +287,9 @@ DictionaryHierarchyParentToChildIndexPtr HashedArrayDictionary & parent_keys_container = std::get>(hierarchical_attribute.container); @@ -707,6 +711,16 @@ void HashedArrayDictionary::loadData() getFullName()); } +template +void HashedArrayDictionary::buildHierarchyParentToChildIndexIfNeeded() +{ + if (!dict_struct.hierarchical_attribute_index) + return; + + if (dict_struct.attributes[*dict_struct.hierarchical_attribute_index].bidirectional) + hierarchy_parent_to_child_index = getHierarchyParentToChildIndex(); +} + template void HashedArrayDictionary::calculateBytesAllocated() { @@ -744,10 +758,16 @@ void HashedArrayDictionary::calculateBytesAllocated() bytes_allocated += (*attribute.is_index_null).size(); } - bytes_allocated += string_arena.size(); - if (update_field_loaded_block) bytes_allocated += update_field_loaded_block->allocatedBytes(); + + if (hierarchy_parent_to_child_index) + { + bytes_allocated += hierarchy_parent_to_child_index->getSizeInBytes(); + std::cout << "Hierarchy index size " << hierarchy_parent_to_child_index->getSizeInBytes() << std::endl; + } + + bytes_allocated += string_arena.size(); } template diff --git a/src/Dictionaries/HashedArrayDictionary.h b/src/Dictionaries/HashedArrayDictionary.h index 17bd5c17102..52e63701d61 100644 --- a/src/Dictionaries/HashedArrayDictionary.h +++ b/src/Dictionaries/HashedArrayDictionary.h @@ -176,6 +176,8 @@ private: void loadData(); + void buildHierarchyParentToChildIndexIfNeeded(); + void calculateBytesAllocated(); template @@ -224,6 +226,7 @@ private: BlockPtr update_field_loaded_block; Arena string_arena; + DictionaryHierarchyParentToChildIndexPtr hierarchy_parent_to_child_index; }; extern template class HashedArrayDictionary; diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index 3392ca048db..398664306ae 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -54,6 +54,7 @@ HashedDictionary::HashedDictionary( { createAttributes(); loadData(); + buildHierarchyParentToChildIndexIfNeeded(); calculateBytesAllocated(); } @@ -322,6 +323,9 @@ DictionaryHierarchyParentToChildIndexPtr HashedDictionary & parent_keys = std::get>(hierarchical_attribute.container); @@ -646,6 +650,16 @@ void HashedDictionary::loadData() getFullName()); } +template +void HashedDictionary::buildHierarchyParentToChildIndexIfNeeded() +{ + if (!dict_struct.hierarchical_attribute_index) + return; + + if (dict_struct.attributes[*dict_struct.hierarchical_attribute_index].bidirectional) + hierarchy_parent_to_child_index = getHierarchyParentToChildIndex(); +} + template void HashedDictionary::calculateBytesAllocated() { @@ -699,10 +713,13 @@ void HashedDictionary::calculateBytesAllocated() } } - bytes_allocated += string_arena.size(); - if (update_field_loaded_block) bytes_allocated += update_field_loaded_block->allocatedBytes(); + + if (hierarchy_parent_to_child_index) + bytes_allocated += hierarchy_parent_to_child_index->getSizeInBytes(); + + bytes_allocated += string_arena.size(); } template diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h index 433bb7eef67..75b8ed33a43 100644 --- a/src/Dictionaries/HashedDictionary.h +++ b/src/Dictionaries/HashedDictionary.h @@ -197,6 +197,8 @@ private: void loadData(); + void buildHierarchyParentToChildIndexIfNeeded(); + void calculateBytesAllocated(); template @@ -229,6 +231,7 @@ private: BlockPtr update_field_loaded_block; Arena string_arena; NoAttributesCollectionType no_attributes_container; + DictionaryHierarchyParentToChildIndexPtr hierarchy_parent_to_child_index; }; extern template class HashedDictionary; diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index 28a842ccd8c..edc3c34fe81 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -290,6 +290,14 @@ void buildSingleAttribute( attribute_element->appendChild(hierarchical_element); } + if (dict_attr->bidirectional) + { + AutoPtr bidirectional_element(doc->createElement("bidirectional")); + AutoPtr bidirectional(doc->createTextNode("true")); + bidirectional_element->appendChild(bidirectional); + attribute_element->appendChild(bidirectional_element); + } + if (dict_attr->injective) { AutoPtr injective_element(doc->createElement("injective")); diff --git a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp index e9c50839a98..760b96b0927 100644 --- a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp +++ b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp @@ -58,6 +58,9 @@ void ASTDictionaryAttributeDeclaration::formatImpl(const FormatSettings & settin if (hierarchical) settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "HIERARCHICAL"; + if (bidirectional) + settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "BIDIRECTIONAL"; + if (injective) settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "INJECTIVE"; diff --git a/src/Parsers/ASTDictionaryAttributeDeclaration.h b/src/Parsers/ASTDictionaryAttributeDeclaration.h index b34ebc539ec..b6572e89d16 100644 --- a/src/Parsers/ASTDictionaryAttributeDeclaration.h +++ b/src/Parsers/ASTDictionaryAttributeDeclaration.h @@ -20,6 +20,8 @@ public: ASTPtr expression; /// Is attribute mirrored to the parent identifier bool hierarchical; + /// Is hierarchical attribute bidirectional + bool bidirectional; /// Flag that shows whether the id->attribute image is injective bool injective; /// MongoDB object ID diff --git a/src/Parsers/ParserDictionaryAttributeDeclaration.cpp b/src/Parsers/ParserDictionaryAttributeDeclaration.cpp index a248c9def07..44bb7fb6057 100644 --- a/src/Parsers/ParserDictionaryAttributeDeclaration.cpp +++ b/src/Parsers/ParserDictionaryAttributeDeclaration.cpp @@ -15,6 +15,7 @@ bool ParserDictionaryAttributeDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserKeyword s_default{"DEFAULT"}; ParserKeyword s_expression{"EXPRESSION"}; ParserKeyword s_hierarchical{"HIERARCHICAL"}; + ParserKeyword s_bidirectional{"BIDIRECTIONAL"}; ParserKeyword s_injective{"INJECTIVE"}; ParserKeyword s_is_object_id{"IS_OBJECT_ID"}; ParserLiteral default_parser; @@ -30,6 +31,7 @@ bool ParserDictionaryAttributeDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ASTPtr default_value; ASTPtr expression; bool hierarchical = false; + bool bidirectional = false; bool injective = false; bool is_object_id = false; @@ -63,6 +65,12 @@ bool ParserDictionaryAttributeDeclaration::parseImpl(Pos & pos, ASTPtr & node, E continue; } + if (!bidirectional && s_bidirectional.ignore(pos, expected)) + { + bidirectional = true; + continue; + } + if (!injective && s_injective.ignore(pos, expected)) { injective = true; @@ -101,6 +109,7 @@ bool ParserDictionaryAttributeDeclaration::parseImpl(Pos & pos, ASTPtr & node, E } attribute_declaration->hierarchical = hierarchical; + attribute_declaration->bidirectional = bidirectional; attribute_declaration->injective = injective; attribute_declaration->is_object_id = is_object_id;