From 2cac8d13cc25c105953d175552931cee9cf88523 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 26 Mar 2021 16:57:21 +0300 Subject: [PATCH] Updated tests and documentation --- src/Common/PODArray.h | 23 ++ src/Common/tests/gtest_pod_array.cpp | 13 + src/Dictionaries/HashedDictionary.cpp | 40 +--- src/Dictionaries/HierarchyDictionariesUtils.h | 38 ++- ...ictionary_ssd_cache_dictionary_storage.cpp | 2 - .../gtest_hierarchy_dictionaries_utils.cpp | 225 ++++++++++++++++++ src/Functions/FunctionsExternalDictionaries.h | 133 +++++++---- .../01778_hierarchical_dictionaries.reference | 102 ++++++++ .../01778_hierarchical_dictionaries.sql | 95 ++++++++ 9 files changed, 571 insertions(+), 100 deletions(-) create mode 100644 src/Dictionaries/tests/gtest_hierarchy_dictionaries_utils.cpp create mode 100644 tests/queries/0_stateless/01778_hierarchical_dictionaries.reference create mode 100644 tests/queries/0_stateless/01778_hierarchical_dictionaries.sql diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 57ad3d46177..3d4050fbffa 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -530,6 +530,29 @@ public: this->c_end += bytes_to_copy; } + template + void insertFromItself(iterator from_begin, iterator from_end, TAllocatorParams && ... allocator_params) + { + static_assert(memcpy_can_be_used_for_assignment, std::decay_t>); + + /// Convert iterators to indexes because reserve can invalidate iterators + size_t start_index = from_begin - begin(); + size_t end_index = from_end - begin(); + + assert(start_index <= end_index); + + size_t required_capacity = this->size() + (end_index - start_index); + if (required_capacity > this->capacity()) + this->reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward(allocator_params)...); + + size_t bytes_to_copy = this->byte_size(end_index - start_index); + if (bytes_to_copy) + { + memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); + this->c_end += bytes_to_copy; + } + } + template void insert_assume_reserved(It1 from_begin, It2 from_end) { diff --git a/src/Common/tests/gtest_pod_array.cpp b/src/Common/tests/gtest_pod_array.cpp index 63cf7026757..9cc77b88195 100644 --- a/src/Common/tests/gtest_pod_array.cpp +++ b/src/Common/tests/gtest_pod_array.cpp @@ -33,6 +33,19 @@ TEST(Common, PODArrayInsert) EXPECT_EQ(str, std::string(chars.data(), chars.size())); } +TEST(Common, PODArrayInsertFromItself) +{ + { + PaddedPODArray array { 1 }; + + for (size_t i = 0; i < 3; ++i) + array.insertFromItself(array.begin(), array.end()); + + PaddedPODArray expected {1,1,1,1,1,1,1,1}; + ASSERT_EQ(array,expected); + } +} + TEST(Common, PODPushBackRawMany) { PODArray chars; diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index 0f0c45f9e56..40394370087 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -307,13 +307,7 @@ ColumnPtr HashedDictionary::getHierarchy(ColumnPtr const UInt64 null_value = dictionary_attribute.null_value.get(); const CollectionType & parent_keys_map = std::get>(hierarchical_attribute.container); - auto is_key_valid_func = [&](auto & key) - { - if constexpr (sparse) - return parent_keys_map.find(key) != parent_keys_map.end(); - else - return parent_keys_map.find(key) != nullptr; - }; + auto is_key_valid_func = [&](auto & key) { return parent_keys_map.find(key) != parent_keys_map.end(); }; auto get_parent_func = [&](auto & hierarchy_key) { @@ -321,16 +315,8 @@ ColumnPtr HashedDictionary::getHierarchy(ColumnPtr auto it = parent_keys_map.find(hierarchy_key); - if constexpr (sparse) - { - if (it != parent_keys_map.end()) - result = it->second; - } - else - { - if (it != nullptr) - result = it->getMapped(); - } + if (it != parent_keys_map.end()) + result = getValueFromCell(it); return result; }; @@ -367,13 +353,7 @@ ColumnUInt8::Ptr HashedDictionary::isInHierarchy( const UInt64 null_value = dictionary_attribute.null_value.get(); const CollectionType & parent_keys_map = std::get>(hierarchical_attribute.container); - auto is_key_valid_func = [&](auto & key) - { - if constexpr (sparse) - return parent_keys_map.find(key) != parent_keys_map.end(); - else - return parent_keys_map.find(key) != nullptr; - }; + auto is_key_valid_func = [&](auto & key) { return parent_keys_map.find(key) != parent_keys_map.end(); }; auto get_parent_func = [&](auto & hierarchy_key) { @@ -381,16 +361,8 @@ ColumnUInt8::Ptr HashedDictionary::isInHierarchy( auto it = parent_keys_map.find(hierarchy_key); - if constexpr (sparse) - { - if (it != parent_keys_map.end()) - result = it->second; - } - else - { - if (it != nullptr) - result = it->getMapped(); - } + if (it != parent_keys_map.end()) + result = getValueFromCell(it); return result; }; diff --git a/src/Dictionaries/HierarchyDictionariesUtils.h b/src/Dictionaries/HierarchyDictionariesUtils.h index 8cdcb5f43f2..f8a1252f1b2 100644 --- a/src/Dictionaries/HierarchyDictionariesUtils.h +++ b/src/Dictionaries/HierarchyDictionariesUtils.h @@ -51,8 +51,8 @@ namespace detail * * If hierarchy_null_value will be 0. Requested keys [1, 2, 3, 4, 5]. * Result: [1], [2, 1], [3, 1], [4, 2, 1], [] - * Elements: [1, 2, 1, 3, 4, 4, 2, 1] - * Offsets: [1, 2, 2, 3, 0] + * Elements: [1, 2, 1, 3, 1, 4, 2, 1] + * Offsets: [1, 3, 5, 8, 8] */ template ElementsAndOffsets getHierarchy( @@ -113,14 +113,7 @@ namespace detail size_t end_index = offsets[offset]; current_hierarchy_depth += end_index - start_index; - - /// TODO: Insert part of pod array into itself - while (start_index < end_index) - { - elements.emplace_back(elements[start_index]); - ++start_index; - } - + elements.insertFromItself(elements.begin() + start_index, elements.begin() + end_index); break; } @@ -162,7 +155,7 @@ namespace detail IsKeyValidFunc && is_key_valid_func, GetParentKeyFunc && get_parent_func) { - assert(hierarchy_keys.size() == hierarchy_in_keys.size()); + assert(keys.size() == in_keys.size()); PaddedPODArray result; result.resize_fill(keys.size()); @@ -212,10 +205,15 @@ namespace detail * 4 2 * * Example. Strategy GetAllDescendantsStrategy. - * Requested keys [0, 1, 2, 5]. - * Result: [2, 3, 4], [2, 4], [4], [] - * Elements: [2, 3, 4, 2, 4, 4] - * Offsets: [3, 2, 1, 0] + * Requested keys [0, 1, 2, 3, 4]. + * Result: [1, 2, 3, 4], [2, 2, 4], [4], [], [] + * Elements: [1, 2, 3, 4, 2, 3, 4, 4] + * Offsets: [4, 7, 8, 8, 8] + * + * Example. Strategy GetDescendantsAtSpecificLevelStrategy with level 1. + * Requested keys [0, 1, 2, 3, 4]. + * Result: [1], [2, 3], [4], [], []; + * Offsets: [1, 3, 4, 4, 4]; */ template ElementsAndOffsets getDescendants( @@ -319,13 +317,9 @@ namespace detail } else { - /// TODO: Insert part of pod array - while (range.start_index != range.end_index) - { - descendants.emplace_back(descendants[range.start_index]); - ++range.start_index; - } - + auto insert_start_iterator = descendants.begin() + range.start_index; + auto insert_end_iterator = descendants.begin() + range.end_index; + descendants.insertFromItself(insert_start_iterator, insert_end_iterator); continue; } } diff --git a/src/Dictionaries/tests/gtest_dictionary_ssd_cache_dictionary_storage.cpp b/src/Dictionaries/tests/gtest_dictionary_ssd_cache_dictionary_storage.cpp index 20529e91bd3..9fd9dc9b78c 100644 --- a/src/Dictionaries/tests/gtest_dictionary_ssd_cache_dictionary_storage.cpp +++ b/src/Dictionaries/tests/gtest_dictionary_ssd_cache_dictionary_storage.cpp @@ -1,7 +1,5 @@ #if defined(__linux__) || defined(__FreeBSD__) -#include - #include #include diff --git a/src/Dictionaries/tests/gtest_hierarchy_dictionaries_utils.cpp b/src/Dictionaries/tests/gtest_hierarchy_dictionaries_utils.cpp new file mode 100644 index 00000000000..064f57dfe11 --- /dev/null +++ b/src/Dictionaries/tests/gtest_hierarchy_dictionaries_utils.cpp @@ -0,0 +1,225 @@ +#include + +#include + +#include + +using namespace DB; + +TEST(HierarchyDictionariesUtils, getHierarchy) +{ + { + HashMap child_to_parent; + child_to_parent[1] = 0; + child_to_parent[2] = 1; + child_to_parent[3] = 1; + child_to_parent[4] = 2; + + auto is_key_valid_func = [&](auto key) { return child_to_parent.find(key) != nullptr; }; + + auto get_parent_key_func = [&](auto key) + { + auto it = child_to_parent.find(key); + std::optional value = (it != nullptr ? std::make_optional(it->getMapped()) : std::nullopt); + return value; + }; + + UInt64 hierarchy_null_value_key = 0; + PaddedPODArray keys = {1, 2, 3, 4, 5}; + + auto result = DB::detail::getHierarchy( + keys, + hierarchy_null_value_key, + is_key_valid_func, + get_parent_key_func); + + const auto & actual_elements = result.elements; + const auto & actual_offsets = result.offsets; + + PaddedPODArray expected_elements = {1, 2, 1, 3, 1, 4, 2, 1}; + PaddedPODArray expected_offsets = {1, 3, 5, 8, 8}; + + ASSERT_EQ(actual_elements, expected_elements); + ASSERT_EQ(actual_offsets, expected_offsets); + } + { + HashMap child_to_parent; + child_to_parent[1] = 2; + child_to_parent[2] = 1; + + auto is_key_valid_func = [&](auto key) { return child_to_parent.find(key) != nullptr; }; + + auto get_parent_key_func = [&](auto key) + { + auto it = child_to_parent.find(key); + std::optional value = (it != nullptr ? std::make_optional(it->getMapped()) : std::nullopt); + return value; + }; + + UInt64 hierarchy_null_value_key = 0; + PaddedPODArray keys = {1, 2, 3}; + + auto result = DB::detail::getHierarchy( + keys, + hierarchy_null_value_key, + is_key_valid_func, + get_parent_key_func); + + const auto & actual_elements = result.elements; + const auto & actual_offsets = result.offsets; + + PaddedPODArray expected_elements = {1, 2, 2}; + PaddedPODArray expected_offsets = {2, 3, 3}; + + ASSERT_EQ(actual_elements, expected_elements); + ASSERT_EQ(actual_offsets, expected_offsets); + } +} + +TEST(HierarchyDictionariesUtils, getIsInHierarchy) +{ + { + HashMap child_to_parent; + child_to_parent[1] = 0; + child_to_parent[2] = 1; + child_to_parent[3] = 1; + child_to_parent[4] = 2; + + auto is_key_valid_func = [&](auto key) { return child_to_parent.find(key) != nullptr; }; + + auto get_parent_key_func = [&](auto key) + { + auto it = child_to_parent.find(key); + std::optional value = (it != nullptr ? std::make_optional(it->getMapped()) : std::nullopt); + return value; + }; + + UInt64 hierarchy_null_value_key = 0; + PaddedPODArray keys = {1, 2, 3, 4, 5}; + PaddedPODArray keys_in = {1, 1, 1, 2, 5}; + + PaddedPODArray actual = DB::detail::getIsInHierarchy( + keys, + keys_in, + hierarchy_null_value_key, + is_key_valid_func, + get_parent_key_func); + + PaddedPODArray expected = {1,1,1,1,0}; + + ASSERT_EQ(actual, expected); + } + { + HashMap child_to_parent; + child_to_parent[1] = 2; + child_to_parent[2] = 1; + + auto is_key_valid_func = [&](auto key) + { + return child_to_parent.find(key) != nullptr; + }; + + auto get_parent_key_func = [&](auto key) + { + auto it = child_to_parent.find(key); + std::optional value = (it != nullptr ? std::make_optional(it->getMapped()) : std::nullopt); + return value; + }; + + UInt64 hierarchy_null_value_key = 0; + PaddedPODArray keys = {1, 2, 3}; + PaddedPODArray keys_in = {1, 2, 3}; + + PaddedPODArray actual = DB::detail::getIsInHierarchy( + keys, + keys_in, + hierarchy_null_value_key, + is_key_valid_func, + get_parent_key_func); + + PaddedPODArray expected = {1, 1, 0}; + ASSERT_EQ(actual, expected); + } +} + +TEST(HierarchyDictionariesUtils, getDescendants) +{ + { + HashMap> parent_to_child; + parent_to_child[0].emplace_back(1); + parent_to_child[1].emplace_back(2); + parent_to_child[1].emplace_back(3); + parent_to_child[2].emplace_back(4); + + PaddedPODArray keys = {0, 1, 2, 3, 4}; + + { + auto result = DB::detail::getDescendants( + keys, + parent_to_child, + DB::detail::GetAllDescendantsStrategy()); + + const auto & actual_elements = result.elements; + const auto & actual_offsets = result.offsets; + + PaddedPODArray expected_elements = {1, 2, 3, 4, 2, 3, 4, 4}; + PaddedPODArray expected_offsets = {4, 7, 8, 8, 8}; + + ASSERT_EQ(actual_elements, expected_elements); + ASSERT_EQ(actual_offsets, expected_offsets); + } + { + auto result = DB::detail::getDescendants( + keys, + parent_to_child, + DB::detail::GetDescendantsAtSpecificLevelStrategy{1}); + + const auto & actual_elements = result.elements; + const auto & actual_offsets = result.offsets; + + PaddedPODArray expected_elements = {1, 2, 3, 4}; + PaddedPODArray expected_offsets = {1, 3, 4, 4, 4}; + + ASSERT_EQ(actual_elements, expected_elements); + ASSERT_EQ(actual_offsets, expected_offsets); + } + } + { + HashMap> parent_to_child; + parent_to_child[1].emplace_back(2); + parent_to_child[2].emplace_back(1); + + PaddedPODArray keys = {1, 2, 3}; + + { + auto result = DB::detail::getDescendants( + keys, + parent_to_child, + DB::detail::GetAllDescendantsStrategy()); + + const auto & actual_elements = result.elements; + const auto & actual_offsets = result.offsets; + + PaddedPODArray expected_elements = {2, 1, 1}; + PaddedPODArray expected_offsets = {2, 3, 3}; + + ASSERT_EQ(actual_elements, expected_elements); + ASSERT_EQ(actual_offsets, expected_offsets); + } + { + auto result = DB::detail::getDescendants( + keys, + parent_to_child, + DB::detail::GetDescendantsAtSpecificLevelStrategy{1}); + + const auto & actual_elements = result.elements; + const auto & actual_offsets = result.offsets; + + PaddedPODArray expected_elements = {2, 1}; + PaddedPODArray expected_offsets = {1, 2, 2}; + + ASSERT_EQ(actual_elements, expected_elements); + ASSERT_EQ(actual_offsets, expected_offsets); + } + } +} diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index b57ff369779..0bc22826f4e 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -79,9 +79,13 @@ public: return dict; } - std::shared_ptr getDictionary(const ColumnWithTypeAndName & column) + std::shared_ptr getDictionary(const ColumnPtr & column) { - const auto * dict_name_col = checkAndGetColumnConst(column.column.get()); + const auto * dict_name_col = checkAndGetColumnConst(column.get()); + + if (!dict_name_col) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Expected const String column"); + return getDictionary(dict_name_col->getValue()); } @@ -176,7 +180,7 @@ private: if (input_rows_count == 0) return result_type->createColumn(); - auto dictionary = helper.getDictionary(arguments[0]); + auto dictionary = helper.getDictionary(arguments[0].column); auto dictionary_key_type = dictionary->getKeyType(); const ColumnWithTypeAndName & key_column_with_type = arguments[1]; @@ -716,12 +720,16 @@ private: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of first argument of function ({}). Expected String. Actual type ({})", + getName(), + arguments[0]->getName()); if (!WhichDataType(arguments[1]).isUInt64()) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of second argument of function ({}). Expected UInt64. Actual type ({})", + getName(), + arguments[1]->getName()); return std::make_shared(std::make_shared()); } @@ -733,7 +741,7 @@ private: if (input_rows_count == 0) return result_type->createColumn(); - auto dictionary = helper.getDictionary(arguments[0]); + auto dictionary = helper.getDictionary(arguments[0].column); if (!dictionary->hasHierarchy()) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, @@ -772,16 +780,22 @@ private: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of first argument of function ({}). Expected String. Actual type ({})", + getName(), + arguments[0]->getName()); if (!WhichDataType(arguments[1]).isUInt64()) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of second argument of function ({}). Expected UInt64. Actual type ({})", + getName(), + arguments[1]->getName()); if (!WhichDataType(arguments[2]).isUInt64()) - throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName() - + ", must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of third argument of function ({}). Expected UInt64. Actual type ({})", + getName(), + arguments[2]->getName()); return std::make_shared(); } @@ -793,7 +807,7 @@ private: if (input_rows_count == 0) return result_type->createColumn(); - auto dict = helper.getDictionary(arguments[0]); + auto dict = helper.getDictionary(arguments[0].column); if (!dict->hasHierarchy()) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Dictionary ({}) does not support hierarchy", dict->getFullName()); @@ -826,35 +840,40 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } + bool isDeterministic() const override { return false; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of first argument of function ({}). Expected String. Actual type ({})", + getName(), + arguments[0]->getName()); if (!WhichDataType(arguments[1]).isUInt64()) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of second argument of function ({}). Expected UInt64. Actual type ({})", + getName(), + arguments[1]->getName()); return std::make_shared(std::make_shared()); } - bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); - auto dict = helper.getDictionary(arguments[0]); + auto dictionary = helper.getDictionary(arguments[0].column); - if (!dict->hasHierarchy()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Dictionary ({}) does not support hierarchy", dict->getFullName()); + if (!dictionary->hasHierarchy()) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "Dictionary ({}) does not support hierarchy", + dictionary->getFullName()); - ColumnPtr res = dict->getDescendants(arguments[1].column, std::make_shared(), 1); + ColumnPtr result = dictionary->getDescendants(arguments[1].column, std::make_shared(), 1); - return res; + return result; } mutable FunctionDictHelper helper; @@ -876,43 +895,73 @@ public: String getName() const override { return name; } private: - size_t getNumberOfArguments() const override { return 3; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 2}; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } + bool isDeterministic() const override { return false; } + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { + size_t arguments_size = arguments.size(); + if (arguments_size < 2 || arguments_size > 3) + { + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Illegal arguments size of function ({}). Expects 2 or 3 arguments size. Actual size ({})", + getName(), + arguments_size); + } + if (!isString(arguments[0])) - throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of first argument of function ({}). Expected const String. Actual type ({})", + getName(), + arguments[0]->getName()); if (!WhichDataType(arguments[1]).isUInt64()) - throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + ", must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of second argument of function ({}). Expected UInt64. Actual type ({})", + getName(), + arguments[1]->getName()); - if (!isUnsignedInteger(arguments[2])) - throw Exception{"Illegal type " + arguments[1]->getName() + " of third argument of function " + getName() - + ", must be const unsigned integer.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + if (arguments.size() == 3 && !isUnsignedInteger(arguments[2])) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of third argument of function ({}). Expected const unsigned integer. Actual type ({})", + getName(), + arguments[2]->getName()); + } return std::make_shared(std::make_shared()); } - bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); - size_t level = static_cast(arguments[2].column->get64(0)); + auto dictionary = helper.getDictionary(arguments[0].column); - auto dict = helper.getDictionary(arguments[0]); + size_t level = 0; - if (!dict->hasHierarchy()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Dictionary ({}) does not support hierarchy", dict->getFullName()); + if (arguments.size() == 3) + { + if (!isColumnConst(*arguments[2].column)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of third argument of function ({}). Expected const unsigned integer.", + getName()); - ColumnPtr res = dict->getDescendants(arguments[1].column, std::make_shared(), level); + level = static_cast(arguments[2].column->get64(0)); + } + + if (!dictionary->hasHierarchy()) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "Dictionary ({}) does not support hierarchy", + dictionary->getFullName()); + + ColumnPtr res = dictionary->getDescendants(arguments[1].column, std::make_shared(), level); return res; } diff --git a/tests/queries/0_stateless/01778_hierarchical_dictionaries.reference b/tests/queries/0_stateless/01778_hierarchical_dictionaries.reference new file mode 100644 index 00000000000..48245356ced --- /dev/null +++ b/tests/queries/0_stateless/01778_hierarchical_dictionaries.reference @@ -0,0 +1,102 @@ +Flat dictionary +Get hierarchy +[] +[1] +[2,1] +[3,1] +[4,2,1] +[] +Get is in hierarchy +0 +1 +1 +1 +1 +0 +Get children +[1] +[2,3] +[4] +[] +[] +[] +Get all descendants +[1,2,3,4] +[2,3,4] +[4] +[] +[] +[] +Get descendants at first level +[1] +[2,3] +[4] +[] +[] +[] +Hashed dictionary +Get hierarchy +[] +[1] +[2,1] +[3,1] +[4,2,1] +[] +Get is in hierarchy +0 +1 +1 +1 +1 +0 +Get children +[1] +[2,3] +[4] +[] +[] +[] +Get all descendants +[1,2,3,4] +[2,3,4] +[4] +[] +[] +[] +Get descendants at first level +[1] +[2,3] +[4] +[] +[] +[] +Cache dictionary +Get hierarchy +[] +[1] +[2,1] +[3,1] +[4,2,1] +[] +Get is in hierarchy +0 +1 +1 +1 +1 +0 +Direct dictionary +Get hierarchy +[] +[1] +[2,1] +[3,1] +[4,2,1] +[] +Get is in hierarchy +0 +1 +1 +1 +1 +0 diff --git a/tests/queries/0_stateless/01778_hierarchical_dictionaries.sql b/tests/queries/0_stateless/01778_hierarchical_dictionaries.sql new file mode 100644 index 00000000000..eaf609e50fa --- /dev/null +++ b/tests/queries/0_stateless/01778_hierarchical_dictionaries.sql @@ -0,0 +1,95 @@ +DROP DATABASE IF EXISTS 01778_db; +CREATE DATABASE 01778_db; + +CREATE TABLE 01778_db.simple_key_hierarchy_source_table (id UInt64, parent_id UInt64) ENGINE = TinyLog; +INSERT INTO 01778_db.simple_key_hierarchy_source_table VALUES (1, 0), (2, 1), (3, 1), (4, 2); + +CREATE DICTIONARY 01778_db.simple_key_hierarchy_flat_dictionary +( + id UInt64, + parent_id UInt64 HIERARCHICAL +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'simple_key_hierarchy_source_table' DB '01778_db')) +LAYOUT(FLAT()) +LIFETIME(MIN 1 MAX 1000); + +SELECT 'Flat dictionary'; + +SELECT 'Get hierarchy'; +SELECT dictGetHierarchy('01778_db.simple_key_hierarchy_flat_dictionary', number) FROM system.numbers LIMIT 6; +SELECT 'Get is in hierarchy'; +SELECT dictIsIn('01778_db.simple_key_hierarchy_flat_dictionary', number, number) FROM system.numbers LIMIT 6; +SELECT 'Get children'; +SELECT dictGetChildren('01778_db.simple_key_hierarchy_flat_dictionary', number) FROM system.numbers LIMIT 6; +SELECT 'Get all descendants'; +SELECT dictGetDescendants('01778_db.simple_key_hierarchy_flat_dictionary', number) FROM system.numbers LIMIT 6; +SELECT 'Get descendants at first level'; +SELECT dictGetDescendants('01778_db.simple_key_hierarchy_flat_dictionary', number, 1) FROM system.numbers LIMIT 6; + +DROP DICTIONARY 01778_db.simple_key_hierarchy_flat_dictionary; + +CREATE DICTIONARY 01778_db.simple_key_hierarchy_hashed_dictionary +( + id UInt64, + parent_id UInt64 HIERARCHICAL +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'simple_key_hierarchy_source_table' DB '01778_db')) +LAYOUT(FLAT()) +LIFETIME(MIN 1 MAX 1000); + +SELECT 'Hashed dictionary'; + +SELECT 'Get hierarchy'; +SELECT dictGetHierarchy('01778_db.simple_key_hierarchy_hashed_dictionary', number) FROM system.numbers LIMIT 6; +SELECT 'Get is in hierarchy'; +SELECT dictIsIn('01778_db.simple_key_hierarchy_hashed_dictionary', number, number) FROM system.numbers LIMIT 6; +SELECT 'Get children'; +SELECT dictGetChildren('01778_db.simple_key_hierarchy_hashed_dictionary', number) FROM system.numbers LIMIT 6; +SELECT 'Get all descendants'; +SELECT dictGetDescendants('01778_db.simple_key_hierarchy_hashed_dictionary', number) FROM system.numbers LIMIT 6; +SELECT 'Get descendants at first level'; +SELECT dictGetDescendants('01778_db.simple_key_hierarchy_hashed_dictionary', number, 1) FROM system.numbers LIMIT 6; + +DROP DICTIONARY 01778_db.simple_key_hierarchy_hashed_dictionary; + +CREATE DICTIONARY 01778_db.simple_key_hierarchy_cache_dictionary +( + id UInt64, + parent_id UInt64 HIERARCHICAL +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'simple_key_hierarchy_source_table' DB '01778_db')) +LAYOUT(CACHE(SIZE_IN_CELLS 10)) +LIFETIME(MIN 1 MAX 1000); + +SELECT 'Cache dictionary'; + +SELECT 'Get hierarchy'; +SELECT dictGetHierarchy('01778_db.simple_key_hierarchy_cache_dictionary', number) FROM system.numbers LIMIT 6; +SELECT 'Get is in hierarchy'; +SELECT dictIsIn('01778_db.simple_key_hierarchy_cache_dictionary', number, number) FROM system.numbers LIMIT 6; + +DROP DICTIONARY 01778_db.simple_key_hierarchy_cache_dictionary; + +CREATE DICTIONARY 01778_db.simple_key_hierarchy_direct_dictionary +( + id UInt64, + parent_id UInt64 HIERARCHICAL +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'simple_key_hierarchy_source_table' DB '01778_db')) +LAYOUT(DIRECT()); + +SELECT 'Direct dictionary'; + +SELECT 'Get hierarchy'; +SELECT dictGetHierarchy('01778_db.simple_key_hierarchy_direct_dictionary', number) FROM system.numbers LIMIT 6; +SELECT 'Get is in hierarchy'; +SELECT dictIsIn('01778_db.simple_key_hierarchy_direct_dictionary', number, number) FROM system.numbers LIMIT 6; + +DROP DICTIONARY 01778_db.simple_key_hierarchy_direct_dictionary; + +DROP TABLE 01778_db.simple_key_hierarchy_source_table; +DROP DATABASE 01778_db;