Merge pull request #49587 from xbthink/master

automatically choose the "complex key" layout variant

src/Dictionaries/DictionaryFactory.cpp

@@ -17,13 +17,13 @@ namespace ErrorCodes
     extern const int UNKNOWN_ELEMENT_IN_CONFIG;
 }
 
-void DictionaryFactory::registerLayout(const std::string & layout_type, LayoutCreateFunction create_layout, bool is_layout_complex)
+void DictionaryFactory::registerLayout(const std::string & layout_type, LayoutCreateFunction create_layout, bool is_layout_complex, bool has_layout_complex)
 {
     auto it = registered_layouts.find(layout_type);
     if (it != registered_layouts.end())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "DictionaryFactory: the layout name '{}' is not unique", layout_type);
 
-    RegisteredLayout layout { .layout_create_function = create_layout, .is_layout_complex = is_layout_complex };
+    RegisteredLayout layout { .layout_create_function = create_layout, .is_layout_complex = is_layout_complex, .has_layout_complex = has_layout_complex };
     registered_layouts.emplace(layout_type, std::move(layout));
 }
 
@@ -89,6 +89,25 @@ bool DictionaryFactory::isComplex(const std::string & layout_type) const
     return it->second.is_layout_complex;
 }
 
+bool DictionaryFactory::convertToComplex(std::string & layout_type) const
+{
+    auto it = registered_layouts.find(layout_type);
+
+    if (it == registered_layouts.end())
+    {
+        throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG,
+                        "Unknown dictionary layout type: {}",
+                        layout_type);
+    }
+
+    if (!it->second.is_layout_complex && it->second.has_layout_complex)
+    {
+        layout_type = "complex_key_" + layout_type;
+        return true;
+    }
+    return false;
+}
+
 
 DictionaryFactory & DictionaryFactory::instance()
 {

src/Dictionaries/DictionaryFactory.h

@@ -55,13 +55,18 @@ public:
 
     bool isComplex(const std::string & layout_type) const;
 
-    void registerLayout(const std::string & layout_type, LayoutCreateFunction create_layout, bool is_layout_complex);
+    /// If the argument `layout_type` is not complex layout and has corresponding complex layout,
+    /// change `layout_type` to corresponding complex and return true; otherwise do nothing and return false.
+    bool convertToComplex(std::string & layout_type) const;
+
+    void registerLayout(const std::string & layout_type, LayoutCreateFunction create_layout, bool is_layout_complex, bool has_layout_complex = true);
 
 private:
     struct RegisteredLayout
     {
         LayoutCreateFunction layout_create_function;
         bool is_layout_complex;
+        bool has_layout_complex;
     };
 
     using LayoutRegistry = std::unordered_map<std::string, RegisteredLayout>;

src/Dictionaries/FlatDictionary.cpp

@@ -683,7 +683,7 @@ void registerDictionaryFlat(DictionaryFactory & factory)
         return std::make_unique<FlatDictionary>(dict_id, dict_struct, std::move(source_ptr), configuration);
     };
 
-    factory.registerLayout("flat", create_layout, false);
+    factory.registerLayout("flat", create_layout, false, false);
 }
 
 

src/Dictionaries/getDictionaryConfigurationFromAST.cpp

@@ -19,6 +19,7 @@
 #include <Functions/FunctionFactory.h>
 #include <Common/isLocalAddress.h>
 #include <Interpreters/Context.h>
+#include <DataTypes/DataTypeFactory.h>
 
 
 namespace DB
@@ -614,6 +615,16 @@ getDictionaryConfigurationFromAST(const ASTCreateQuery & query, ContextPtr conte
 
     checkPrimaryKey(all_attr_names_and_types, pk_attrs);
 
+    /// If the pk size is 1 and pk's DataType is not number, we should convert to complex.
+    /// NOTE: the data type of Numeric key(simple layout) is UInt64, so if the type is not under UInt64, type casting will lead to precision loss.
+    DataTypePtr first_key_type = DataTypeFactory::instance().get(all_attr_names_and_types.find(pk_attrs[0])->second.type);
+    if ((pk_attrs.size() > 1 || (pk_attrs.size() == 1 && !isNumber(first_key_type)))
+        && !complex
+        && DictionaryFactory::instance().convertToComplex(dictionary_layout->layout_type))
+    {
+        complex = true;
+    }
+
     buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list);
 
     buildLayoutConfiguration(xml_document, current_dictionary, query.dictionary->dict_settings, dictionary_layout);

tests/queries/0_stateless/01018_ddl_dictionaries_bad_queries.reference

@@ -4,4 +4,3 @@
 2
 2
 2
-2

tests/queries/0_stateless/01018_ddl_dictionaries_bad_queries.sh

@@ -9,21 +9,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -q "DROP DICTIONARY IF  EXISTS dict1"
 
-# Simple layout, but with two keys
-$CLICKHOUSE_CLIENT -q "
-    CREATE DICTIONARY dict1
-    (
-        key1 UInt64,
-        key2 UInt64,
-        value String
-    )
-    PRIMARY KEY key1, key2
-    LAYOUT(HASHED())
-    SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict1' DB '$CLICKHOUSE_DATABASE'))
-    LIFETIME(MIN 1 MAX 10)
-" 2>&1 | grep -c 'Primary key for simple dictionary must contain exactly one element'
-
-
 # Simple layout, but with non existing key
 $CLICKHOUSE_CLIENT -q "
     CREATE DICTIONARY dict1

tests/queries/0_stateless/02391_hashed_dictionary_shards.sql

@@ -89,7 +89,7 @@ SOURCE(CLICKHOUSE(TABLE test_table_string))
 LAYOUT(SPARSE_HASHED(SHARDS 10))
 LIFETIME(0);
 
-SYSTEM RELOAD DICTIONARY test_dictionary_10_shards_string; -- { serverError CANNOT_PARSE_TEXT }
+SYSTEM RELOAD DICTIONARY test_dictionary_10_shards_string;
 
 DROP DICTIONARY test_dictionary_10_shards_string;
 

tests/queries/0_stateless/02731_auto_convert_dictionary_layout_to_complex_by_complex_keys.reference

@@ -0,0 +1,5 @@
+dict_flat_simple	Flat
+dict_hashed_simple_Decimal128	Hashed
+dict_hashed_simple_Float32	Hashed
+dict_hashed_simple_String	ComplexKeyHashed
+dict_hashed_simple_auto_convert	ComplexKeyHashed

tests/queries/0_stateless/02731_auto_convert_dictionary_layout_to_complex_by_complex_keys.sql

@@ -0,0 +1,35 @@
+DROP DICTIONARY IF EXISTS dict_flat_simple;
+DROP DICTIONARY IF EXISTS dict_hashed_simple_Decimal128;
+DROP DICTIONARY IF EXISTS dict_hashed_simple_Float32;
+DROP DICTIONARY IF EXISTS dict_hashed_simple_String;
+DROP DICTIONARY IF EXISTS dict_hashed_simple_auto_convert;
+DROP TABLE IF EXISTS dict_data;
+
+CREATE TABLE dict_data (v0 UInt16, v1 Int16, v2 Float32, v3 Decimal128(10), v4 String) engine=Memory()  AS SELECT number, number%65535, number*1.1, number*1.1, 'foo' FROM numbers(10);;
+
+CREATE DICTIONARY dict_flat_simple (v0 UInt16, v1 UInt16, v2 UInt16) PRIMARY KEY v0 SOURCE(CLICKHOUSE(TABLE 'dict_data')) LIFETIME(0) LAYOUT(flat());
+SYSTEM RELOAD DICTIONARY dict_flat_simple;
+SELECT name, type FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_flat_simple';
+DROP DICTIONARY dict_flat_simple;
+
+CREATE DICTIONARY dict_hashed_simple_Decimal128 (v3 Decimal128(10), v1 UInt16, v2 Float32) PRIMARY KEY v3 SOURCE(CLICKHOUSE(TABLE 'dict_data')) LIFETIME(0) LAYOUT(hashed());
+SYSTEM RELOAD DICTIONARY dict_hashed_simple_Decimal128;
+SELECT name, type FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_hashed_simple_Decimal128';
+DROP DICTIONARY dict_hashed_simple_Decimal128;
+
+CREATE DICTIONARY dict_hashed_simple_Float32 (v2 Float32, v3 Decimal128(10), v4 String) PRIMARY KEY v2 SOURCE(CLICKHOUSE(TABLE 'dict_data')) LIFETIME(0) LAYOUT(hashed());
+SYSTEM RELOAD DICTIONARY dict_hashed_simple_Float32;
+SELECT name, type FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_hashed_simple_Float32';
+DROP DICTIONARY dict_hashed_simple_Float32;
+
+CREATE DICTIONARY dict_hashed_simple_String (v4 String, v3 Decimal128(10), v2 Float32) PRIMARY KEY v4 SOURCE(CLICKHOUSE(TABLE 'dict_data')) LIFETIME(0) LAYOUT(hashed());
+SYSTEM RELOAD DICTIONARY dict_hashed_simple_String;
+SELECT name, type FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_hashed_simple_String';
+DROP DICTIONARY dict_hashed_simple_String;
+
+CREATE DICTIONARY dict_hashed_simple_auto_convert (v0 UInt16, v1 Int16, v2 UInt16) PRIMARY KEY v0,v1 SOURCE(CLICKHOUSE(TABLE 'dict_data')) LIFETIME(0) LAYOUT(hashed());
+SYSTEM RELOAD DICTIONARY dict_hashed_simple_auto_convert;
+SELECT name, type FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_hashed_simple_auto_convert';
+DROP DICTIONARY dict_hashed_simple_auto_convert;
+
+DROP TABLE dict_data;