Merge pull request #5448 from yandex/low-cardinality

Add setting suspicious type for low cardinality (fixed #4965)

dbms/src/Common/ErrorCodes.cpp

@@ -429,6 +429,7 @@ namespace ErrorCodes
     extern const int SETTING_CONSTRAINT_VIOLATION = 452;
     extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES = 453;
     extern const int OPENSSL_ERROR = 454;
+    extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY = 455;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Core/Settings.h

@@ -85,6 +85,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingFloat, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.") \
     \
     M(SettingBool, compile, false, "Whether query compilation is enabled.") \
+    M(SettingBool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.") \
     M(SettingBool, compile_expressions, false, "Compile some scalar functions and operators to native code.") \
     M(SettingUInt64, min_count_to_compile, 3, "The number of structurally identical queries before they are compiled.") \
     M(SettingUInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled") \

dbms/src/Interpreters/InterpreterCreateQuery.cpp

@@ -38,6 +38,8 @@
 #include <DataTypes/NestedUtils.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypesDecimal.h>
+#include <DataTypes/DataTypeLowCardinality.h>
+#include <DataTypes/DataTypeNullable.h>
 
 #include <Databases/DatabaseFactory.h>
 #include <Databases/IDatabase.h>
@@ -65,6 +67,7 @@ namespace ErrorCodes
     extern const int QUERY_IS_PROHIBITED;
     extern const int THERE_IS_NO_DEFAULT_VALUE;
     extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE;
+    extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY;
 }
 
 
@@ -523,6 +526,20 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     /// Set and retrieve list of columns.
     ColumnsDescription columns = setColumns(create, as_select_sample, as_storage);
 
+    /// Check low cardinality types in creating table if it was not allowed in setting
+    if (!create.attach && !context.getSettingsRef().allow_suspicious_low_cardinality_types)
+    {
+        for (const auto & name_and_type_pair : columns.getAllPhysical())
+        {
+            if (const auto * current_type_ptr = typeid_cast<const DataTypeLowCardinality *>(name_and_type_pair.type.get()))
+            {
+                if (!isStringOrFixedString(*removeNullable(current_type_ptr->getDictionaryType())))
+                    throw Exception("Creating columns of type " + current_type_ptr->getName() + " is prohibited by default due to expected negative impact on performance. It can be enabled with the \"allow_suspicious_low_cardinality_types\" setting.",
+                        ErrorCodes::SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY);
+            }
+        }
+    }
+
     /// Set the table engine if it was not specified explicitly.
     setEngine(create);
 

dbms/tests/queries/0_stateless/00688_low_cardinality_in.sql

@@ -1,3 +1,4 @@
+set allow_suspicious_low_cardinality_types = 1;
 drop table if exists lc_00688;
 create table lc_00688 (str StringWithDictionary, val UInt8WithDictionary) engine = MergeTree order by tuple();
 insert into lc_00688 values ('a', 1), ('b', 2);

dbms/tests/queries/0_stateless/00688_low_cardinality_nullable_cast.sql

@@ -1,3 +1,4 @@
+set allow_suspicious_low_cardinality_types = 1;
 SELECT CAST(NULL, 'LowCardinality(Nullable(Int8))');
 
 drop table if exists lc_null_int8_defnull;

dbms/tests/queries/0_stateless/00688_low_cardinality_syntax.sql

@@ -1,3 +1,4 @@
+set allow_suspicious_low_cardinality_types = 1;
 drop table if exists lc_str_0;
 drop table if exists lc_str_1;
 drop table if exists lc_null_str_0;

dbms/tests/queries/0_stateless/00717_low_cardinaliry_distributed_group_by.sql

@@ -1,3 +1,4 @@
+set allow_suspicious_low_cardinality_types = 1;
 DROP TABLE IF EXISTS test.test_low_null_float;
 DROP TABLE IF EXISTS test.dist;
 

dbms/tests/queries/0_stateless/00718_low_cardinaliry_alter.sql

@@ -1,3 +1,4 @@
+set allow_suspicious_low_cardinality_types = 1;
 drop table if exists tab_00718;
 create table tab_00718 (a String, b LowCardinality(UInt32)) engine = MergeTree order by a;
 insert into tab_00718 values ('a', 1);

dbms/tests/queries/0_stateless/00751_low_cardinality_nullable_group_by.sql

@@ -1,3 +1,4 @@
+set allow_suspicious_low_cardinality_types = 1;
 drop table if exists low_null_float;
 CREATE TABLE low_null_float (a LowCardinality(Nullable(Float64))) ENGINE = MergeTree order by tuple();
 INSERT INTO low_null_float (a) SELECT if(number % 3 == 0, Null, number)  FROM system.numbers LIMIT 1000000;

dbms/tests/queries/0_stateless/00800_low_cardinality_distinct_numeric.sql

@@ -1,3 +1,4 @@
+set allow_suspicious_low_cardinality_types = 1;
 drop table if exists lc_00800_2;
 create table lc_00800_2 (val LowCardinality(UInt64)) engine = MergeTree order by val;
 insert into lc_00800_2 select number % 123 from system.numbers limit 100000;