From 2ced42ed4166035011be6e815c99590597425f76 Mon Sep 17 00:00:00 2001
From: Anton Popov <ap@clickhouse.com>
Date: Wed, 16 Mar 2022 16:51:23 +0000
Subject: [PATCH] add experimental settings for Object type

---
 src/Core/Settings.h                           |  1 +
 src/Formats/registerFormats.cpp               |  2 --
 src/Interpreters/InterpreterCreateQuery.cpp   | 26 ++++++++++++++++++-
 .../Impl/JSONAsStringRowInputFormat.cpp       |  5 ----
 .../queries/0_stateless/01825_type_json_1.sql |  4 +++
 .../queries/0_stateless/01825_type_json_2.sql |  2 ++
 .../0_stateless/01825_type_json_3.sql.j2      |  2 ++
 .../queries/0_stateless/01825_type_json_4.sh  |  2 +-
 .../queries/0_stateless/01825_type_json_5.sql |  2 ++
 .../queries/0_stateless/01825_type_json_6.sh  |  2 +-
 .../queries/0_stateless/01825_type_json_7.sh  |  2 +-
 .../queries/0_stateless/01825_type_json_8.sh  |  2 +-
 .../0_stateless/01825_type_json_btc.sh        |  4 +--
 .../0_stateless/01825_type_json_describe.sql  |  3 +++
 .../01825_type_json_distributed.sql           |  2 ++
 .../0_stateless/01825_type_json_field.sql     |  2 ++
 .../0_stateless/01825_type_json_ghdata.sh     |  4 +--
 .../01825_type_json_insert_select.sql         |  2 ++
 .../0_stateless/01825_type_json_nbagames.sh   |  4 +--
 .../0_stateless/01825_type_json_nullable.sql  |  2 ++
 .../01825_type_json_schema_race_long.sh       |  2 +-
 21 files changed, 58 insertions(+), 19 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index cdb54e55b10..3ff61379e39 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -567,6 +567,7 @@ class IColumn;
     /** Experimental functions */ \
     M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \
     M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \
+    M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \
     M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \
 // End of COMMON_SETTINGS
 // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.
diff --git a/src/Formats/registerFormats.cpp b/src/Formats/registerFormats.cpp
index 15da5355041..78851e5ebb0 100644
--- a/src/Formats/registerFormats.cpp
+++ b/src/Formats/registerFormats.cpp
@@ -13,7 +13,6 @@ void registerFileSegmentationEngineCSV(FormatFactory & factory);
 void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory);
 void registerFileSegmentationEngineRegexp(FormatFactory & factory);
 void registerFileSegmentationEngineJSONAsString(FormatFactory & factory);
-void registerFileSegmentationEngineJSONAsObject(FormatFactory & factory);
 void registerFileSegmentationEngineJSONCompactEachRow(FormatFactory & factory);
 
 /// Formats for both input/output.
@@ -121,7 +120,6 @@ void registerFormats()
     registerFileSegmentationEngineJSONEachRow(factory);
     registerFileSegmentationEngineRegexp(factory);
     registerFileSegmentationEngineJSONAsString(factory);
-    registerFileSegmentationEngineJSONAsObject(factory);
     registerFileSegmentationEngineJSONCompactEachRow(factory);
 
     registerInputFormatNative(factory);
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index ba4063b4c38..f7dbd1c8b65 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -53,6 +53,7 @@
 #include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeAggregateFunction.h>
+#include <DataTypes/ObjectUtils.h>
 
 #include <Databases/DatabaseFactory.h>
 #include <Databases/DatabaseReplicated.h>
@@ -733,11 +734,26 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
             {
                 String message = "Cannot create table with column '" + name_and_type_pair.name + "' which type is '"
                                  + type + "' because experimental geo types are not allowed. "
-                                 + "Set setting allow_experimental_geo_types = 1 in order to allow it.";
+                                 + "Set setting allow_experimental_geo_types = 1 in order to allow it";
                 throw Exception(message, ErrorCodes::ILLEGAL_COLUMN);
             }
         }
     }
+
+    if (!create.attach && !settings.allow_experimental_object_type)
+    {
+        for (const auto & [name, type] : properties.columns.getAllPhysical())
+        {
+            if (isObject(type))
+            {
+                throw Exception(ErrorCodes::ILLEGAL_COLUMN,
+                    "Cannot create table with column '{}' which type is '{}' "
+                    "because experimental Object type is not allowed. "
+                    "Set setting allow_experimental_object_type = 1 in order to allow it",
+                    name, type->getName());
+            }
+        }
+    }
 }
 
 String InterpreterCreateQuery::getTableEngineName(DefaultTableEngine default_table_engine)
@@ -1230,6 +1246,14 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
     /// we can safely destroy the object without a call to "shutdown", because there is guarantee
     /// that no background threads/similar resources remain after exception from "startup".
 
+    if (!res->supportsDynamicSubcolumns() && hasObjectColumns(res->getInMemoryMetadataPtr()->getColumns()))
+    {
+        throw Exception(ErrorCodes::ILLEGAL_COLUMN,
+            "Cannot create table with column of type Object, "
+            "because storage {} doesn't support dynamic subcolumns",
+            res->getName());
+    }
+
     res->startup();
     return true;
 }
diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp
index 6602d658a72..914ec27fc46 100644
--- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp
@@ -240,11 +240,6 @@ void registerInputFormatJSONAsObject(FormatFactory & factory)
     });
 }
 
-void registerFileSegmentationEngineJSONAsObject(FormatFactory & factory)
-{
-    factory.registerFileSegmentationEngine("JSONAsObject", &fileSegmentationEngineJSONEachRow);
-}
-
 void registerNonTrivialPrefixAndSuffixCheckerJSONAsObject(FormatFactory & factory)
 {
     factory.registerNonTrivialPrefixAndSuffixChecker("JSONAsObject", nonTrivialPrefixAndSuffixCheckerJSONEachRowImpl);
diff --git a/tests/queries/0_stateless/01825_type_json_1.sql b/tests/queries/0_stateless/01825_type_json_1.sql
index c2110f35c33..e74faf2d4c7 100644
--- a/tests/queries/0_stateless/01825_type_json_1.sql
+++ b/tests/queries/0_stateless/01825_type_json_1.sql
@@ -1,5 +1,7 @@
 -- Tags: no-fasttest
 
+SET allow_experimental_object_type = 1;
+
 DROP TABLE IF EXISTS t_json;
 
 CREATE TABLE t_json(id UInt64, data Object('JSON'))
@@ -79,3 +81,5 @@ WHERE table = 't_json' AND database = currentDatabase() AND active AND column =
 ORDER BY name;
 
 DROP TABLE IF EXISTS t_json;
+
+CREATE TABLE t_json(id UInt64, data Object('JSON')) ENGINE = Log; -- { serverError 44 }
diff --git a/tests/queries/0_stateless/01825_type_json_2.sql b/tests/queries/0_stateless/01825_type_json_2.sql
index cf5a74ad1e8..d2d26ce4106 100644
--- a/tests/queries/0_stateless/01825_type_json_2.sql
+++ b/tests/queries/0_stateless/01825_type_json_2.sql
@@ -1,5 +1,7 @@
 -- Tags: no-fasttest
 
+SET allow_experimental_object_type = 1;
+
 DROP TABLE IF EXISTS t_json_2;
 
 CREATE TABLE t_json_2(id UInt64, data Object('JSON'))
diff --git a/tests/queries/0_stateless/01825_type_json_3.sql.j2 b/tests/queries/0_stateless/01825_type_json_3.sql.j2
index 66c74bd17a0..62d86c3efd4 100644
--- a/tests/queries/0_stateless/01825_type_json_3.sql.j2
+++ b/tests/queries/0_stateless/01825_type_json_3.sql.j2
@@ -2,6 +2,8 @@
 
 {% for engine in ["ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_3/t_json_3', 'r1') ORDER BY tuple()", "Memory"] -%}
 
+SET allow_experimental_object_type = 1;
+
 DROP TABLE IF EXISTS t_json_3;
 
 CREATE TABLE t_json_3(id UInt64, data JSON)
diff --git a/tests/queries/0_stateless/01825_type_json_4.sh b/tests/queries/0_stateless/01825_type_json_4.sh
index d25a0f3a7dd..4d81e9516c9 100755
--- a/tests/queries/0_stateless/01825_type_json_4.sh
+++ b/tests/queries/0_stateless/01825_type_json_4.sh
@@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_4"
 
 $CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_4(id UInt64, data JSON) \
-ENGINE = MergeTree ORDER BY tuple()"
+ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
 
 echo '{"id": 1, "data": {"k1": "v1"}}, {"id": 2, "data": {"k1": [1, 2]}}' \
     | $CLICKHOUSE_CLIENT  -q "INSERT INTO t_json_4 FORMAT JSONEachRow" 2>&1 | grep -o -m1 "Code: 645"
diff --git a/tests/queries/0_stateless/01825_type_json_5.sql b/tests/queries/0_stateless/01825_type_json_5.sql
index eeea03432b4..b939a960e32 100644
--- a/tests/queries/0_stateless/01825_type_json_5.sql
+++ b/tests/queries/0_stateless/01825_type_json_5.sql
@@ -1,5 +1,7 @@
 -- Tags: no-fasttest
 
+SET allow_experimental_object_type = 1;
+
 SELECT '{"a": {"b": 1, "c": 2}}'::JSON AS s;
 SELECT '{"a": {"b": 1, "c": 2}}'::JSON AS s format JSONEachRow;
 
diff --git a/tests/queries/0_stateless/01825_type_json_6.sh b/tests/queries/0_stateless/01825_type_json_6.sh
index b992023fee0..8bbb1abee4a 100755
--- a/tests/queries/0_stateless/01825_type_json_6.sh
+++ b/tests/queries/0_stateless/01825_type_json_6.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_6;"
 
-$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_6 (data JSON) ENGINE = MergeTree ORDER BY tuple();"
+$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_6 (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
 
 cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_6 FORMAT JSONAsObject"
 {
diff --git a/tests/queries/0_stateless/01825_type_json_7.sh b/tests/queries/0_stateless/01825_type_json_7.sh
index 41708d80ed7..a6c8c53f995 100755
--- a/tests/queries/0_stateless/01825_type_json_7.sh
+++ b/tests/queries/0_stateless/01825_type_json_7.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_7;"
 
-$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_7 (data JSON) ENGINE = MergeTree ORDER BY tuple();"
+$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_7 (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
 
 cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_7 FORMAT JSONAsObject"
 {
diff --git a/tests/queries/0_stateless/01825_type_json_8.sh b/tests/queries/0_stateless/01825_type_json_8.sh
index e3ae8f23d36..e73569e6777 100755
--- a/tests/queries/0_stateless/01825_type_json_8.sh
+++ b/tests/queries/0_stateless/01825_type_json_8.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_8"
 
-$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_8 (data JSON) ENGINE = MergeTree ORDER BY tuple()"
+$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_8 (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
 
 cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_8 FORMAT JSONAsObject"
 {
diff --git a/tests/queries/0_stateless/01825_type_json_btc.sh b/tests/queries/0_stateless/01825_type_json_btc.sh
index 7d95046bb75..bfa209b72d1 100755
--- a/tests/queries/0_stateless/01825_type_json_btc.sh
+++ b/tests/queries/0_stateless/01825_type_json_btc.sh
@@ -7,9 +7,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS btc"
 
-${CLICKHOUSE_CLIENT} -q "CREATE TABLE btc (data JSON) ENGINE = MergeTree ORDER BY tuple()"
+${CLICKHOUSE_CLIENT} -q "CREATE TABLE btc (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
 
-cat $CUR_DIR/data_json/btc_transactions.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO btc FORMAT JSONAsObject" --input_format_parallel_parsing 0
+cat $CUR_DIR/data_json/btc_transactions.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO btc FORMAT JSONAsObject"
 
 ${CLICKHOUSE_CLIENT} -q "SELECT count() FROM btc WHERE NOT ignore(*)"
 ${CLICKHOUSE_CLIENT} -q "DESC btc SETTINGS describe_extend_object_types = 1"
diff --git a/tests/queries/0_stateless/01825_type_json_describe.sql b/tests/queries/0_stateless/01825_type_json_describe.sql
index f4aabaed715..cd7c4ff8c8c 100644
--- a/tests/queries/0_stateless/01825_type_json_describe.sql
+++ b/tests/queries/0_stateless/01825_type_json_describe.sql
@@ -1,5 +1,8 @@
 -- Tags: no-fasttest
 
+SET allow_experimental_object_type = 1;
+
+
 DROP TABLE IF EXISTS t_json_desc;
 
 CREATE TABLE t_json_desc (data JSON) ENGINE = MergeTree ORDER BY tuple();
diff --git a/tests/queries/0_stateless/01825_type_json_distributed.sql b/tests/queries/0_stateless/01825_type_json_distributed.sql
index a528352e938..70cc0743556 100644
--- a/tests/queries/0_stateless/01825_type_json_distributed.sql
+++ b/tests/queries/0_stateless/01825_type_json_distributed.sql
@@ -1,5 +1,7 @@
 -- Tags: no-fasttest
 
+SET allow_experimental_object_type = 1;
+
 DROP TABLE IF EXISTS t_json_local;
 DROP TABLE IF EXISTS t_json_dist;
 
diff --git a/tests/queries/0_stateless/01825_type_json_field.sql b/tests/queries/0_stateless/01825_type_json_field.sql
index f6df156030b..6c906023cef 100644
--- a/tests/queries/0_stateless/01825_type_json_field.sql
+++ b/tests/queries/0_stateless/01825_type_json_field.sql
@@ -1,5 +1,7 @@
 -- Tags: no-fasttest
 
+SET allow_experimental_object_type = 1;
+
 DROP TABLE IF EXISTS t_json_field;
 
 CREATE TABLE t_json_field (id UInt32, data JSON)
diff --git a/tests/queries/0_stateless/01825_type_json_ghdata.sh b/tests/queries/0_stateless/01825_type_json_ghdata.sh
index 87bc8740b46..7486571cc22 100755
--- a/tests/queries/0_stateless/01825_type_json_ghdata.sh
+++ b/tests/queries/0_stateless/01825_type_json_ghdata.sh
@@ -7,9 +7,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata"
 
-${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata (data JSON) ENGINE = MergeTree ORDER BY tuple()"
+${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
 
-cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO ghdata FORMAT JSONAsObject" --input_format_parallel_parsing 0
+cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO ghdata FORMAT JSONAsObject"
 
 ${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ghdata WHERE NOT ignore(*)"
 
diff --git a/tests/queries/0_stateless/01825_type_json_insert_select.sql b/tests/queries/0_stateless/01825_type_json_insert_select.sql
index 878c106ca1e..8bb03f84f5a 100644
--- a/tests/queries/0_stateless/01825_type_json_insert_select.sql
+++ b/tests/queries/0_stateless/01825_type_json_insert_select.sql
@@ -1,5 +1,7 @@
 -- Tags: no-fasttest
 
+SET allow_experimental_object_type = 1;
+
 DROP TABLE IF EXISTS type_json_src;
 DROP TABLE IF EXISTS type_json_dst;
 
diff --git a/tests/queries/0_stateless/01825_type_json_nbagames.sh b/tests/queries/0_stateless/01825_type_json_nbagames.sh
index 783901a93c4..18e7c050680 100755
--- a/tests/queries/0_stateless/01825_type_json_nbagames.sh
+++ b/tests/queries/0_stateless/01825_type_json_nbagames.sh
@@ -7,9 +7,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames"
 
-${CLICKHOUSE_CLIENT} -q "CREATE TABLE nbagames (data JSON) ENGINE = MergeTree ORDER BY tuple()"
+${CLICKHOUSE_CLIENT} -q "CREATE TABLE nbagames (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
 
-cat $CUR_DIR/data_json/nbagames_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO nbagames FORMAT JSONAsObject" --input_format_parallel_parsing 0
+cat $CUR_DIR/data_json/nbagames_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO nbagames FORMAT JSONAsObject"
 
 ${CLICKHOUSE_CLIENT} -q "SELECT count() FROM nbagames WHERE NOT ignore(*)"
 ${CLICKHOUSE_CLIENT} -q "DESC nbagames SETTINGS describe_extend_object_types = 1"
diff --git a/tests/queries/0_stateless/01825_type_json_nullable.sql b/tests/queries/0_stateless/01825_type_json_nullable.sql
index 3a9f893a0da..65589243f43 100644
--- a/tests/queries/0_stateless/01825_type_json_nullable.sql
+++ b/tests/queries/0_stateless/01825_type_json_nullable.sql
@@ -1,5 +1,7 @@
 -- Tags: no-fasttest
 
+SET allow_experimental_object_type = 1;
+
 DROP TABLE IF EXISTS t_json_null;
 
 CREATE TABLE t_json_null(id UInt64, data Object(Nullable('JSON')))
diff --git a/tests/queries/0_stateless/01825_type_json_schema_race_long.sh b/tests/queries/0_stateless/01825_type_json_schema_race_long.sh
index a119dd17ead..38d1432cef6 100755
--- a/tests/queries/0_stateless/01825_type_json_schema_race_long.sh
+++ b/tests/queries/0_stateless/01825_type_json_schema_race_long.sh
@@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_race"
-$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_race (data JSON) ENGINE = MergeTree ORDER BY tuple()"
+$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_race (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
 
 function test_case()
 {