From 3bd96d709d944378dbe9a82cb0334c450059e76f Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Sat, 11 Sep 2021 03:20:54 +0300 Subject: [PATCH] dynamic columns: fix several cases of parsing json --- src/Columns/ColumnObject.cpp | 58 ++++++++++++++----- ....reference => 01825_type_json_1.reference} | 0 ...25_type_json.sql => 01825_type_json_1.sql} | 0 .../0_stateless/01825_type_json_6.reference | 3 + .../queries/0_stateless/01825_type_json_6.sh | 56 ++++++++++++++++++ 5 files changed, 103 insertions(+), 14 deletions(-) rename tests/queries/0_stateless/{01825_type_json.reference => 01825_type_json_1.reference} (100%) rename tests/queries/0_stateless/{01825_type_json.sql => 01825_type_json_1.sql} (100%) create mode 100644 tests/queries/0_stateless/01825_type_json_6.reference create mode 100755 tests/queries/0_stateless/01825_type_json_6.sh diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 6b5610ce7d1..0953e1ec538 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -13,7 +13,6 @@ #include #include - namespace DB { @@ -28,25 +27,45 @@ namespace ErrorCodes namespace { +Array createEmptyArrayField(size_t num_dimensions) +{ + Array array; + Array * current_array = &array; + for (size_t i = 1; i < num_dimensions; ++i) + { + current_array->push_back(Array()); + current_array = ¤t_array->back().get(); + } + + return array; +} + class FieldVisitorReplaceNull : public StaticVisitor { public: - [[maybe_unused]] explicit FieldVisitorReplaceNull(const Field & replacement_) + [[maybe_unused]] explicit FieldVisitorReplaceNull( + const Field & replacement_, size_t num_dimensions_) : replacement(replacement_) + , num_dimensions(num_dimensions_) { } - Field operator()(const Null &) const { return replacement; } - template Field operator()(const T & x) const { - if constexpr (std::is_base_of_v) + if constexpr (std::is_same_v) { + return num_dimensions + ? createEmptyArrayField(num_dimensions) + : replacement; + } + else if constexpr (std::is_same_v) + { + assert(num_dimensions > 0); const size_t size = x.size(); - T res(size); + Array res(size); for (size_t i = 0; i < size; ++i) - res[i] = applyVisitor(*this, x[i]); + res[i] = applyVisitor(FieldVisitorReplaceNull(replacement, num_dimensions - 1), x[i]); return res; } else @@ -54,7 +73,8 @@ public: } private: - Field replacement; + const Field & replacement; + size_t num_dimensions; }; class FieldVisitorToNumberOfDimensions : public StaticVisitor @@ -66,16 +86,24 @@ public: return 1; const size_t size = x.size(); - size_t dimensions = applyVisitor(*this, x[0]); - for (size_t i = 1; i < size; ++i) + std::optional dimensions; + + for (size_t i = 0; i < size; ++i) { + /// Do not count Nulls, because they will be replaced by default + /// values with proper number of dimensions. + if (x[i].isNull()) + continue; + size_t current_dimensions = applyVisitor(*this, x[i]); - if (current_dimensions != dimensions) + if (!dimensions) + dimensions = current_dimensions; + else if (current_dimensions != *dimensions) throw Exception(ErrorCodes::NUMBER_OF_DIMENSIONS_MISMATHED, "Number of dimensions mismatched among array elements"); } - return 1 + dimensions; + return 1 + dimensions.value_or(0); } template @@ -239,6 +267,7 @@ void ColumnObject::Subcolumn::insert(Field field) if (is_nullable && !base_type->isNullable()) base_type = makeNullable(base_type); + auto value_type = createArrayOfType(base_type, value_dim); if (!is_nullable && base_type->isNullable()) { base_type = removeNullable(base_type); @@ -248,10 +277,11 @@ void ColumnObject::Subcolumn::insert(Field field) return; } - field = applyVisitor(FieldVisitorReplaceNull(base_type->getDefault()), std::move(field)); + value_type = createArrayOfType(base_type, value_dim); + auto default_value = value_type->getDefault(); + field = applyVisitor(FieldVisitorReplaceNull(default_value, value_dim), std::move(field)); } - auto value_type = createArrayOfType(base_type, value_dim); bool type_changed = false; if (data.empty()) diff --git a/tests/queries/0_stateless/01825_type_json.reference b/tests/queries/0_stateless/01825_type_json_1.reference similarity index 100% rename from tests/queries/0_stateless/01825_type_json.reference rename to tests/queries/0_stateless/01825_type_json_1.reference diff --git a/tests/queries/0_stateless/01825_type_json.sql b/tests/queries/0_stateless/01825_type_json_1.sql similarity index 100% rename from tests/queries/0_stateless/01825_type_json.sql rename to tests/queries/0_stateless/01825_type_json_1.sql diff --git a/tests/queries/0_stateless/01825_type_json_6.reference b/tests/queries/0_stateless/01825_type_json_6.reference new file mode 100644 index 00000000000..2efc2031545 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_6.reference @@ -0,0 +1,3 @@ +Tuple(key String, `out.outputs.index` Array(Array(Int32)), `out.outputs.n` Array(Array(Int8)), `out.type` Array(Int8), `out.value` Array(Int8)) +v1 [0,0] [1,2] [[],[1960131]] [[],[1960131]] +v2 [1,1] [4,3] [[1881212],[]] [[1881212],[]] diff --git a/tests/queries/0_stateless/01825_type_json_6.sh b/tests/queries/0_stateless/01825_type_json_6.sh new file mode 100755 index 00000000000..5ab0b77ab6b --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_6.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_6;" + +$CLICKHOUSE_CLIENT -q"CREATE TABLE t_json_6 (data JSON) ENGINE = MergeTree ORDER BY tuple();" + +cat <