From cf8c614fedd7f01836a7124998ab9a393bbde8fe Mon Sep 17 00:00:00 2001 From: VanDarkholme7 <980181529@qq.com> Date: Sat, 16 Sep 2023 13:04:19 +0000 Subject: [PATCH] Fix bug on DELETE and UPDATE with json subcolumns --- .../MergeTree/MergeTreeSequentialSource.cpp | 8 ++- .../MergeTree/StorageFromMergeTreeDataPart.h | 17 +++++++ .../02874_mutations_subcolumns.reference | 10 ++++ .../02874_mutations_subcolumns.sql | 51 +++++++++++++++++++ 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02874_mutations_subcolumns.reference create mode 100644 tests/queries/0_stateless/02874_mutations_subcolumns.sql diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 8a9faa5cee4..9bd9db6a8c4 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -119,7 +119,11 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( addTotalRowsApprox(data_part->rows_count); /// Add columns because we don't want to read empty blocks - injectRequiredColumns(LoadedMergeTreeDataPartInfoForReader(data_part, alter_conversions), storage_snapshot, /*with_subcolumns=*/ false, columns_to_read); + injectRequiredColumns( + LoadedMergeTreeDataPartInfoForReader(data_part, alter_conversions), + storage_snapshot, + storage.supportsSubcolumns(), + columns_to_read); NamesAndTypesList columns_for_reader; if (take_column_types_from_storage) @@ -127,6 +131,8 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical) .withExtendedObjects() .withSystemColumns(); + if (storage.supportsSubcolumns()) + options.withSubcolumns(); columns_for_reader = storage_snapshot->getColumnsByNames(options, columns_to_read); } else diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 17078d3e73b..21db0893a4e 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -59,6 +59,21 @@ public: return std::make_shared(*this, metadata_snapshot, std::move(object_columns)); } + StorageSnapshotPtr getStorageSnapshotForQuery( + const StorageMetadataPtr & metadata_snapshot, const ASTPtr & /*query*/, ContextPtr /*query_context*/) const override + { + const auto & storage_columns = metadata_snapshot->getColumns(); + if (!hasDynamicSubcolumns(storage_columns)) + return std::make_shared(*this, metadata_snapshot); + + auto data_parts = storage.getDataPartsVectorForInternalUsage(); + + auto object_columns = getConcreteObjectColumns( + data_parts.begin(), data_parts.end(), storage_columns, [](const auto & part) -> const auto & { return part->getColumns(); }); + + return std::make_shared(*this, metadata_snapshot, std::move(object_columns)); + } + void read( QueryPlan & query_plan, const Names & column_names, @@ -89,6 +104,8 @@ public: bool supportsDynamicSubcolumns() const override { return true; } + bool supportsSubcolumns() const override { return true; } + bool mayBenefitFromIndexForIn( const ASTPtr & left_in_operand, ContextPtr query_context, const StorageMetadataPtr & metadata_snapshot) const override { diff --git a/tests/queries/0_stateless/02874_mutations_subcolumns.reference b/tests/queries/0_stateless/02874_mutations_subcolumns.reference new file mode 100644 index 00000000000..b702755e127 --- /dev/null +++ b/tests/queries/0_stateless/02874_mutations_subcolumns.reference @@ -0,0 +1,10 @@ +6 1 +5 2 +4 3 +3 4 +4 ttt +5 ttt +6 ttt +{"a":"1","obj":{"k1":1,"k2":0,"k3":0}} +{"a":"3","obj":{"k1":0,"k2":0,"k3":1}} +{"a":"1","obj":{"k1":1,"k2":0,"k3":0}} diff --git a/tests/queries/0_stateless/02874_mutations_subcolumns.sql b/tests/queries/0_stateless/02874_mutations_subcolumns.sql new file mode 100644 index 00000000000..54799f2481f --- /dev/null +++ b/tests/queries/0_stateless/02874_mutations_subcolumns.sql @@ -0,0 +1,51 @@ +-- Tags: no-replicated-database +-- This won't work in case there are misssing subcolumns in different shards + +DROP TABLE IF EXISTS t_mutations_subcolumns; + +SET allow_experimental_object_type = 1; + +CREATE TABLE t_mutations_subcolumns (id UInt64, n String, obj JSON) +ENGINE = MergeTree ORDER BY id; + +INSERT INTO t_mutations_subcolumns VALUES (1, 'aaa', '{"k1": {"k2": "foo"}, "k3": 5}'); +INSERT INTO t_mutations_subcolumns VALUES (2, 'bbb', '{"k1": {"k2": "fee"}, "k3": 4}'); +INSERT INTO t_mutations_subcolumns VALUES (3, 'ccc', '{"k1": {"k2": "foo", "k4": "baz"}, "k3": 4}'); +INSERT INTO t_mutations_subcolumns VALUES (4, 'ddd', '{"k1": {"k2": "foo"}, "k3": 4}'); +INSERT INTO t_mutations_subcolumns VALUES (5, 'eee', '{"k1": {"k2": "foo"}, "k3": 4}'); +INSERT INTO t_mutations_subcolumns VALUES (6, 'fff', '{"k1": {"k2": "foo"}, "k3": 4}'); + +OPTIMIZE TABLE t_mutations_subcolumns FINAL; + +SELECT count(), min(id) FROM t_mutations_subcolumns; + +SET mutations_sync = 2; + +ALTER TABLE t_mutations_subcolumns DELETE WHERE obj.k3 = 5; +SELECT count(), min(id) FROM t_mutations_subcolumns; + +DELETE FROM t_mutations_subcolumns WHERE obj.k1.k2 = 'fee'; +SELECT count(), min(id) FROM t_mutations_subcolumns; + +ALTER TABLE t_mutations_subcolumns DELETE WHERE obj.k1 = ('foo', 'baz'); +SELECT count(), min(id) FROM t_mutations_subcolumns; + +ALTER TABLE t_mutations_subcolumns UPDATE n = 'ttt' WHERE obj.k1.k2 = 'foo'; +SELECT id, n FROM t_mutations_subcolumns; + +DROP TABLE IF EXISTS t_mutations_subcolumns; + +CREATE TABLE t_mutations_subcolumns (a UInt64, obj JSON) +ENGINE = MergeTree ORDER BY a PARTITION BY a; + +INSERT INTO t_mutations_subcolumns VALUES (1, '{"k1": 1}'); +INSERT INTO t_mutations_subcolumns VALUES (2, '{"k2": 1}'); +INSERT INTO t_mutations_subcolumns VALUES (3, '{"k3": 1}'); + +ALTER TABLE t_mutations_subcolumns DELETE WHERE obj.k2 = 1; +SELECT * FROM t_mutations_subcolumns ORDER BY a FORMAT JSONEachRow; + +ALTER TABLE t_mutations_subcolumns DELETE WHERE obj.k1 = 0; +SELECT * FROM t_mutations_subcolumns ORDER BY a FORMAT JSONEachRow; + +DROP TABLE t_mutations_subcolumns;