From 1b5f6e37cb0ff69e02cd53573b97ca00e16bebe4 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 21 Jan 2020 17:02:40 +0100 Subject: [PATCH 1/3] First attempt to fix issue https://github.com/ClickHouse/ClickHouse/issues/7878 --- .../ConvertingBlockInputStream.cpp | 25 +++++++++++++++++-- .../DataStreams/ConvertingBlockInputStream.h | 5 +++- .../PushingToViewsBlockOutputStream.cpp | 2 +- ...rialized_view_alter_target_table.reference | 8 ++++++ ...9_materialized_view_alter_target_table.sql | 15 +++++++++++ 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.reference create mode 100644 dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.sql diff --git a/dbms/src/DataStreams/ConvertingBlockInputStream.cpp b/dbms/src/DataStreams/ConvertingBlockInputStream.cpp index 44f4989f3cc..d1f5b2713a6 100644 --- a/dbms/src/DataStreams/ConvertingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ConvertingBlockInputStream.cpp @@ -4,7 +4,7 @@ #include #include #include - +#include namespace DB { @@ -65,8 +65,20 @@ ConvertingBlockInputStream::ConvertingBlockInputStream( throw Exception("Cannot find column " + backQuote(res_elem.name) + " in source stream", ErrorCodes::THERE_IS_NO_COLUMN); break; + + case MatchColumnsMode::NameOrDefault: + if (input_header.has(res_elem.name)) + conversion[result_col_num] = input_header.getPositionByName(res_elem.name); + else { + std::cerr << "column " << result_col_num << " will be set to default\n"; + conversion[result_col_num] = USE_DEFAULT; + } + break; } + if (conversion[result_col_num] == USE_DEFAULT) + continue; + const auto & src_elem = input_header.getByPosition(conversion[result_col_num]); /// Check constants. @@ -100,9 +112,18 @@ Block ConvertingBlockInputStream::readImpl() Block res = header.cloneEmpty(); for (size_t res_pos = 0, size = conversion.size(); res_pos < size; ++res_pos) { - const auto & src_elem = src.getByPosition(conversion[res_pos]); auto & res_elem = res.getByPosition(res_pos); + if (conversion[res_pos] == USE_DEFAULT) + { + // Create a column with default values + auto column_with_defaults = res_elem.type->createColumn()->cloneResized(src.rows()); + res_elem.column = std::move(column_with_defaults); + continue; + } + + const auto & src_elem = src.getByPosition(conversion[res_pos]); + ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context); if (isColumnConst(*src_elem.column) && !isColumnConst(*res_elem.column)) diff --git a/dbms/src/DataStreams/ConvertingBlockInputStream.h b/dbms/src/DataStreams/ConvertingBlockInputStream.h index 553d9221dd6..26d8190aa8b 100644 --- a/dbms/src/DataStreams/ConvertingBlockInputStream.h +++ b/dbms/src/DataStreams/ConvertingBlockInputStream.h @@ -28,7 +28,9 @@ public: /// Require same number of columns in source and result. Match columns by corresponding positions, regardless to names. Position, /// Find columns in source by their names. Allow excessive columns in source. - Name + Name, + /// Find columns in source by their names if present else use the default. Allow excessive columns in source. + NameOrDefault }; ConvertingBlockInputStream( @@ -48,6 +50,7 @@ private: /// How to construct result block. Position in source block, where to get each column. using Conversion = std::vector; + const size_t USE_DEFAULT = static_cast(-1); Conversion conversion; }; diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 0adc7b43ed1..7388f8b6904 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -230,7 +230,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n /// and two-level aggregation is triggered). in = std::make_shared( in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); - in = std::make_shared(context, in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Name); + in = std::make_shared(context, in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::NameOrDefault); } else in = std::make_shared(block); diff --git a/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.reference b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.reference new file mode 100644 index 00000000000..a07a1e62f3a --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.reference @@ -0,0 +1,8 @@ +1 +1 +2 +3 +1 0 +1 0 +2 0 +3 0 diff --git a/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.sql b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.sql new file mode 100644 index 00000000000..4a9e814c0d2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_materialized_view_alter_target_table.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS mv_source; +DROP TABLE IF EXISTS mv_target; + +CREATE TABLE mv_source (`a` UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE mv_target (`a` UInt64) ENGINE = MergeTree ORDER BY tuple(); + +CREATE MATERIALIZED VIEW mv TO mv_target AS SELECT * FROM mv_source; + +INSERT INTO mv_source VALUES (1); + +ALTER TABLE mv_target ADD COLUMN b UInt8; +INSERT INTO mv_source VALUES (1),(2),(3); + +SELECT * FROM mv ORDER BY a; +SELECT * FROM mv_target ORDER BY a; From b2bfa6f9385aa84a3fd404983ffd9766f046ad1d Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 21 Jan 2020 17:11:21 +0100 Subject: [PATCH 2/3] Removing code used for debugging. --- dbms/src/DataStreams/ConvertingBlockInputStream.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dbms/src/DataStreams/ConvertingBlockInputStream.cpp b/dbms/src/DataStreams/ConvertingBlockInputStream.cpp index d1f5b2713a6..b40b622953e 100644 --- a/dbms/src/DataStreams/ConvertingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ConvertingBlockInputStream.cpp @@ -4,7 +4,6 @@ #include #include #include -#include namespace DB { @@ -69,10 +68,8 @@ ConvertingBlockInputStream::ConvertingBlockInputStream( case MatchColumnsMode::NameOrDefault: if (input_header.has(res_elem.name)) conversion[result_col_num] = input_header.getPositionByName(res_elem.name); - else { - std::cerr << "column " << result_col_num << " will be set to default\n"; + else conversion[result_col_num] = USE_DEFAULT; - } break; } From 507609b611de2480b5264275fc1c2989fd6531a2 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 21 Jan 2020 11:18:41 -0500 Subject: [PATCH 3/3] Fixing styling issues. --- .../ConvertingBlockInputStream.cpp | 21 ++++++++++--------- .../DataStreams/ConvertingBlockInputStream.h | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/dbms/src/DataStreams/ConvertingBlockInputStream.cpp b/dbms/src/DataStreams/ConvertingBlockInputStream.cpp index b40b622953e..693b8e34fb8 100644 --- a/dbms/src/DataStreams/ConvertingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ConvertingBlockInputStream.cpp @@ -68,13 +68,13 @@ ConvertingBlockInputStream::ConvertingBlockInputStream( case MatchColumnsMode::NameOrDefault: if (input_header.has(res_elem.name)) conversion[result_col_num] = input_header.getPositionByName(res_elem.name); - else + else conversion[result_col_num] = USE_DEFAULT; - break; + break; } - if (conversion[result_col_num] == USE_DEFAULT) - continue; + if (conversion[result_col_num] == USE_DEFAULT) + continue; const auto & src_elem = input_header.getByPosition(conversion[result_col_num]); @@ -112,14 +112,14 @@ Block ConvertingBlockInputStream::readImpl() auto & res_elem = res.getByPosition(res_pos); if (conversion[res_pos] == USE_DEFAULT) - { - // Create a column with default values - auto column_with_defaults = res_elem.type->createColumn()->cloneResized(src.rows()); - res_elem.column = std::move(column_with_defaults); - continue; + { + // Create a column with default values + auto column_with_defaults = res_elem.type->createColumn()->cloneResized(src.rows()); + res_elem.column = std::move(column_with_defaults); + continue; } - const auto & src_elem = src.getByPosition(conversion[res_pos]); + const auto & src_elem = src.getByPosition(conversion[res_pos]); ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context); @@ -132,3 +132,4 @@ Block ConvertingBlockInputStream::readImpl() } } + diff --git a/dbms/src/DataStreams/ConvertingBlockInputStream.h b/dbms/src/DataStreams/ConvertingBlockInputStream.h index 26d8190aa8b..89bbb3db378 100644 --- a/dbms/src/DataStreams/ConvertingBlockInputStream.h +++ b/dbms/src/DataStreams/ConvertingBlockInputStream.h @@ -30,7 +30,7 @@ public: /// Find columns in source by their names. Allow excessive columns in source. Name, /// Find columns in source by their names if present else use the default. Allow excessive columns in source. - NameOrDefault + NameOrDefault }; ConvertingBlockInputStream(