Strict order of evaluated and added columns

dbms/src/DataStreams/AddingDefaultBlockOutputStream.cpp

@@ -11,6 +11,7 @@
 namespace DB
 {
 
+
 void AddingDefaultBlockOutputStream::write(const Block & block)
 {
     Block res = block;
@@ -37,14 +38,15 @@ void AddingDefaultBlockOutputStream::write(const Block & block)
         }
     }
 
-    for (const auto & requested_column : required_columns)
+    size_t i = 0;
+    for (auto col = required_columns.begin(); col != required_columns.end(); ++col, ++i)
     {
-        if (res.has(requested_column.name) || column_defaults.count(requested_column.name))
+        if (res.has(col->name) || column_defaults.count(col->name))
             continue;
 
         ColumnWithTypeAndName column_to_add;
-        column_to_add.name = requested_column.name;
-        column_to_add.type = requested_column.type;
+        column_to_add.name = col->name;
+        column_to_add.type = col->type;
 
         String offsets_name = Nested::extractTableName(column_to_add.name);
         if (offset_columns.count(offsets_name))
@@ -64,7 +66,7 @@ void AddingDefaultBlockOutputStream::write(const Block & block)
             column_to_add.column = column_to_add.type->createColumnConstWithDefaultValue(rows)->convertToFullColumnIfConst();
         }
 
-        res.insert(std::move(column_to_add));
+        res.insert(i, std::move(column_to_add));
     }
 
     /// Computes explicitly specified values (in column_defaults) by default.

dbms/src/DataStreams/AddingDefaultBlockOutputStream.h

@@ -4,6 +4,7 @@
 #include <Columns/ColumnConst.h>
 #include <Storages/ColumnDefault.h>
 #include <Interpreters/Context.h>
+#include <iostream>
 
 
 namespace DB
@@ -36,6 +37,7 @@ public:
     void writeSuffix() override;
 
 private:
+
     BlockOutputStreamPtr output;
     Block header;
     NamesAndTypesList required_columns;

dbms/src/DataStreams/IProfilingBlockInputStream.cpp

@@ -81,14 +81,16 @@ Block IProfilingBlockInputStream::read()
 
     progress(Progress(res.rows(), res.bytes()));
 
-#ifndef NDEBUG
-    if (res)
-    {
-        Block header = getHeader();
-        if (header)
-            assertBlocksHaveEqualStructure(res, header, getName());
-    }
-#endif
+/// This code commented, because some streams (for example Native) break this
+/// protocol. This must be fixed.
+//#ifndef NDEBUG
+//    if (res)
+//    {
+//        Block header = getHeader();
+//        if (header)
+//            assertBlocksHaveEqualStructure(res, header, getName());
+//    }
+//#endif
 
     return res;
 }

dbms/src/DataStreams/SquashingBlockOutputStream.cpp

@@ -9,17 +9,15 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-SquashingBlockOutputStream::SquashingBlockOutputStream(BlockOutputStreamPtr & dst, size_t min_block_size_rows, size_t min_block_size_bytes)
-    : output(dst), transform(min_block_size_rows, min_block_size_bytes)
+SquashingBlockOutputStream::SquashingBlockOutputStream(BlockOutputStreamPtr & dst, const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
+    : output(dst), header(header), transform(min_block_size_rows, min_block_size_bytes)
 {
 }
 
 
 void SquashingBlockOutputStream::write(const Block & block)
 {
-    if (!header)
-        header = block.cloneEmpty();
-
+    /// Get header from real data
     SquashingTransform::Result result = transform.add(Block(block).mutateColumns());
     if (result.ready)
         output->write(header.cloneWithColumns(std::move(result.columns)));
@@ -33,8 +31,6 @@ void SquashingBlockOutputStream::finalize()
 
     all_written = true;
 
-    if (!header)
-        throw Exception("writeSuffix called without writing data.", ErrorCodes::LOGICAL_ERROR);
 
     SquashingTransform::Result result = transform.add({});
     if (result.ready && !result.columns.empty())

dbms/src/DataStreams/SquashingBlockOutputStream.h

@@ -12,7 +12,7 @@ namespace DB
 class SquashingBlockOutputStream : public IBlockOutputStream
 {
 public:
-    SquashingBlockOutputStream(BlockOutputStreamPtr & dst, size_t min_block_size_rows, size_t min_block_size_bytes);
+    SquashingBlockOutputStream(BlockOutputStreamPtr & dst, const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes);
 
     Block getHeader() const override { return header; }
     void write(const Block & block) override;

dbms/src/Formats/PrettyCompactBlockOutputStream.cpp

@@ -152,7 +152,7 @@ void registerOutputFormatPrettyCompact(FormatFactory & factory)
         const FormatSettings & format_settings)
     {
         BlockOutputStreamPtr impl = std::make_shared<PrettyCompactBlockOutputStream>(buf, sample, format_settings);
-        auto res = std::make_shared<SquashingBlockOutputStream>(impl, format_settings.pretty.max_rows, 0);
+        auto res = std::make_shared<SquashingBlockOutputStream>(impl, impl->getHeader(), format_settings.pretty.max_rows, 0);
         res->disableFlush();
         return res;
     });

dbms/src/Interpreters/evaluateMissingDefaults.cpp

@@ -50,12 +50,17 @@ void evaluateMissingDefaults(Block & block,
     ExpressionAnalyzer{default_expr_list, context, {}, available_columns}.getActions(true)->execute(copy_block);
 
     /// move evaluated columns to the original block, materializing them at the same time
-    for (auto & column_name_type : copy_block)
+    size_t pos = 0;
+    for (auto col = required_columns.begin(); col != required_columns.end(); ++col, ++pos)
     {
-        if (ColumnPtr converted = column_name_type.column->convertToFullColumnIfConst())
-            column_name_type.column = converted;
+        if (copy_block.has(col->name))
+        {
+            auto evaluated_col = copy_block.getByName(col->name);
+            if (ColumnPtr converted = evaluated_col.column->convertToFullColumnIfConst())
+                evaluated_col.column = converted;
 
-        block.insert(std::move(column_name_type));
+            block.insert(pos, std::move(evaluated_col));
+        }
     }
 }
 

dbms/src/Storages/ITableDeclaration.cpp

@@ -47,7 +47,7 @@ Block ITableDeclaration::getSampleBlock() const
 {
     Block res;
 
-    for (const auto & col : boost::join(getColumns().ordinary, getColumns().materialized))
+    for (const auto & col : getColumns().getAllPhysical())
         res.insert({ col.type->createColumn(), col.type, col.name });
 
     return res;

dbms/tests/queries/0_stateless/00719_insert_block_without_column.sh

@@ -5,11 +5,17 @@ set -e
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
+[ -e ${CLICKHOUSE_TMP}/test_squashing_block_without_column.out ] && rm ${CLICKHOUSE_TMP}/test_squashing_block_without_column.out
+
 ${CLICKHOUSE_CLIENT} --query "select number as SomeID, number+1 as OtherID from system.numbers limit 1000 into outfile '${CLICKHOUSE_TMP}/test_squashing_block_without_column.out' format Native"
 
 ${CLICKHOUSE_CLIENT} --query "drop table if exists test.squashed_numbers"
 ${CLICKHOUSE_CLIENT} --query "create table test.squashed_numbers (SomeID UInt64, DifferentID UInt64, OtherID UInt64) engine Memory"
 
-cat ${CLICKHOUSE_TMP}/test_squashing_block_without_column.out | ${CLICKHOUSE_CLIENT} --query "insert into test.squashed_numbers format Native"
+address=${CLICKHOUSE_HOST}
+port=${CLICKHOUSE_PORT_HTTP}
+url="${CLICKHOUSE_PORT_HTTP_PROTO}://$address:$port/"
+
+${CLICKHOUSE_CURL} -sS --data-binary "@${CLICKHOUSE_TMP}/test_squashing_block_without_column.out" "${url}?query=insert%20into%20test.squashed_numbers%20format%20Native"
 
 ${CLICKHOUSE_CLIENT} --query "select 'Still alive'"