From 597778c074368b7f1759f1b157c927c156eac911 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 6 Jul 2018 22:19:06 +0300 Subject: [PATCH 01/29] metadata in capn proto (in progress) --- dbms/src/Proto/protoHelpers.cpp | 81 ++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 16 deletions(-) diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index 9af677156b7..567faaba2f0 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -4,18 +4,39 @@ #include #include #include +#include +#include #include #include #include #include -#include -#include -#include +#include namespace DB { + template + static MutableColumnPtr serializeProto(const ColumnT & column_type, capnp::MessageBuilder & message) + { + MutableColumnPtr data = column_type.createColumn(); + + kj::Array serialized = messageToFlatArray(message); + + data->insertData(reinterpret_cast(serialized.begin()), serialized.size() * sizeof(capnp::word)); + return data; + } + + template + typename T::Reader deserializeProto(const char * data, size_t data_size) + { + const capnp::word * ptr = reinterpret_cast(data); + auto serialized = kj::arrayPtr(ptr, data_size / sizeof(capnp::word)); + + capnp::FlatArrayMessageReader reader(serialized); + return reader.getRoot(); + } + ColumnWithTypeAndName storeContext(Context & context) { capnp::MallocMessageBuilder message; @@ -76,23 +97,51 @@ namespace DB ColumnWithTypeAndName proto_column; proto_column.name = "context"; proto_column.type = std::make_shared(); - MutableColumnPtr data = proto_column.type->createColumn(); - - kj::Array serialized = messageToFlatArray(message); - data->insertData(reinterpret_cast(serialized.begin()), serialized.size() * sizeof(capnp::word)); - - proto_column.column = std::move(data); + proto_column.column = std::move(serializeProto(*proto_column.type, message)); return proto_column; } - void loadContext(const ColumnWithTypeAndName & , Context & ) + void loadContext(const ColumnWithTypeAndName & proto_column, Context & context) { -#if 0 - kj::Array messageToFlatArray(MessageBuilder& builder); + StringRef plain_data = proto_column.column->getDataAt(0); + size_t data_size = proto_column.column->byteSize(); + Proto::Context::Reader proto_context = deserializeProto(plain_data.data, data_size); - capnp::MallocMessageBuilder message; - Proto::ServerMessage::Builder serverMessage = message.initRoot(); - /// TODO -#endif + // or ParserCompoundColumnDeclaration ? + ParserColumnDeclaration parser_defaults; + + for (auto proto_database : proto_context.getDatabases()) + { + String database_name = proto_database.getName().cStr(); + if (!context.isDatabaseExist(database_name)) + { + // TODO + } + + for (auto proto_table : proto_database.getTables()) + { + String table_name = proto_table.getName().cStr(); + if (!context.isTableExist(database_name, table_name)) + { + // TODO + } + + StoragePtr table = context.tryGetTable(database_name, table_name); + // TODO: throw on fail + + ColumnsDescription column_description; + for (auto column : proto_table.getColumns()) + { + String column_name = column.getName().cStr(); + String expression = column.getDefault().getExpression().cStr(); + ColumnDefaultKind expression_kind = static_cast(column.getDefault().getKind()); + ASTPtr ast = parseQuery(parser_defaults, expression, expression.size()); + + column_description.defaults[column_name] = ColumnDefault{expression_kind, ast}; + } + + table->setColumns(column_description); + } + } } } From 2e4c2328af07de9e229da4f498f288255d0155fd Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 9 Jul 2018 19:31:24 +0300 Subject: [PATCH 02/29] metadata in capn proto format (in progress - need apply on client) --- dbms/programs/client/Client.cpp | 2 +- dbms/programs/server/TCPHandler.cpp | 19 ++++++++++ dbms/programs/server/TCPHandler.h | 1 + dbms/src/Client/Connection.cpp | 1 + dbms/src/Proto/protoHelpers.cpp | 59 ++++++++++++++++++++--------- dbms/src/Proto/protoHelpers.h | 6 +-- 6 files changed, 67 insertions(+), 21 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 84d2d640d80..3c7f5e013ce 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1087,7 +1087,7 @@ private: return false; case Protocol::Server::CapnProto: - loadContext(packet.block.getColumnsWithTypeAndName()[0], context); + loadContextBlock(packet.block, context); return receiveSampleBlock(out); default: diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 53ca6c8699f..128dc0090bd 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -28,6 +28,8 @@ #include +#include + #include "TCPHandler.h" #include @@ -310,6 +312,10 @@ void TCPHandler::processInsertQuery(const Settings & global_settings) */ state.io.out->writePrefix(); + /// Send query metadata (column defaults) + Block meta_block = storeContextBlock(query_context); + sendMetadata(meta_block); + /// Send block to the client - table structure. Block block = state.io.out->getHeader(); sendData(block); @@ -762,6 +768,19 @@ void TCPHandler::sendData(const Block & block) } +void TCPHandler::sendMetadata(const Block & block) +{ + initBlockOutput(block); + + writeVarUInt(Protocol::Server::CapnProto, *out); + writeStringBinary("", *out); + + state.block_out->write(block); + state.maybe_compressed_out->next(); + out->next(); +} + + void TCPHandler::sendException(const Exception & e) { writeVarUInt(Protocol::Server::Exception, *out); diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index e01987d3bbd..0a764ac9e2a 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -139,6 +139,7 @@ private: void sendHello(); void sendData(const Block & block); /// Write a block to the network. + void sendMetadata(const Block & block); void sendException(const Exception & e); void sendProgress(); void sendEndOfStream(); diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index c461b4cafde..7dce0afa0c0 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -521,6 +521,7 @@ Connection::Packet Connection::receivePacket() switch (res.type) { case Protocol::Server::Data: + case Protocol::Server::CapnProto: res.block = receiveData(); return res; diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index 567faaba2f0..a95e5be8964 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -5,25 +5,30 @@ #include #include #include -#include +#include #include #include +#include #include #include #include +/// @sa https://capnproto.org/cxx.html namespace DB { - template - static MutableColumnPtr serializeProto(const ColumnT & column_type, capnp::MessageBuilder & message) + static MutableColumnPtr serializeProto(capnp::MessageBuilder & message) { - MutableColumnPtr data = column_type.createColumn(); + MutableColumnPtr data = DataTypeUInt8().createColumn(); kj::Array serialized = messageToFlatArray(message); + kj::ArrayPtr bytes = serialized.asChars(); + + data->reserve(bytes.size()); + for (size_t i = 0 ; i < bytes.size(); ++i) + data->insertData(&bytes[i], 1); - data->insertData(reinterpret_cast(serialized.begin()), serialized.size() * sizeof(capnp::word)); return data; } @@ -37,7 +42,7 @@ namespace DB return reader.getRoot(); } - ColumnWithTypeAndName storeContext(Context & context) + static ColumnWithTypeAndName storeContext(const String & column_name, Context & context) { capnp::MallocMessageBuilder message; Proto::Context::Builder proto_context = message.initRoot(); @@ -48,11 +53,14 @@ namespace DB size_t db_nomber = 0; for (auto & pr_db : dbs) { - const String& db_name = pr_db.first; - IDatabase& db = *pr_db.second; + const String & database_name = pr_db.first; + if (database_name == "system") + continue; + + IDatabase & db = *pr_db.second; auto proto_db = proto_databases[db_nomber]; - proto_db.setName(db_name); + proto_db.setName(database_name); std::unordered_map tables; DatabaseIteratorPtr it_tables = db.getIterator(context); @@ -95,24 +103,23 @@ namespace DB } ColumnWithTypeAndName proto_column; - proto_column.name = "context"; - proto_column.type = std::make_shared(); - proto_column.column = std::move(serializeProto(*proto_column.type, message)); + proto_column.name = column_name; + proto_column.type = std::make_shared(); + proto_column.column = std::move(serializeProto(message)); return proto_column; } - void loadContext(const ColumnWithTypeAndName & proto_column, Context & context) + static void loadContext(const ColumnWithTypeAndName & proto_column, Context & context) { StringRef plain_data = proto_column.column->getDataAt(0); size_t data_size = proto_column.column->byteSize(); Proto::Context::Reader proto_context = deserializeProto(plain_data.data, data_size); - // or ParserCompoundColumnDeclaration ? - ParserColumnDeclaration parser_defaults; + ParserExpressionElement parser; for (auto proto_database : proto_context.getDatabases()) { - String database_name = proto_database.getName().cStr(); + const String & database_name = proto_database.getName().cStr(); if (!context.isDatabaseExist(database_name)) { // TODO @@ -135,8 +142,8 @@ namespace DB String column_name = column.getName().cStr(); String expression = column.getDefault().getExpression().cStr(); ColumnDefaultKind expression_kind = static_cast(column.getDefault().getKind()); - ASTPtr ast = parseQuery(parser_defaults, expression, expression.size()); + ASTPtr ast = parseQuery(parser, expression, expression.size()); column_description.defaults[column_name] = ColumnDefault{expression_kind, ast}; } @@ -144,4 +151,22 @@ namespace DB } } } + + static constexpr const char * contextColumnName() + { + return "context"; + } + + Block storeContextBlock(Context & context) + { + Block block; + block.insert(storeContext(contextColumnName(), context)); + return block; + } + + void loadContextBlock(const Block & block, Context & context) + { + const ColumnWithTypeAndName & column = block.getByName(contextColumnName()); + loadContext(column, context); + } } diff --git a/dbms/src/Proto/protoHelpers.h b/dbms/src/Proto/protoHelpers.h index 64c99995722..14512d3453c 100644 --- a/dbms/src/Proto/protoHelpers.h +++ b/dbms/src/Proto/protoHelpers.h @@ -4,8 +4,8 @@ namespace DB { class Context; - struct ColumnWithTypeAndName; + class Block; - ColumnWithTypeAndName storeContext(Context & context); - void loadContext(const ColumnWithTypeAndName & proto_column, Context & context); + Block storeContextBlock(Context & context); + void loadContextBlock(const Block & block, Context & context); } From bac1f77620492b5096600479972cba42257db743 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 10 Jul 2018 14:40:33 +0300 Subject: [PATCH 03/29] insert defaults (first working version) --- dbms/programs/client/Client.cpp | 28 +++++++++++----------------- dbms/src/Proto/protoHelpers.cpp | 28 +++++++++------------------- dbms/src/Proto/protoHelpers.h | 14 +++++++++++++- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 3c7f5e013ce..414ba81f3c5 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -865,11 +865,12 @@ private: /// Receive description of table structure. Block sample; - if (receiveSampleBlock(sample)) + TableMetaInfo table_meta(parsed_insert_query.database, parsed_insert_query.table); + if (receiveSampleBlock(sample, table_meta)) { /// If structure was received (thus, server has not thrown an exception), /// send our data with that structure. - sendData(sample); + sendData(sample, table_meta); receivePacket(); } } @@ -907,7 +908,7 @@ private: } - void sendData(Block & sample) + void sendData(Block & sample, const TableMetaInfo & table_meta) { /// If INSERT data must be sent. const ASTInsertQuery * parsed_insert_query = typeid_cast(&*parsed_query); @@ -918,35 +919,28 @@ private: { /// Send data contained in the query. ReadBufferFromMemory data_in(parsed_insert_query->data, parsed_insert_query->end - parsed_insert_query->data); - sendDataFrom(data_in, sample); + sendDataFrom(data_in, sample, table_meta); } else if (!is_interactive) { /// Send data read from stdin. - sendDataFrom(std_in, sample); + sendDataFrom(std_in, sample, table_meta); } else throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT); } - void sendDataFrom(ReadBuffer & buf, Block & sample) + void sendDataFrom(ReadBuffer & buf, Block & sample, const TableMetaInfo & table_meta) { String current_format = insert_format; - ColumnDefaults column_defaults; + const ColumnDefaults & column_defaults = table_meta.column_defaults; /// Data format can be specified in the INSERT query. if (ASTInsertQuery * insert = typeid_cast(&*parsed_query)) { if (!insert->format.empty()) current_format = insert->format; - - if (context.isTableExist(insert->database, insert->table)) - { - StoragePtr table = context.getTable(insert->database, insert->table); - if (table) - column_defaults = table->getColumns().defaults; - } } BlockInputStreamPtr block_input = context.getInputFormat( @@ -1071,7 +1065,7 @@ private: /// Receive the block that serves as an example of the structure of table where data will be inserted. - bool receiveSampleBlock(Block & out) + bool receiveSampleBlock(Block & out, TableMetaInfo & table_meta) { Connection::Packet packet = connection->receivePacket(); @@ -1087,8 +1081,8 @@ private: return false; case Protocol::Server::CapnProto: - loadContextBlock(packet.block, context); - return receiveSampleBlock(out); + loadTableMetaInfo(packet.block, table_meta); + return receiveSampleBlock(out, table_meta); default: throw NetException("Unexpected packet from server (expected Data, got " diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index a95e5be8964..2ae1fda40f4 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -109,7 +109,7 @@ namespace DB return proto_column; } - static void loadContext(const ColumnWithTypeAndName & proto_column, Context & context) + static void loadTableMetaInfo(const ColumnWithTypeAndName & proto_column, TableMetaInfo & table_meta) { StringRef plain_data = proto_column.column->getDataAt(0); size_t data_size = proto_column.column->byteSize(); @@ -120,23 +120,15 @@ namespace DB for (auto proto_database : proto_context.getDatabases()) { const String & database_name = proto_database.getName().cStr(); - if (!context.isDatabaseExist(database_name)) - { - // TODO - } + if (database_name != table_meta.database) + continue; for (auto proto_table : proto_database.getTables()) { String table_name = proto_table.getName().cStr(); - if (!context.isTableExist(database_name, table_name)) - { - // TODO - } + if (table_name != table_meta.table) + continue; - StoragePtr table = context.tryGetTable(database_name, table_name); - // TODO: throw on fail - - ColumnsDescription column_description; for (auto column : proto_table.getColumns()) { String column_name = column.getName().cStr(); @@ -144,10 +136,8 @@ namespace DB ColumnDefaultKind expression_kind = static_cast(column.getDefault().getKind()); ASTPtr ast = parseQuery(parser, expression, expression.size()); - column_description.defaults[column_name] = ColumnDefault{expression_kind, ast}; + table_meta.column_defaults.emplace(column_name, ColumnDefault{expression_kind, ast}); } - - table->setColumns(column_description); } } } @@ -164,9 +154,9 @@ namespace DB return block; } - void loadContextBlock(const Block & block, Context & context) + void loadTableMetaInfo(const Block & block, TableMetaInfo & table_meta) { const ColumnWithTypeAndName & column = block.getByName(contextColumnName()); - loadContext(column, context); + loadTableMetaInfo(column, table_meta); } } diff --git a/dbms/src/Proto/protoHelpers.h b/dbms/src/Proto/protoHelpers.h index 14512d3453c..ee3da1649a1 100644 --- a/dbms/src/Proto/protoHelpers.h +++ b/dbms/src/Proto/protoHelpers.h @@ -1,11 +1,23 @@ #pragma once +#include namespace DB { class Context; class Block; + struct TableMetaInfo + { + TableMetaInfo(const String & database_, const String & table_) + : database(database_), table(table_) + {} + + const String & database; + const String & table; + ColumnDefaults column_defaults; + }; + Block storeContextBlock(Context & context); - void loadContextBlock(const Block & block, Context & context); + void loadTableMetaInfo(const Block & block, TableMetaInfo & table_meta); } From 5036309d26f0423b5adeab0dd59e96d5525c2701 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 10 Jul 2018 16:33:41 +0300 Subject: [PATCH 04/29] fix const defaults --- dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp index 82c36bfab85..99f5f28f88d 100644 --- a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -69,7 +69,12 @@ Block AddingDefaultsBlockInputStream::readImpl() for (size_t row_idx = 0; row_idx < column_read.column->size(); ++row_idx) { if (mask[row_idx]) - column_mixed->insertFrom(*column_def.column, row_idx); + { + if (column_def.column->isColumnConst()) + column_mixed->insert((*column_def.column)[row_idx]); + else + column_mixed->insertFrom(*column_def.column, row_idx); + } else column_mixed->insertFrom(*column_read.column, row_idx); } From 519102b11037759ce0b3b44cddffbc07f6516b93 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 10 Jul 2018 20:20:55 +0300 Subject: [PATCH 05/29] defaults for http inserts --- dbms/programs/client/Client.cpp | 11 +++++---- .../InputStreamFromASTInsertQuery.cpp | 10 ++++++-- dbms/src/Proto/protoHelpers.cpp | 7 +++--- dbms/src/Proto/protoHelpers.h | 16 ++----------- dbms/src/Storages/TableMetadata.cpp | 14 +++++++++++ dbms/src/Storages/TableMetadata.h | 23 +++++++++++++++++++ 6 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 dbms/src/Storages/TableMetadata.cpp create mode 100644 dbms/src/Storages/TableMetadata.h diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 414ba81f3c5..da18cbad6e1 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -55,6 +55,7 @@ #include #include #include +#include #include /// http://en.wikipedia.org/wiki/ANSI_escape_code @@ -865,7 +866,7 @@ private: /// Receive description of table structure. Block sample; - TableMetaInfo table_meta(parsed_insert_query.database, parsed_insert_query.table); + TableMetadata table_meta(parsed_insert_query.database, parsed_insert_query.table); if (receiveSampleBlock(sample, table_meta)) { /// If structure was received (thus, server has not thrown an exception), @@ -908,7 +909,7 @@ private: } - void sendData(Block & sample, const TableMetaInfo & table_meta) + void sendData(Block & sample, const TableMetadata & table_meta) { /// If INSERT data must be sent. const ASTInsertQuery * parsed_insert_query = typeid_cast(&*parsed_query); @@ -931,7 +932,7 @@ private: } - void sendDataFrom(ReadBuffer & buf, Block & sample, const TableMetaInfo & table_meta) + void sendDataFrom(ReadBuffer & buf, Block & sample, const TableMetadata & table_meta) { String current_format = insert_format; const ColumnDefaults & column_defaults = table_meta.column_defaults; @@ -1065,7 +1066,7 @@ private: /// Receive the block that serves as an example of the structure of table where data will be inserted. - bool receiveSampleBlock(Block & out, TableMetaInfo & table_meta) + bool receiveSampleBlock(Block & out, TableMetadata & table_meta) { Connection::Packet packet = connection->receivePacket(); @@ -1081,7 +1082,7 @@ private: return false; case Protocol::Server::CapnProto: - loadTableMetaInfo(packet.block, table_meta); + loadTableMetadata(packet.block, table_meta); return receiveSampleBlock(out, table_meta); default: diff --git a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp index 0e4f876925d..163fee44ea5 100644 --- a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp +++ b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp @@ -4,7 +4,8 @@ #include #include #include - +#include +#include namespace DB { @@ -43,7 +44,12 @@ InputStreamFromASTInsertQuery::InputStreamFromASTInsertQuery( input_buffer_contacenated = std::make_unique(buffers); - res_stream = context.getInputFormat(format, *input_buffer_contacenated, streams.out->getHeader(), context.getSettings().max_insert_block_size); + TableMetadata table_meta(ast_insert_query->database, ast_insert_query->table); + table_meta.loadFromContext(context); + + BlockInputStreamPtr block_input = + context.getInputFormat(format, *input_buffer_contacenated, streams.out->getHeader(), context.getSettings().max_insert_block_size); + res_stream = std::make_shared(block_input, table_meta.column_defaults, context); } } diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index 2ae1fda40f4..5121529c207 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -109,7 +110,7 @@ namespace DB return proto_column; } - static void loadTableMetaInfo(const ColumnWithTypeAndName & proto_column, TableMetaInfo & table_meta) + static void loadTableMetadata(const ColumnWithTypeAndName & proto_column, TableMetadata & table_meta) { StringRef plain_data = proto_column.column->getDataAt(0); size_t data_size = proto_column.column->byteSize(); @@ -154,9 +155,9 @@ namespace DB return block; } - void loadTableMetaInfo(const Block & block, TableMetaInfo & table_meta) + void loadTableMetadata(const Block & block, TableMetadata & table_meta) { const ColumnWithTypeAndName & column = block.getByName(contextColumnName()); - loadTableMetaInfo(column, table_meta); + loadTableMetadata(column, table_meta); } } diff --git a/dbms/src/Proto/protoHelpers.h b/dbms/src/Proto/protoHelpers.h index ee3da1649a1..c0a514a5bb1 100644 --- a/dbms/src/Proto/protoHelpers.h +++ b/dbms/src/Proto/protoHelpers.h @@ -1,23 +1,11 @@ #pragma once -#include - namespace DB { class Context; class Block; - - struct TableMetaInfo - { - TableMetaInfo(const String & database_, const String & table_) - : database(database_), table(table_) - {} - - const String & database; - const String & table; - ColumnDefaults column_defaults; - }; + class TableMetadata; Block storeContextBlock(Context & context); - void loadTableMetaInfo(const Block & block, TableMetaInfo & table_meta); + void loadTableMetadata(const Block & block, TableMetadata & table_meta); } diff --git a/dbms/src/Storages/TableMetadata.cpp b/dbms/src/Storages/TableMetadata.cpp new file mode 100644 index 00000000000..e07c6a31e97 --- /dev/null +++ b/dbms/src/Storages/TableMetadata.cpp @@ -0,0 +1,14 @@ +#include +#include +#include "TableMetadata.h" + + +namespace DB +{ + void TableMetadata::loadFromContext(const Context & context) + { + StoragePtr storage = context.getTable(database, table); + const ColumnsDescription & table_columns = storage->getColumns(); + column_defaults = table_columns.defaults; + } +} diff --git a/dbms/src/Storages/TableMetadata.h b/dbms/src/Storages/TableMetadata.h new file mode 100644 index 00000000000..a88808ed96a --- /dev/null +++ b/dbms/src/Storages/TableMetadata.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +namespace DB +{ + class Context; + class Block; + + /// Addition information for query that could not be get from sample block + struct TableMetadata + { + TableMetadata(const String & database_, const String & table_) + : database(database_), table(table_) + {} + + const String & database; + const String & table; + ColumnDefaults column_defaults; + + void loadFromContext(const Context & context); + }; +} From fe1b393e55636aac2e8743560281711854599f5a Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 11 Jul 2018 15:05:04 +0300 Subject: [PATCH 06/29] defaults for storages with formated input --- dbms/programs/client/Client.cpp | 5 +++-- dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp | 8 +++++--- dbms/src/Storages/StorageFile.cpp | 7 ++++++- dbms/src/Storages/StorageKafka.cpp | 5 +++++ dbms/src/Storages/StorageURL.cpp | 10 ++++++++-- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index da18cbad6e1..af16a4998fb 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -947,8 +947,9 @@ private: BlockInputStreamPtr block_input = context.getInputFormat( current_format, buf, sample, insert_format_max_block_size); - BlockInputStreamPtr defs_block_input = std::make_shared(block_input, column_defaults, context); - BlockInputStreamPtr async_block_input = std::make_shared(defs_block_input); + if (!column_defaults.empty()) + block_input = std::make_shared(block_input, column_defaults, context); + BlockInputStreamPtr async_block_input = std::make_shared(block_input); async_block_input->readPrefix(); diff --git a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp index 163fee44ea5..945fc782327 100644 --- a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp +++ b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp @@ -47,9 +47,11 @@ InputStreamFromASTInsertQuery::InputStreamFromASTInsertQuery( TableMetadata table_meta(ast_insert_query->database, ast_insert_query->table); table_meta.loadFromContext(context); - BlockInputStreamPtr block_input = - context.getInputFormat(format, *input_buffer_contacenated, streams.out->getHeader(), context.getSettings().max_insert_block_size); - res_stream = std::make_shared(block_input, table_meta.column_defaults, context); + res_stream = context.getInputFormat( + format, *input_buffer_contacenated, streams.out->getHeader(), context.getSettings().max_insert_block_size); + + if (!table_meta.column_defaults.empty()) + res_stream = std::make_shared(res_stream, table_meta.column_defaults, context); } } diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index c63e3a263ed..f24686cfea3 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -194,7 +195,11 @@ BlockInputStreams StorageFile::read( size_t max_block_size, unsigned /*num_streams*/) { - return BlockInputStreams(1, std::make_shared(*this, context, max_block_size)); + BlockInputStreamPtr block_input = std::make_shared(*this, context, max_block_size); + const ColumnsDescription & columns = getColumns(); + if (columns.defaults.empty()) + return {block_input}; + return {std::make_shared(block_input, columns.defaults, context)}; } diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp index a9666bab22c..9afbbfd656f 100644 --- a/dbms/src/Storages/StorageKafka.cpp +++ b/dbms/src/Storages/StorageKafka.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -145,6 +146,10 @@ public: LOG_TRACE(storage.log, "Creating formatted reader"); read_buf = std::make_unique(consumer->stream, storage.log); reader = FormatFactory::instance().getInput(storage.format_name, *read_buf, storage.getSampleBlock(), context, max_block_size); + + const ColumnsDescription & columns = getColumns(); + if (!columns.defaults.empty()) + reader = std::make_shared(reader, columns.defaults, context); } ~KafkaBlockInputStream() override diff --git a/dbms/src/Storages/StorageURL.cpp b/dbms/src/Storages/StorageURL.cpp index 1c3b8246492..fa8c30c24a6 100644 --- a/dbms/src/Storages/StorageURL.cpp +++ b/dbms/src/Storages/StorageURL.cpp @@ -13,6 +13,7 @@ #include #include +#include #include @@ -135,14 +136,19 @@ BlockInputStreams StorageURL::read( size_t max_block_size, unsigned /*num_streams*/) { - return {std::make_shared( + BlockInputStreamPtr block_input = std::make_shared( uri, format_name, getName(), getSampleBlock(), context, max_block_size, - ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef()))}; + ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef())); + + const ColumnsDescription & columns = getColumns(); + if (columns.defaults.empty()) + return {block_input}; + return {std::make_shared(block_input, columns.defaults, context)}; } void StorageURL::rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/) {} From 82d22574d40370e6c1c54ed682e8abd2f6403c92 Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 11 Jul 2018 17:11:47 +0300 Subject: [PATCH 07/29] fix empty defaults mask issue and add some comments --- dbms/src/Core/Block.h | 2 + dbms/src/Core/BlockInfo.cpp | 6 +-- dbms/src/Core/BlockInfo.h | 16 ++++---- .../AddingDefaultsBlockInputStream.cpp | 37 ++++++++++--------- dbms/src/Formats/IRowInputStream.h | 6 +-- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/dbms/src/Core/Block.h b/dbms/src/Core/Block.h index b7c19548963..0db8954247e 100644 --- a/dbms/src/Core/Block.h +++ b/dbms/src/Core/Block.h @@ -35,6 +35,8 @@ private: public: BlockInfo info; + /// Input stream could use delayed_defaults to add addition info at which rows it have inserted default values. + /// Such values would be replaced later by column defaults in AddingDefaultsBlockInputStream (if any). BlockDelayedDefaults delayed_defaults; Block() = default; diff --git a/dbms/src/Core/BlockInfo.cpp b/dbms/src/Core/BlockInfo.cpp index 15d7d9efa12..f214d2782e3 100644 --- a/dbms/src/Core/BlockInfo.cpp +++ b/dbms/src/Core/BlockInfo.cpp @@ -60,14 +60,14 @@ void BlockInfo::read(ReadBuffer & in) void BlockDelayedDefaults::setBit(size_t column_idx, size_t row_idx) { - BitMask & mask = columns_defaults[column_idx]; + RowsBitMask & mask = columns_defaults[column_idx]; mask.resize(row_idx + 1); mask[row_idx] = true; } -const BlockDelayedDefaults::BitMask & BlockDelayedDefaults::getColumnBitmask(size_t column_idx) const +const BlockDelayedDefaults::RowsBitMask & BlockDelayedDefaults::getDefaultsBitmask(size_t column_idx) const { - static BitMask none; + static RowsBitMask none; auto it = columns_defaults.find(column_idx); if (it != columns_defaults.end()) return it->second; diff --git a/dbms/src/Core/BlockInfo.h b/dbms/src/Core/BlockInfo.h index 00c0821a33e..abc5383ddcb 100644 --- a/dbms/src/Core/BlockInfo.h +++ b/dbms/src/Core/BlockInfo.h @@ -45,22 +45,24 @@ struct BlockInfo void read(ReadBuffer & in); }; -/// Block extention to support delayed defaults. -/// It's expected that it would be lots unset defaults or none. -/// NOTE It's possible to make better solution for sparse values. +/// Block extention to support delayed defaults. Used in AddingDefaultsBlockInputStream to replace type defauls set by RowInputStream +/// with column defaults. class BlockDelayedDefaults { public: - using BitMask = std::vector; - using MaskById = std::unordered_map; + using RowsBitMask = std::vector; /// a bit per row for a column - const BitMask & getColumnBitmask(size_t column_idx) const; + const RowsBitMask & getDefaultsBitmask(size_t column_idx) const; void setBit(size_t column_idx, size_t row_idx); bool empty() const { return columns_defaults.empty(); } size_t size() const { return columns_defaults.size(); } private: - MaskById columns_defaults; + using RowsMaskByColumnId = std::unordered_map; + + /// If columns_defaults[column_id][row_id] is true related value in Block should be replaced with column default. + /// It could contain less columns and rows then related block. + RowsMaskByColumnId columns_defaults; }; } diff --git a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp index 99f5f28f88d..ffde573a24f 100644 --- a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -63,25 +63,28 @@ Block AddingDefaultsBlockInputStream::readImpl() if (column_read.column->size() != column_def.column->size()) throw Exception("Mismach column sizes while adding defaults", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); - const BlockDelayedDefaults::BitMask & mask = delayed_defaults.getColumnBitmask(block_column_position); - MutableColumnPtr column_mixed = column_read.column->cloneEmpty(); - - for (size_t row_idx = 0; row_idx < column_read.column->size(); ++row_idx) + const auto & defaults_mask = delayed_defaults.getDefaultsBitmask(block_column_position); + if (!defaults_mask.empty()) { - if (mask[row_idx]) - { - if (column_def.column->isColumnConst()) - column_mixed->insert((*column_def.column)[row_idx]); - else - column_mixed->insertFrom(*column_def.column, row_idx); - } - else - column_mixed->insertFrom(*column_read.column, row_idx); - } + MutableColumnPtr column_mixed = column_read.column->cloneEmpty(); - ColumnWithTypeAndName mix = column_read.cloneEmpty(); - mix.column = std::move(column_mixed); - mixed_columns.emplace_back(std::move(mix)); + for (size_t row_idx = 0; row_idx < column_read.column->size(); ++row_idx) + { + if (defaults_mask[row_idx]) + { + if (column_def.column->isColumnConst()) + column_mixed->insert((*column_def.column)[row_idx]); + else + column_mixed->insertFrom(*column_def.column, row_idx); + } + else + column_mixed->insertFrom(*column_read.column, row_idx); + } + + ColumnWithTypeAndName mix = column_read.cloneEmpty(); + mix.column = std::move(column_mixed); + mixed_columns.emplace_back(std::move(mix)); + } } for (auto & column : mixed_columns) diff --git a/dbms/src/Formats/IRowInputStream.h b/dbms/src/Formats/IRowInputStream.h index 1ffa205edc6..8b3e7043f3b 100644 --- a/dbms/src/Formats/IRowInputStream.h +++ b/dbms/src/Formats/IRowInputStream.h @@ -10,13 +10,13 @@ namespace DB { -/// A way to set some extentions to read and return extra information too. +/// A way to set some extentions to read and return extra information too. IRowInputStream.extendedRead() output. struct RowReadExtention { using BitMask = std::vector; - /// IRowInputStream.extendedRead() output value. - /// Contains true for columns that actually read from the source and false for defaults + /// Contains one bit per column in resently read row. IRowInputStream could leave it empty, or partialy set. + /// It should contain true for columns that actually read from the source and false for defaults. BitMask read_columns; }; From a765aef4dbdc3ea3a362b25161d82584301c61fe Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 11 Jul 2018 19:24:29 +0300 Subject: [PATCH 08/29] fix KafkaStorage build and add sql test for defaults --- dbms/src/Storages/StorageKafka.cpp | 2 +- .../00651_insert_json_with_defaults.reference | 7 +++++++ .../00651_insert_json_with_defaults.sql | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.reference create mode 100644 dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp index 9afbbfd656f..43966aec22a 100644 --- a/dbms/src/Storages/StorageKafka.cpp +++ b/dbms/src/Storages/StorageKafka.cpp @@ -147,7 +147,7 @@ public: read_buf = std::make_unique(consumer->stream, storage.log); reader = FormatFactory::instance().getInput(storage.format_name, *read_buf, storage.getSampleBlock(), context, max_block_size); - const ColumnsDescription & columns = getColumns(); + const ColumnsDescription & columns = storage.getColumns(); if (!columns.defaults.empty()) reader = std::make_shared(reader, columns.defaults, context); } diff --git a/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.reference b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.reference new file mode 100644 index 00000000000..f513c6d6f40 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.reference @@ -0,0 +1,7 @@ +0 0 6 6 6 +0 5 5 1.7917595 5 +1 1 2 1.0986123 42 +1 1 2 1.0986123 42 +2 2 4 1.609438 2 +3 3 3 3 3 +4 0 4 1.609438 42 diff --git a/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql new file mode 100644 index 00000000000..d25ecdc1f5a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql @@ -0,0 +1,19 @@ +CREATE DATABASE IF NOT EXISTS test; +DROP TABLE IF EXISTS test.defaults; +CREATE TABLE IF NOT EXISTS test.defaults +( + x UInt32, + y UInt32, + a UInt32 DEFAULT x + y, + b Float32 DEFAULT log(1 + x + y), + c UInt32 DEFAULT 42 +) ENGINE = Memory; + +INSERT INTO test.defaults FORMAT JSONEachRow {"x":1, "y":1}; +INSERT INTO test.defaults (x, y) SELECT x, y FROM test.defaults LIMIT 1; +INSERT INTO test.defaults FORMAT JSONEachRow {"x":2, "y":2, "c":2}; +INSERT INTO test.defaults FORMAT JSONEachRow {"x":3, "y":3, "a":3, "b":3, "c":3}; +INSERT INTO test.defaults FORMAT JSONEachRow {"x":4} {"y":5, "c":5} {"a":6, "b":6, "c":6}; + +SELECT * FROM test.defaults ORDER BY (x, y); +DROP TABLE IF EXISTS test.defaults; From 3c39f2fc691865459aba046cc4515f3532047384 Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 11 Jul 2018 21:34:12 +0300 Subject: [PATCH 09/29] fix some tests --- dbms/src/DataStreams/RemoteBlockOutputStream.cpp | 6 +++++- dbms/src/DataStreams/RemoteBlockOutputStream.h | 1 + .../00651_insert_json_with_defaults.reference | 14 +++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp index d9095ec91b9..7d0c17407c7 100644 --- a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp @@ -33,13 +33,17 @@ RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, const if (!header) throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR); } + else if (Protocol::Server::CapnProto == packet.type) + { + metadata = packet.block; + } else if (Protocol::Server::Exception == packet.type) { packet.exception->rethrow(); return; } else - throw NetException("Unexpected packet from server (expected Data or Exception, got " + throw NetException("Unexpected packet from server (expected Data, CapnProto or Exception, got " + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } diff --git a/dbms/src/DataStreams/RemoteBlockOutputStream.h b/dbms/src/DataStreams/RemoteBlockOutputStream.h index 41740c39837..e0269a1ea4d 100644 --- a/dbms/src/DataStreams/RemoteBlockOutputStream.h +++ b/dbms/src/DataStreams/RemoteBlockOutputStream.h @@ -35,6 +35,7 @@ private: String query; const Settings * settings; Block header; + Block metadata; bool finished = false; }; diff --git a/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.reference b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.reference index f513c6d6f40..9d95fa08690 100644 --- a/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.reference +++ b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.reference @@ -1,7 +1,7 @@ -0 0 6 6 6 -0 5 5 1.7917595 5 -1 1 2 1.0986123 42 -1 1 2 1.0986123 42 -2 2 4 1.609438 2 -3 3 3 3 3 -4 0 4 1.609438 42 +0 0 6 6 6 +0 5 5 1.7917595 5 +1 1 2 1.0986123 42 +1 1 2 1.0986123 42 +2 2 4 1.609438 2 +3 3 3 3 3 +4 0 4 1.609438 42 From 2876aadba77cd91818fdbcc6f0679533fed52a05 Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 12 Jul 2018 11:49:20 +0300 Subject: [PATCH 10/29] fix materialized --- dbms/src/Proto/protoHelpers.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index 5121529c207..c59516d42c2 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -136,8 +136,11 @@ namespace DB String expression = column.getDefault().getExpression().cStr(); ColumnDefaultKind expression_kind = static_cast(column.getDefault().getKind()); - ASTPtr ast = parseQuery(parser, expression, expression.size()); - table_meta.column_defaults.emplace(column_name, ColumnDefault{expression_kind, ast}); + if (expression_kind == ColumnDefaultKind::Default) + { + ASTPtr ast = parseQuery(parser, expression, expression.size()); + table_meta.column_defaults.emplace(column_name, ColumnDefault{expression_kind, ast}); + } } } } From a7fcae2759d2285b7feece194f66270231367c38 Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 12 Jul 2018 13:03:49 +0300 Subject: [PATCH 11/29] fix defaults with list expressions --- dbms/src/Proto/protoHelpers.cpp | 4 ++-- .../queries/0_stateless/00651_insert_json_with_defaults.sql | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index c59516d42c2..329cd40b5f7 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -116,7 +116,7 @@ namespace DB size_t data_size = proto_column.column->byteSize(); Proto::Context::Reader proto_context = deserializeProto(plain_data.data, data_size); - ParserExpressionElement parser; + ParserTernaryOperatorExpression parser; for (auto proto_database : proto_context.getDatabases()) { diff --git a/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql index d25ecdc1f5a..270778d9b49 100644 --- a/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql +++ b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql @@ -6,7 +6,10 @@ CREATE TABLE IF NOT EXISTS test.defaults y UInt32, a UInt32 DEFAULT x + y, b Float32 DEFAULT log(1 + x + y), - c UInt32 DEFAULT 42 + c UInt32 DEFAULT 42, + d DEFAULT x + y, + e MATERIALIZED x + y, + f ALIAS x + y ) ENGINE = Memory; INSERT INTO test.defaults FORMAT JSONEachRow {"x":1, "y":1}; From f89e476c0f540fb17a89e861053114f7dcd43b35 Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 12 Jul 2018 17:33:57 +0300 Subject: [PATCH 12/29] add option to disable sending metadata --- dbms/programs/server/TCPHandler.cpp | 8 ++++++-- dbms/src/Interpreters/Settings.h | 1 + dbms/src/Storages/StorageDistributed.cpp | 3 +++ .../0_stateless/00651_insert_json_with_defaults.sql | 3 +-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 128dc0090bd..9f8800ec1fe 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -313,8 +313,12 @@ void TCPHandler::processInsertQuery(const Settings & global_settings) state.io.out->writePrefix(); /// Send query metadata (column defaults) - Block meta_block = storeContextBlock(query_context); - sendMetadata(meta_block); + if (global_settings.insert_sample_with_metadata && + query_context.getSettingsRef().insert_sample_with_metadata) + { + Block meta_block = storeContextBlock(query_context); + sendMetadata(meta_block); + } /// Send block to the client - table structure. Block block = state.io.out->getHeader(); diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index c4d0d7654e6..8165cc776d9 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -123,6 +123,7 @@ struct Settings M(SettingUInt64, max_concurrent_queries_for_user, 0, "The maximum number of concurrent requests per user.") \ \ M(SettingBool, insert_deduplicate, true, "For INSERT queries in the replicated table, specifies that deduplication of insertings blocks should be preformed") \ + M(SettingBool, insert_sample_with_metadata, true, "For INSERT queries, specifies that need add metadata before sample block") \ \ M(SettingUInt64, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled.") \ M(SettingMilliseconds, insert_quorum_timeout, 600000, "") \ diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index a125806515c..49b20acf9eb 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -165,6 +165,9 @@ StorageDistributed::StorageDistributed( if (num_local_shards && remote_database == database_name && remote_table == table_name) throw Exception("Distributed table " + table_name + " looks at itself", ErrorCodes::INFINITE_LOOP); } + + /// HACK: disable metadata for StorageDistributed queries + const_cast(context).getSettingsRef().insert_sample_with_metadata = false; } diff --git a/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql index 270778d9b49..12b9e4538d1 100644 --- a/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql +++ b/dbms/tests/queries/0_stateless/00651_insert_json_with_defaults.sql @@ -4,10 +4,9 @@ CREATE TABLE IF NOT EXISTS test.defaults ( x UInt32, y UInt32, - a UInt32 DEFAULT x + y, + a DEFAULT x + y, b Float32 DEFAULT log(1 + x + y), c UInt32 DEFAULT 42, - d DEFAULT x + y, e MATERIALIZED x + y, f ALIAS x + y ) ENGINE = Memory; From c78a67d57316a39ec40116558d00ce5e0b93fb6b Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 13 Jul 2018 01:05:03 +0300 Subject: [PATCH 13/29] fix case with unexpected aliases --- dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp index ffde573a24f..7cc90f6a641 100644 --- a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -43,7 +43,11 @@ Block AddingDefaultsBlockInputStream::readImpl() Block evaluate_block{res}; for (const auto & column : column_defaults) - evaluate_block.erase(column.first); + { + /// column_defaults contain aliases that could be ommited in evaluate_block + if (evaluate_block.has(column.first)) + evaluate_block.erase(column.first); + } evaluateMissingDefaultsUnsafe(evaluate_block, header.getNamesAndTypesList(), column_defaults, context); From 4e0d9aa8d44e92f744925797c29e9df2a94f11b8 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 13 Jul 2018 15:46:29 +0300 Subject: [PATCH 14/29] revert unneeded changes --- dbms/programs/client/Client.cpp | 4 +--- dbms/src/DataStreams/AddingDefaultBlockOutputStream.cpp | 1 - dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp | 5 ++--- dbms/src/DataStreams/RemoteBlockOutputStream.cpp | 6 +----- dbms/src/DataStreams/RemoteBlockOutputStream.h | 1 - 5 files changed, 4 insertions(+), 13 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 5dddf053482..5ee25d38c33 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -936,18 +936,16 @@ private: void sendDataFrom(ReadBuffer & buf, Block & sample, const TableMetadata & table_meta) { String current_format = insert_format; - const ColumnDefaults & column_defaults = table_meta.column_defaults; /// Data format can be specified in the INSERT query. if (ASTInsertQuery * insert = typeid_cast(&*parsed_query)) - { if (!insert->format.empty()) current_format = insert->format; - } BlockInputStreamPtr block_input = context.getInputFormat( current_format, buf, sample, insert_format_max_block_size); + const ColumnDefaults & column_defaults = table_meta.column_defaults; if (!column_defaults.empty()) block_input = std::make_shared(block_input, column_defaults, context); BlockInputStreamPtr async_block_input = std::make_shared(block_input); diff --git a/dbms/src/DataStreams/AddingDefaultBlockOutputStream.cpp b/dbms/src/DataStreams/AddingDefaultBlockOutputStream.cpp index 7a1ab14e7fc..fe773b40776 100644 --- a/dbms/src/DataStreams/AddingDefaultBlockOutputStream.cpp +++ b/dbms/src/DataStreams/AddingDefaultBlockOutputStream.cpp @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB diff --git a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp index 945fc782327..855840a15ee 100644 --- a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp +++ b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp @@ -44,12 +44,11 @@ InputStreamFromASTInsertQuery::InputStreamFromASTInsertQuery( input_buffer_contacenated = std::make_unique(buffers); + res_stream = context.getInputFormat(format, *input_buffer_contacenated, streams.out->getHeader(), context.getSettings().max_insert_block_size); + TableMetadata table_meta(ast_insert_query->database, ast_insert_query->table); table_meta.loadFromContext(context); - res_stream = context.getInputFormat( - format, *input_buffer_contacenated, streams.out->getHeader(), context.getSettings().max_insert_block_size); - if (!table_meta.column_defaults.empty()) res_stream = std::make_shared(res_stream, table_meta.column_defaults, context); } diff --git a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp index 7d0c17407c7..d9095ec91b9 100644 --- a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp @@ -33,17 +33,13 @@ RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, const if (!header) throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR); } - else if (Protocol::Server::CapnProto == packet.type) - { - metadata = packet.block; - } else if (Protocol::Server::Exception == packet.type) { packet.exception->rethrow(); return; } else - throw NetException("Unexpected packet from server (expected Data, CapnProto or Exception, got " + throw NetException("Unexpected packet from server (expected Data or Exception, got " + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } diff --git a/dbms/src/DataStreams/RemoteBlockOutputStream.h b/dbms/src/DataStreams/RemoteBlockOutputStream.h index e0269a1ea4d..41740c39837 100644 --- a/dbms/src/DataStreams/RemoteBlockOutputStream.h +++ b/dbms/src/DataStreams/RemoteBlockOutputStream.h @@ -35,7 +35,6 @@ private: String query; const Settings * settings; Block header; - Block metadata; bool finished = false; }; From 2c4949dd5d5b735574dac9f32843a341737e8920 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 13 Jul 2018 16:47:13 +0300 Subject: [PATCH 15/29] fix build without CAPNP --- cmake/find_capnp.cmake | 4 ++++ dbms/programs/client/CMakeLists.txt | 2 +- dbms/programs/client/Client.cpp | 2 ++ dbms/programs/server/TCPHandler.cpp | 2 ++ dbms/src/Proto/CMakeLists.txt | 1 + dbms/src/Proto/protoHelpers.h | 3 +++ 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index 03ecadda6a1..426031db15e 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -1,5 +1,9 @@ option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC}) +unset (USE_CAPNP CACHE) +unset (USE_INTERNAL_CAPNP_LIBRARY CACHE) +unset (MISSING_INTERNAL_CAPNP_LIBRARY CACHE) + if (ENABLE_CAPNP) # cmake 3.5.1 bug: # capnproto uses this cmake feature: diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt index 0f84270c8c9..659e8018a49 100644 --- a/dbms/programs/client/CMakeLists.txt +++ b/dbms/programs/client/CMakeLists.txt @@ -1,5 +1,5 @@ add_library (clickhouse-client-lib Client.cpp) -target_link_libraries (clickhouse-client-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_proto +target_link_libraries (clickhouse-client-lib clickhouse_functions clickhouse_aggregate_functions ${PROTO_LIB} ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 5ee25d38c33..74eff658133 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1082,7 +1082,9 @@ private: return false; case Protocol::Server::CapnProto: +#if USE_CAPNP loadTableMetadata(packet.block, table_meta); +#endif return receiveSampleBlock(out, table_meta); default: diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 9f8800ec1fe..c5800aa11df 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -312,6 +312,7 @@ void TCPHandler::processInsertQuery(const Settings & global_settings) */ state.io.out->writePrefix(); +#if USE_CAPNP /// Send query metadata (column defaults) if (global_settings.insert_sample_with_metadata && query_context.getSettingsRef().insert_sample_with_metadata) @@ -319,6 +320,7 @@ void TCPHandler::processInsertQuery(const Settings & global_settings) Block meta_block = storeContextBlock(query_context); sendMetadata(meta_block); } +#endif /// Send block to the client - table structure. Block block = state.io.out->getHeader(); diff --git a/dbms/src/Proto/CMakeLists.txt b/dbms/src/Proto/CMakeLists.txt index b3871cd53e8..4ca555914f7 100644 --- a/dbms/src/Proto/CMakeLists.txt +++ b/dbms/src/Proto/CMakeLists.txt @@ -6,6 +6,7 @@ add_custom_command (OUTPUT ServerMessage.capnp.c++ ServerMessage.capnp.h COMMAND ${CMAKE_COMMAND} -E env PATH=${CAPNP_PATH} ${CAPNP_BIN} compile -I ${CAPNP_INCLUDE_DIR} -oc++ ServerMessage.capnp WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ServerMessage.capnp) +set (PROTO_LIB clickhouse_proto) add_library (clickhouse_proto ServerMessage.capnp.c++ protoHelpers.cpp) target_link_libraries (clickhouse_proto clickhouse_common_io ${CAPNP_LIBRARY}) target_include_directories (clickhouse_proto PUBLIC ${CAPNP_INCLUDE_DIR} ${DBMS_INCLUDE_DIR}) diff --git a/dbms/src/Proto/protoHelpers.h b/dbms/src/Proto/protoHelpers.h index c0a514a5bb1..88e3c299fa2 100644 --- a/dbms/src/Proto/protoHelpers.h +++ b/dbms/src/Proto/protoHelpers.h @@ -1,4 +1,5 @@ #pragma once +#if USE_CAPNP namespace DB { @@ -9,3 +10,5 @@ namespace DB Block storeContextBlock(Context & context); void loadTableMetadata(const Block & block, TableMetadata & table_meta); } + +#endif From bc5d5bcf4a2a79e920b8aa238e2ada77a8af488e Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 13 Jul 2018 18:10:42 +0300 Subject: [PATCH 16/29] fix build with CAPNP --- dbms/CMakeLists.txt | 1 + dbms/src/Proto/CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index eaf21b0b6ac..3c7a7ac7198 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -208,6 +208,7 @@ if (USE_CAPNP) if (NOT USE_INTERNAL_CAPNP_LIBRARY) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR}) endif () + set (PROTO_LIB clickhouse_proto) endif () if (USE_RDKAFKA) diff --git a/dbms/src/Proto/CMakeLists.txt b/dbms/src/Proto/CMakeLists.txt index 4ca555914f7..b3871cd53e8 100644 --- a/dbms/src/Proto/CMakeLists.txt +++ b/dbms/src/Proto/CMakeLists.txt @@ -6,7 +6,6 @@ add_custom_command (OUTPUT ServerMessage.capnp.c++ ServerMessage.capnp.h COMMAND ${CMAKE_COMMAND} -E env PATH=${CAPNP_PATH} ${CAPNP_BIN} compile -I ${CAPNP_INCLUDE_DIR} -oc++ ServerMessage.capnp WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ServerMessage.capnp) -set (PROTO_LIB clickhouse_proto) add_library (clickhouse_proto ServerMessage.capnp.c++ protoHelpers.cpp) target_link_libraries (clickhouse_proto clickhouse_common_io ${CAPNP_LIBRARY}) target_include_directories (clickhouse_proto PUBLIC ${CAPNP_INCLUDE_DIR} ${DBMS_INCLUDE_DIR}) From 1d7f6c32a06bf7dc042486ed92d50ed880a7bf88 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 13 Jul 2018 18:32:20 +0300 Subject: [PATCH 17/29] one more build fix --- libs/libcommon/include/common/config_common.h.in | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/libcommon/include/common/config_common.h.in b/libs/libcommon/include/common/config_common.h.in index 0cc0950efba..bc24a0c6fe2 100644 --- a/libs/libcommon/include/common/config_common.h.in +++ b/libs/libcommon/include/common/config_common.h.in @@ -4,6 +4,7 @@ #cmakedefine01 USE_TCMALLOC #cmakedefine01 USE_JEMALLOC +#cmakedefine01 USE_CAPNP #cmakedefine01 USE_READLINE #cmakedefine01 USE_LIBEDIT #cmakedefine01 HAVE_READLINE_HISTORY From 53f1f4794dd440fcc842c0bc845460db732709da Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 13 Jul 2018 21:06:47 +0300 Subject: [PATCH 18/29] fix inserts into unknown table, ex. 'table function url(...)' --- dbms/src/Storages/TableMetadata.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/src/Storages/TableMetadata.cpp b/dbms/src/Storages/TableMetadata.cpp index e07c6a31e97..a489e5ba92e 100644 --- a/dbms/src/Storages/TableMetadata.cpp +++ b/dbms/src/Storages/TableMetadata.cpp @@ -7,6 +7,9 @@ namespace DB { void TableMetadata::loadFromContext(const Context & context) { + if (!context.isTableExist(database, table)) + return; + StoragePtr storage = context.getTable(database, table); const ColumnsDescription & table_columns = storage->getColumns(); column_defaults = table_columns.defaults; From 925e4c7dbb43845c4fad955f30bb01d2d2bde930 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 16 Jul 2018 14:28:22 +0300 Subject: [PATCH 19/29] backward compatibility --- dbms/cmake/version.cmake | 4 ++-- dbms/programs/server/TCPHandler.cpp | 3 ++- dbms/src/Core/Defines.h | 1 + dbms/src/Proto/protoHelpers.cpp | 8 ++++++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index bba2600d441..9d5cf8d5bf4 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,6 +1,6 @@ # This strings autochanged from release_lib.sh: -set(VERSION_DESCRIBE v1.1.54394-testing) -set(VERSION_REVISION 54394) +set(VERSION_DESCRIBE v1.1.54400-testing) +set(VERSION_REVISION 54400) set(VERSION_GITHASH 875ea0f4eaa3592f1fe628b6a1150d91b04ad574) # end of autochange diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index c5800aa11df..e85c08a5574 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -314,7 +314,8 @@ void TCPHandler::processInsertQuery(const Settings & global_settings) #if USE_CAPNP /// Send query metadata (column defaults) - if (global_settings.insert_sample_with_metadata && + if (client_revision >= DBMS_MIN_REVISION_WITH_PROTO_METADATA && + global_settings.insert_sample_with_metadata && query_context.getSettingsRef().insert_sample_with_metadata) { Block meta_block = storeContextBlock(query_context); diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 2ed07dce9b6..35198c2cb4f 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -49,6 +49,7 @@ #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337 #define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372 +#define DBMS_MIN_REVISION_WITH_PROTO_METADATA 54400 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index 329cd40b5f7..0d451625ea6 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -160,7 +160,11 @@ namespace DB void loadTableMetadata(const Block & block, TableMetadata & table_meta) { - const ColumnWithTypeAndName & column = block.getByName(contextColumnName()); - loadTableMetadata(column, table_meta); + /// select metadata type by column name + if (block.has(contextColumnName())) + { + const ColumnWithTypeAndName & column = block.getByName(contextColumnName()); + loadTableMetadata(column, table_meta); + } } } From 2cddb4b840553d439d0a394637fcfa5fabf3fd79 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 16 Jul 2018 17:52:02 +0300 Subject: [PATCH 20/29] send metadata only for affected table --- dbms/programs/server/TCPHandler.cpp | 12 +- dbms/src/Interpreters/Context.cpp | 15 +++ dbms/src/Interpreters/Context.h | 3 + .../Interpreters/InterpreterInsertQuery.cpp | 7 ++ .../src/Interpreters/InterpreterInsertQuery.h | 2 + dbms/src/Interpreters/executeQuery.cpp | 8 ++ dbms/src/Proto/protoHelpers.cpp | 108 +++++++----------- dbms/src/Proto/protoHelpers.h | 2 +- dbms/src/Storages/TableMetadata.cpp | 5 +- dbms/src/Storages/TableMetadata.h | 3 +- 10 files changed, 91 insertions(+), 74 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index e85c08a5574..096eea8512c 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -25,6 +25,7 @@ #include #include +#include #include @@ -313,13 +314,16 @@ void TCPHandler::processInsertQuery(const Settings & global_settings) state.io.out->writePrefix(); #if USE_CAPNP - /// Send query metadata (column defaults) + /// Send table metadata (column defaults) if (client_revision >= DBMS_MIN_REVISION_WITH_PROTO_METADATA && - global_settings.insert_sample_with_metadata && query_context.getSettingsRef().insert_sample_with_metadata) { - Block meta_block = storeContextBlock(query_context); - sendMetadata(meta_block); + TableMetadata table_meta(query_context.getCurrentDatabase(), query_context.getCurrentTable()); + if (table_meta.loadFromContext(query_context) && table_meta.hasDefaults()) + { + Block meta_block = storeTableMetadata(table_meta); + sendMetadata(meta_block); + } } #endif diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 9fed370cfbc..4b48b6e233c 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1003,6 +1003,12 @@ String Context::getCurrentDatabase() const } +String Context::getCurrentTable() const +{ + return current_table; +} + + String Context::getCurrentQueryId() const { return client_info.current_query_id; @@ -1017,6 +1023,15 @@ void Context::setCurrentDatabase(const String & name) } +void Context::setCurrentTable(const String & database, const String & table) +{ + auto lock = getLock(); + assertTableExists(database, table); + current_database = database; + current_table = table; +} + + void Context::setCurrentQueryId(const String & query_id) { if (!client_info.current_query_id.empty()) diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 1c867d65e8f..4ab675dfaf2 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -104,6 +104,7 @@ private: std::shared_ptr quota; /// Current quota. By default - empty quota, that have no limits. String current_database; + String current_table; Settings settings; /// Setting for query execution. using ProgressCallback = std::function; ProgressCallback progress_callback; /// Callback for tracking progress of query execution. @@ -211,8 +212,10 @@ public: std::unique_ptr getDDLGuardIfTableDoesntExist(const String & database, const String & table, const String & message) const; String getCurrentDatabase() const; + String getCurrentTable() const; String getCurrentQueryId() const; void setCurrentDatabase(const String & name); + void setCurrentTable(const String & database, const String & table); void setCurrentQueryId(const String & query_id); String getDefaultFormat() const; /// If default_format is not specified, some global default format is returned. diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 86164ef2704..f11c12b67d9 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -153,4 +153,11 @@ void InterpreterInsertQuery::checkAccess(const ASTInsertQuery & query) throw Exception("Cannot insert into table in readonly mode", ErrorCodes::READONLY); } +void InterpreterInsertQuery::getDatabaseTable(String & database, String & table) const +{ + ASTInsertQuery & query = typeid_cast(*query_ptr); + database = query.database; + table = query.table; +} + } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.h b/dbms/src/Interpreters/InterpreterInsertQuery.h index 2180ebe0550..4ec7460fb75 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.h +++ b/dbms/src/Interpreters/InterpreterInsertQuery.h @@ -24,6 +24,8 @@ public: */ BlockIO execute() override; + void getDatabaseTable(String & database, String & table) const; + private: StoragePtr getTable(const ASTInsertQuery & query); Block getSampleBlock(const ASTInsertQuery & query, const StoragePtr & table); diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index de36a84fd26..ebfc71ef537 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -200,6 +200,14 @@ static std::tuple executeQueryImpl( auto interpreter = InterpreterFactory::get(ast, context, stage); res = interpreter->execute(); + if (InterpreterInsertQuery * insertInterpreter = typeid_cast(&*interpreter)) + { + String database; + String table_name; + insertInterpreter->getDatabaseTable(database, table_name); + if (!database.empty()) + context.setCurrentTable(database, table_name); + } /// Delayed initialization of query streams (required for KILL QUERY purposes) if (process_list_entry) diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index 0d451625ea6..371f17d2d81 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -33,6 +33,7 @@ namespace DB return data; } + template typename T::Reader deserializeProto(const char * data, size_t data_size) { @@ -43,78 +44,45 @@ namespace DB return reader.getRoot(); } - static ColumnWithTypeAndName storeContext(const String & column_name, Context & context) + + static MutableColumnPtr storeTableMeta(const TableMetadata & meta) { capnp::MallocMessageBuilder message; Proto::Context::Builder proto_context = message.initRoot(); - Databases dbs = context.getDatabases(); - auto proto_databases = proto_context.initDatabases(dbs.size()); + auto proto_databases = proto_context.initDatabases(1); + auto proto_db = proto_databases[0]; + proto_db.setName(meta.database); - size_t db_nomber = 0; - for (auto & pr_db : dbs) + auto proto_db_tables = proto_db.initTables(1); + auto proto_table = proto_db_tables[0]; + proto_table.setName(meta.table); + + auto proto_columns = proto_table.initColumns(meta.column_defaults.size()); + + size_t column_no = 0; + for (const auto & pr_column : meta.column_defaults) { - const String & database_name = pr_db.first; - if (database_name == "system") - continue; + const String & column_name = pr_column.first; + const ColumnDefault & def = pr_column.second; + std::stringstream ss; + ss << def.expression; - IDatabase & db = *pr_db.second; + auto current_column = proto_columns[column_no]; + current_column.setName(column_name); + current_column.getDefault().setKind(static_cast(def.kind)); + current_column.getDefault().setExpression(ss.str()); - auto proto_db = proto_databases[db_nomber]; - proto_db.setName(database_name); - - std::unordered_map tables; - DatabaseIteratorPtr it_tables = db.getIterator(context); - while (it_tables->isValid()) - { - tables[it_tables->name()] = it_tables->table(); - it_tables->next(); - } - - auto proto_tables = proto_db.initTables(tables.size()); - size_t table_no = 0; - for (const auto & pr_table : tables) - { - auto current_table = proto_tables[table_no]; - current_table.setName(pr_table.first); - - const ColumnsDescription & columns = pr_table.second->getColumns(); - auto proto_columns = current_table.initColumns(columns.defaults.size()); - - size_t column_no = 0; - for (const auto& pr_column : columns.defaults) - { - const String & column_name = pr_column.first; - const ColumnDefault & def = pr_column.second; - std::stringstream ss; - ss << def.expression; - - auto current_column = proto_columns[column_no]; - current_column.setName(column_name); - current_column.getDefault().setKind(static_cast(def.kind)); - current_column.getDefault().setExpression(ss.str()); - - ++column_no; - } - - ++table_no; - } - - ++db_nomber; + ++column_no; } - ColumnWithTypeAndName proto_column; - proto_column.name = column_name; - proto_column.type = std::make_shared(); - proto_column.column = std::move(serializeProto(message)); - return proto_column; + return serializeProto(message); } - static void loadTableMetadata(const ColumnWithTypeAndName & proto_column, TableMetadata & table_meta) + + static void loadTableMeta(const char * data, size_t data_size, TableMetadata & table_meta) { - StringRef plain_data = proto_column.column->getDataAt(0); - size_t data_size = proto_column.column->byteSize(); - Proto::Context::Reader proto_context = deserializeProto(plain_data.data, data_size); + Proto::Context::Reader proto_context = deserializeProto(data, data_size); ParserTernaryOperatorExpression parser; @@ -146,25 +114,33 @@ namespace DB } } - static constexpr const char * contextColumnName() + + static constexpr const char * tableMetaColumnName() { - return "context"; + return "tableMeta"; } - Block storeContextBlock(Context & context) + + Block storeTableMetadata(const TableMetadata & table_meta) { + ColumnWithTypeAndName proto_column; + proto_column.name = tableMetaColumnName(); + proto_column.type = std::make_shared(); + proto_column.column = std::move(storeTableMeta(table_meta)); + Block block; - block.insert(storeContext(contextColumnName(), context)); + block.insert(std::move(proto_column)); return block; } + void loadTableMetadata(const Block & block, TableMetadata & table_meta) { /// select metadata type by column name - if (block.has(contextColumnName())) + if (block.has(tableMetaColumnName())) { - const ColumnWithTypeAndName & column = block.getByName(contextColumnName()); - loadTableMetadata(column, table_meta); + const ColumnWithTypeAndName & column = block.getByName(tableMetaColumnName()); + loadTableMeta(column.column->getDataAt(0).data, column.column->byteSize(), table_meta); } } } diff --git a/dbms/src/Proto/protoHelpers.h b/dbms/src/Proto/protoHelpers.h index 88e3c299fa2..2909ebc3e9d 100644 --- a/dbms/src/Proto/protoHelpers.h +++ b/dbms/src/Proto/protoHelpers.h @@ -7,7 +7,7 @@ namespace DB class Block; class TableMetadata; - Block storeContextBlock(Context & context); + Block storeTableMetadata(const TableMetadata & table_meta); void loadTableMetadata(const Block & block, TableMetadata & table_meta); } diff --git a/dbms/src/Storages/TableMetadata.cpp b/dbms/src/Storages/TableMetadata.cpp index a489e5ba92e..5549574275a 100644 --- a/dbms/src/Storages/TableMetadata.cpp +++ b/dbms/src/Storages/TableMetadata.cpp @@ -5,13 +5,14 @@ namespace DB { - void TableMetadata::loadFromContext(const Context & context) + bool TableMetadata::loadFromContext(const Context & context) { if (!context.isTableExist(database, table)) - return; + return false; StoragePtr storage = context.getTable(database, table); const ColumnsDescription & table_columns = storage->getColumns(); column_defaults = table_columns.defaults; + return true; } } diff --git a/dbms/src/Storages/TableMetadata.h b/dbms/src/Storages/TableMetadata.h index a88808ed96a..2194f2c8465 100644 --- a/dbms/src/Storages/TableMetadata.h +++ b/dbms/src/Storages/TableMetadata.h @@ -18,6 +18,7 @@ namespace DB const String & table; ColumnDefaults column_defaults; - void loadFromContext(const Context & context); + bool loadFromContext(const Context & context); + bool hasDefaults() const { return !column_defaults.empty(); } }; } From 2d3aa8cdbaf0a93c75632a2074cdffacc4d69ca7 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 16 Jul 2018 19:41:15 +0300 Subject: [PATCH 21/29] Update TableMetadata.h --- dbms/src/Storages/TableMetadata.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/TableMetadata.h b/dbms/src/Storages/TableMetadata.h index 2194f2c8465..ddb5b79b68b 100644 --- a/dbms/src/Storages/TableMetadata.h +++ b/dbms/src/Storages/TableMetadata.h @@ -7,7 +7,7 @@ namespace DB class Context; class Block; - /// Addition information for query that could not be get from sample block + /// Additional information for query that could not be get from sample block struct TableMetadata { TableMetadata(const String & database_, const String & table_) From 0463fb6dd316f1a3bbe6c0b67ea213338dced3cd Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 16 Jul 2018 19:48:56 +0300 Subject: [PATCH 22/29] Update Settings.h --- dbms/src/Interpreters/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 5d946a7d6cb..b07e9914f34 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -123,7 +123,7 @@ struct Settings M(SettingUInt64, max_concurrent_queries_for_user, 0, "The maximum number of concurrent requests per user.") \ \ M(SettingBool, insert_deduplicate, true, "For INSERT queries in the replicated table, specifies that deduplication of insertings blocks should be preformed") \ - M(SettingBool, insert_sample_with_metadata, true, "For INSERT queries, specifies that need add metadata before sample block") \ + M(SettingBool, insert_sample_with_metadata, true, "For INSERT queries, specifies that the server need to send metadata about column defaults to the client. This will be used to calculate default expressions.") \ \ M(SettingUInt64, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled.") \ M(SettingMilliseconds, insert_quorum_timeout, 600000, "") \ From 07a782e637249a760f15d73bfecb3329cde3f902 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 16 Jul 2018 22:28:07 +0300 Subject: [PATCH 23/29] fix wrong defaults at columns tail and some build fixes --- cmake/find_capnp.cmake | 4 ---- dbms/programs/client/Client.cpp | 1 + .../DataStreams/AddingDefaultsBlockInputStream.cpp | 2 +- .../Formats/BlockInputStreamFromRowInputStream.cpp | 11 ++--------- libs/libcommon/include/common/config_common.h.in | 1 - 5 files changed, 4 insertions(+), 15 deletions(-) diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index 426031db15e..03ecadda6a1 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -1,9 +1,5 @@ option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC}) -unset (USE_CAPNP CACHE) -unset (USE_INTERNAL_CAPNP_LIBRARY CACHE) -unset (MISSING_INTERNAL_CAPNP_LIBRARY CACHE) - if (ENABLE_CAPNP) # cmake 3.5.1 bug: # capnproto uses this cmake feature: diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 74eff658133..51ba397a000 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp index 7cc90f6a641..63d18cd0285 100644 --- a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -74,7 +74,7 @@ Block AddingDefaultsBlockInputStream::readImpl() for (size_t row_idx = 0; row_idx < column_read.column->size(); ++row_idx) { - if (defaults_mask[row_idx]) + if (row_idx < defaults_mask.size() && defaults_mask[row_idx]) { if (column_def.column->isColumnConst()) column_mixed->insert((*column_def.column)[row_idx]); diff --git a/dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp b/dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp index aa4c2968539..f3ec72a4a6c 100644 --- a/dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp +++ b/dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp @@ -16,7 +16,6 @@ namespace ErrorCodes extern const int CANNOT_PARSE_NUMBER; extern const int CANNOT_PARSE_UUID; extern const int TOO_LARGE_STRING_SIZE; - extern const int INCORRECT_NUMBER_OF_COLUMNS; } @@ -62,14 +61,8 @@ Block BlockInputStreamFromRowInputStream::readImpl() break; for (size_t column_idx = 0; column_idx < info.read_columns.size(); ++column_idx) - { - if (!info.read_columns[column_idx]) { - size_t column_size = columns[column_idx]->size(); - if (column_size == 0) - throw Exception("Unexpected empty column", ErrorCodes::INCORRECT_NUMBER_OF_COLUMNS); - delayed_defaults.setBit(column_idx, column_size - 1); - } - } + if (!info.read_columns[column_idx]) + delayed_defaults.setBit(column_idx, rows); } catch (Exception & e) { diff --git a/libs/libcommon/include/common/config_common.h.in b/libs/libcommon/include/common/config_common.h.in index bc24a0c6fe2..0cc0950efba 100644 --- a/libs/libcommon/include/common/config_common.h.in +++ b/libs/libcommon/include/common/config_common.h.in @@ -4,7 +4,6 @@ #cmakedefine01 USE_TCMALLOC #cmakedefine01 USE_JEMALLOC -#cmakedefine01 USE_CAPNP #cmakedefine01 USE_READLINE #cmakedefine01 USE_LIBEDIT #cmakedefine01 HAVE_READLINE_HISTORY From c7d9314189d7449641abacb3fef6bae8fff88d3b Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 17 Jul 2018 15:18:20 +0300 Subject: [PATCH 24/29] allow build with system capnp --- cmake/find_capnp.cmake | 3 +++ dbms/src/Proto/CMakeLists.txt | 7 ++----- utils/build/build_debian_unbundled.sh | 2 +- utils/travis/normal.sh | 2 +- utils/travis/pbuilder.sh | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index 03ecadda6a1..b72746b436a 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -25,6 +25,7 @@ if (ENABLE_CAPNP) if (NOT USE_INTERNAL_CAPNP_LIBRARY) set (CAPNP_PATHS "/usr/local/opt/capnp/lib") + set (CAPNP_BIN_PATH "/usr/bin:/usr/local/bin") set (CAPNP_INCLUDE_PATHS "/usr/local/opt/capnp/include") find_library (CAPNP capnp PATHS ${CAPNP_PATHS}) find_library (CAPNPC capnpc PATHS ${CAPNP_PATHS}) @@ -40,6 +41,8 @@ if (ENABLE_CAPNP) set (CAPNP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src") set (CAPNP_LIBRARY capnpc) set (USE_CAPNP 1) + set (CAPNP_BIN_PATH ${ClickHouse_BINARY_DIR}/contrib/capnproto/c++/src/capnp) + set (CAPNP_BIN_TARGET capnp_tool) endif () endif () diff --git a/dbms/src/Proto/CMakeLists.txt b/dbms/src/Proto/CMakeLists.txt index b3871cd53e8..1f5d07089af 100644 --- a/dbms/src/Proto/CMakeLists.txt +++ b/dbms/src/Proto/CMakeLists.txt @@ -1,10 +1,7 @@ -set (CAPNP_PATH ${CMAKE_BINARY_DIR}/contrib/capnproto/c++/src/capnp) -set (CAPNP_BIN ${CAPNP_PATH}/capnp) - add_custom_command (OUTPUT ServerMessage.capnp.c++ ServerMessage.capnp.h COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/ServerMessage.capnp ${CMAKE_CURRENT_BINARY_DIR}/ServerMessage.capnp - COMMAND ${CMAKE_COMMAND} -E env PATH=${CAPNP_PATH} ${CAPNP_BIN} compile -I ${CAPNP_INCLUDE_DIR} -oc++ ServerMessage.capnp - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ServerMessage.capnp) + COMMAND ${CMAKE_COMMAND} -E env PATH=${CAPNP_BIN_PATH} capnp compile -I ${CAPNP_INCLUDE_DIR} -oc++ ServerMessage.capnp + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ServerMessage.capnp ${CAPNP_BIN_TARGET}) add_library (clickhouse_proto ServerMessage.capnp.c++ protoHelpers.cpp) target_link_libraries (clickhouse_proto clickhouse_common_io ${CAPNP_LIBRARY}) diff --git a/utils/build/build_debian_unbundled.sh b/utils/build/build_debian_unbundled.sh index dc47c8fc3a3..53b7a12a239 100755 --- a/utils/build/build_debian_unbundled.sh +++ b/utils/build/build_debian_unbundled.sh @@ -22,5 +22,5 @@ env TEST_RUN=1 \ `# Use all possible contrib libs from system` \ `# psmisc - killall` \ `# gdb - symbol test in pbuilder` \ - EXTRAPACKAGES="psmisc gdb clang-6.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev libunwind-dev googletest libcctz-dev $EXTRAPACKAGES" \ + EXTRAPACKAGES="psmisc gdb clang-6.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev capnproto libcapnp-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev libunwind-dev googletest libcctz-dev $EXTRAPACKAGES" \ pdebuild --configfile $ROOT_DIR/debian/.pbuilderrc $PDEBUILD_OPT diff --git a/utils/travis/normal.sh b/utils/travis/normal.sh index b361744a3ec..f2e935f9bfa 100755 --- a/utils/travis/normal.sh +++ b/utils/travis/normal.sh @@ -32,7 +32,7 @@ cmake $CUR_DIR/../.. -DCMAKE_CXX_COMPILER=`which $DEB_CXX $CXX` -DCMAKE_C_COMPIL `# Use all possible contrib libs from system` \ -DUNBUNDLED=1 \ `# Disable all features` \ - -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_TCMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 $CMAKE_FLAGS \ + -DENABLE_RDKAFKA=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_TCMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 $CMAKE_FLAGS \ && ninja clickhouse-bundle \ `# Skip tests:` \ `# 00281 requires internal compiler` \ diff --git a/utils/travis/pbuilder.sh b/utils/travis/pbuilder.sh index 796dcf3e8d9..a1487ba2783 100755 --- a/utils/travis/pbuilder.sh +++ b/utils/travis/pbuilder.sh @@ -24,10 +24,10 @@ env TEST_RUN=${TEST_RUN=1} \ DEB_CC=${DEB_CC=$CC} DEB_CXX=${DEB_CXX=$CXX} \ CCACHE_SIZE=${CCACHE_SIZE:=4G} \ `# Disable all features` \ - CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Debug -DUNBUNDLED=1 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 -DCMAKE_C_FLAGS_ADD='-O0 -g0' -DCMAKE_CXX_FLAGS_ADD='-O0 -g0' $CMAKE_FLAGS" \ + CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Debug -DUNBUNDLED=1 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_RDKAFKA=0 -DUSE_INTERNAL_CAPNP_LIBRARY=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 -DCMAKE_C_FLAGS_ADD='-O0 -g0' -DCMAKE_CXX_FLAGS_ADD='-O0 -g0' $CMAKE_FLAGS" \ `# Use all possible contrib libs from system` \ `# psmisc - killall` \ - EXTRAPACKAGES="psmisc clang-5.0 lld-5.0 liblld-5.0-dev libclang-5.0-dev liblld-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev $EXTRAPACKAGES" \ + EXTRAPACKAGES="psmisc clang-5.0 lld-5.0 liblld-5.0-dev libclang-5.0-dev liblld-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev libcapnp-dev capnproto librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev $EXTRAPACKAGES" \ `# Travis trusty cant unpack bionic: E: debootstrap failed, TODO: check again, can be fixed` \ DIST=${DIST=artful} \ $CUR_DIR/../../release $RELEASE_OPT From 2edda7dee56111b671f03d4744cb23a091ca22e7 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 17 Jul 2018 16:55:55 +0300 Subject: [PATCH 25/29] try fix Travis build (use internal capnp) --- utils/travis/pbuilder.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/travis/pbuilder.sh b/utils/travis/pbuilder.sh index a1487ba2783..7257e597f1a 100755 --- a/utils/travis/pbuilder.sh +++ b/utils/travis/pbuilder.sh @@ -24,10 +24,10 @@ env TEST_RUN=${TEST_RUN=1} \ DEB_CC=${DEB_CC=$CC} DEB_CXX=${DEB_CXX=$CXX} \ CCACHE_SIZE=${CCACHE_SIZE:=4G} \ `# Disable all features` \ - CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Debug -DUNBUNDLED=1 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_RDKAFKA=0 -DUSE_INTERNAL_CAPNP_LIBRARY=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 -DCMAKE_C_FLAGS_ADD='-O0 -g0' -DCMAKE_CXX_FLAGS_ADD='-O0 -g0' $CMAKE_FLAGS" \ + CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Debug -DUNBUNDLED=1 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_RDKAFKA=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 -DCMAKE_C_FLAGS_ADD='-O0 -g0' -DCMAKE_CXX_FLAGS_ADD='-O0 -g0' $CMAKE_FLAGS" \ `# Use all possible contrib libs from system` \ `# psmisc - killall` \ - EXTRAPACKAGES="psmisc clang-5.0 lld-5.0 liblld-5.0-dev libclang-5.0-dev liblld-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev libcapnp-dev capnproto librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev $EXTRAPACKAGES" \ + EXTRAPACKAGES="psmisc clang-5.0 lld-5.0 liblld-5.0-dev libclang-5.0-dev liblld-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev $EXTRAPACKAGES" \ `# Travis trusty cant unpack bionic: E: debootstrap failed, TODO: check again, can be fixed` \ DIST=${DIST=artful} \ $CUR_DIR/../../release $RELEASE_OPT From f12edacbcf79b899bbf781c42a7dc2855e2f6665 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 17 Jul 2018 19:22:31 +0300 Subject: [PATCH 26/29] Travis build with internal capnp (normal) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 705b6977114..401c8d96856 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ matrix: # update: true # sources: # - ubuntu-toolchain-r-test -# packages: [ g++-7, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl ] +# packages: [ g++-7, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl ] # # env: # - MATRIX_EVAL="export CC=gcc-7 && export CXX=g++-7" @@ -38,7 +38,7 @@ matrix: sources: - ubuntu-toolchain-r-test - llvm-toolchain-trusty-5.0 - packages: [ ninja-build, g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl] + packages: [ ninja-build, g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl] env: - MATRIX_EVAL="export CC=clang-5.0 && export CXX=clang++-5.0" From cbf1c220ae8bd17c40054111cdafd4e1f5dde0f7 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 17 Jul 2018 21:18:46 +0300 Subject: [PATCH 27/29] better proto deserialize --- dbms/src/Proto/protoHelpers.cpp | 38 ++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/dbms/src/Proto/protoHelpers.cpp b/dbms/src/Proto/protoHelpers.cpp index 371f17d2d81..c898d182df8 100644 --- a/dbms/src/Proto/protoHelpers.cpp +++ b/dbms/src/Proto/protoHelpers.cpp @@ -19,6 +19,13 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + static MutableColumnPtr serializeProto(capnp::MessageBuilder & message) { MutableColumnPtr data = DataTypeUInt8().createColumn(); @@ -34,19 +41,29 @@ namespace DB } + /// template - typename T::Reader deserializeProto(const char * data, size_t data_size) + class ProtoDeserializer { - const capnp::word * ptr = reinterpret_cast(data); - auto serialized = kj::arrayPtr(ptr, data_size / sizeof(capnp::word)); + public: + ProtoDeserializer(const char * data, size_t data_size) + : serialized(kj::arrayPtr(reinterpret_cast(data), data_size / sizeof(capnp::word))), + reader(serialized) + {} - capnp::FlatArrayMessageReader reader(serialized); - return reader.getRoot(); - } + typename T::Reader getReader() { return reader.getRoot(); } + + private: + kj::ArrayPtr serialized; + capnp::FlatArrayMessageReader reader; + }; static MutableColumnPtr storeTableMeta(const TableMetadata & meta) { + if (meta.database.empty() || meta.table.empty()) + throw Exception("storeTableMeta: table is not set", ErrorCodes::LOGICAL_ERROR); + capnp::MallocMessageBuilder message; Proto::Context::Builder proto_context = message.initRoot(); @@ -82,7 +99,11 @@ namespace DB static void loadTableMeta(const char * data, size_t data_size, TableMetadata & table_meta) { - Proto::Context::Reader proto_context = deserializeProto(data, data_size); + if (data == nullptr || data_size == 0) + throw Exception("loadTableMeta: empty metadata column", ErrorCodes::LOGICAL_ERROR); + + ProtoDeserializer deserializer(data, data_size); + Proto::Context::Reader proto_context = deserializer.getReader(); ParserTernaryOperatorExpression parser; @@ -140,7 +161,8 @@ namespace DB if (block.has(tableMetaColumnName())) { const ColumnWithTypeAndName & column = block.getByName(tableMetaColumnName()); - loadTableMeta(column.column->getDataAt(0).data, column.column->byteSize(), table_meta); + StringRef raw_data = column.column->getRawData(); + loadTableMeta(raw_data.data, raw_data.size, table_meta); } } } From 955293cea78736390501b46cdfa5412f8af5a87c Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 17 Jul 2018 21:42:05 +0300 Subject: [PATCH 28/29] some more build improvements --- cmake/find_capnp.cmake | 2 +- dbms/src/Proto/CMakeLists.txt | 2 +- dbms/src/Proto/protoHelpers.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index b72746b436a..abe46316676 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -42,7 +42,7 @@ if (ENABLE_CAPNP) set (CAPNP_LIBRARY capnpc) set (USE_CAPNP 1) set (CAPNP_BIN_PATH ${ClickHouse_BINARY_DIR}/contrib/capnproto/c++/src/capnp) - set (CAPNP_BIN_TARGET capnp_tool) + set (CAPNP_BIN_TARGETS capnp_tool capnpc_cpp capnpc_capnp) endif () endif () diff --git a/dbms/src/Proto/CMakeLists.txt b/dbms/src/Proto/CMakeLists.txt index 1f5d07089af..1e150d9f86b 100644 --- a/dbms/src/Proto/CMakeLists.txt +++ b/dbms/src/Proto/CMakeLists.txt @@ -1,7 +1,7 @@ add_custom_command (OUTPUT ServerMessage.capnp.c++ ServerMessage.capnp.h COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/ServerMessage.capnp ${CMAKE_CURRENT_BINARY_DIR}/ServerMessage.capnp COMMAND ${CMAKE_COMMAND} -E env PATH=${CAPNP_BIN_PATH} capnp compile -I ${CAPNP_INCLUDE_DIR} -oc++ ServerMessage.capnp - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ServerMessage.capnp ${CAPNP_BIN_TARGET}) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ServerMessage.capnp ${CAPNP_BIN_TARGETS}) add_library (clickhouse_proto ServerMessage.capnp.c++ protoHelpers.cpp) target_link_libraries (clickhouse_proto clickhouse_common_io ${CAPNP_LIBRARY}) diff --git a/dbms/src/Proto/protoHelpers.h b/dbms/src/Proto/protoHelpers.h index 2909ebc3e9d..7cef5a28013 100644 --- a/dbms/src/Proto/protoHelpers.h +++ b/dbms/src/Proto/protoHelpers.h @@ -5,7 +5,7 @@ namespace DB { class Context; class Block; - class TableMetadata; + struct TableMetadata; Block storeTableMetadata(const TableMetadata & table_meta); void loadTableMetadata(const Block & block, TableMetadata & table_meta); From 3d133a6a5a861161a4dde3b7b76f6ae39f9a9781 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 17 Jul 2018 23:02:37 +0300 Subject: [PATCH 29/29] save block structure when add defaults --- .../AddingDefaultsBlockInputStream.cpp | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp index 63d18cd0285..ace3841e67b 100644 --- a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -51,8 +51,7 @@ Block AddingDefaultsBlockInputStream::readImpl() evaluateMissingDefaultsUnsafe(evaluate_block, header.getNamesAndTypesList(), column_defaults, context); - ColumnsWithTypeAndName mixed_columns; - mixed_columns.reserve(std::min(column_defaults.size(), delayed_defaults.size())); + std::unordered_map mixed_columns; for (const ColumnWithTypeAndName & column_def : evaluate_block) { @@ -85,16 +84,21 @@ Block AddingDefaultsBlockInputStream::readImpl() column_mixed->insertFrom(*column_read.column, row_idx); } - ColumnWithTypeAndName mix = column_read.cloneEmpty(); - mix.column = std::move(column_mixed); - mixed_columns.emplace_back(std::move(mix)); + mixed_columns.emplace(std::make_pair(block_column_position, std::move(column_mixed))); } } - for (auto & column : mixed_columns) + if (!mixed_columns.empty()) { - res.erase(column.name); - res.insert(std::move(column)); + /// replace columns saving block structure + MutableColumns mutation = res.mutateColumns(); + for (size_t position = 0; position < mutation.size(); ++position) + { + auto it = mixed_columns.find(position); + if (it != mixed_columns.end()) + mutation[position] = std::move(it->second); + } + res.setColumns(std::move(mutation)); } return res;