From d9a0432c6cd9b9457813fd5895d39239462837e6 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Tue, 9 Oct 2018 21:32:44 +0300 Subject: [PATCH 01/22] don't quote single-field partition keys #3109 [#CLICKHOUSE-4054] --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../Storages/MergeTree/MergeTreePartition.cpp | 32 +++++++++++-------- .../Storages/MergeTree/MergeTreePartition.h | 2 +- .../Storages/System/StorageSystemParts.cpp | 2 +- .../System/StorageSystemPartsColumns.cpp | 2 +- .../00502_custom_partitioning_local.reference | 24 +++++++------- ...artitioning_replicated_zookeeper.reference | 24 +++++++------- 7 files changed, 46 insertions(+), 42 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 0a55bfa97e5..eddf3fbb339 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2184,7 +2184,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context { WriteBufferFromOwnString buf; writeCString("Parsed partition value: ", buf); - partition.serializeTextQuoted(*this, buf, format_settings); + partition.serializeText(*this, buf, format_settings); writeCString(" doesn't match partition value for an existing part with the same partition ID: ", buf); writeString(existing_part_in_partition->name, buf); throw Exception(buf.str(), ErrorCodes::INVALID_PARTITION_VALUE); diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp index 6064625d49e..d4aad4230b0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp @@ -77,32 +77,36 @@ String MergeTreePartition::getID(const MergeTreeData & storage) const return result; } -void MergeTreePartition::serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const +void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const { size_t key_size = storage.partition_key_sample.columns(); if (key_size == 0) { writeCString("tuple()", out); - return; } - - if (key_size > 1) - writeChar('(', out); - - for (size_t i = 0; i < key_size; ++i) + else if (key_size == 1) { - if (i > 0) - writeCString(", ", out); - - const DataTypePtr & type = storage.partition_key_sample.getByPosition(i).type; + const DataTypePtr & type = storage.partition_key_sample.getByPosition(0).type; auto column = type->createColumn(); - column->insert(value[i]); - type->serializeTextQuoted(*column, 0, out, format_settings); + column->insert(value[0]); + type->serializeText(*column, 0, out, format_settings); } + else + { + writeChar('(', out); + for (size_t i = 0; i < key_size; ++i) + { + if (i > 0) + writeCString(", ", out); - if (key_size > 1) + const DataTypePtr & type = storage.partition_key_sample.getByPosition(i).type; + auto column = type->createColumn(); + column->insert(value[i]); + type->serializeTextQuoted(*column, 0, out, format_settings); + } writeChar(')', out); + } } void MergeTreePartition::load(const MergeTreeData & storage, const String & part_path) diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.h b/dbms/src/Storages/MergeTree/MergeTreePartition.h index cb5493cc8bd..425828c2cc1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartition.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartition.h @@ -26,7 +26,7 @@ public: String getID(const MergeTreeData & storage) const; - void serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const; + void serializeText(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const; void load(const MergeTreeData & storage, const String & part_path); void store(const MergeTreeData & storage, const String & part_path, MergeTreeDataPartChecksums & checksums) const; diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp index 71c15004e66..597a4e2cc30 100644 --- a/dbms/src/Storages/System/StorageSystemParts.cpp +++ b/dbms/src/Storages/System/StorageSystemParts.cpp @@ -61,7 +61,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor size_t i = 0; { WriteBufferFromOwnString out; - part->partition.serializeTextQuoted(*info.data, out, format_settings); + part->partition.serializeText(*info.data, out, format_settings); columns[i++]->insert(out.str()); } columns[i++]->insert(part->name); diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 435b3dd287f..de882ee4581 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -106,7 +106,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con size_t j = 0; { WriteBufferFromOwnString out; - part->partition.serializeTextQuoted(*info.data, out, format_settings); + part->partition.serializeText(*info.data, out, format_settings); columns[j++]->insert(out.str()); } columns[j++]->insert(part->name); diff --git a/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.reference b/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.reference index f3811ddcf1d..b8370d9e27a 100644 --- a/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.reference +++ b/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.reference @@ -10,12 +10,12 @@ Sum after DETACH PARTITION: 0 *** Partitioned by week *** Parts before OPTIMIZE: -\'1999-12-27\' 19991227_1_1_0 -\'2000-01-03\' 20000103_2_2_0 -\'2000-01-03\' 20000103_3_3_0 +1999-12-27 19991227_1_1_0 +2000-01-03 20000103_2_2_0 +2000-01-03 20000103_3_3_0 Parts after OPTIMIZE: -\'1999-12-27\' 19991227_1_1_0 -\'2000-01-03\' 20000103_2_3_1 +1999-12-27 19991227_1_1_0 +2000-01-03 20000103_2_3_1 Sum before DROP PARTITION: 15 Sum after DROP PARTITION: @@ -37,14 +37,14 @@ Sum after DETACH PARTITION: 9 *** Partitioned by String *** Parts before OPTIMIZE: -\'bbb\' 7d878f3d88441d2b3dc371e2a3050f6d_2_2_0 -\'bbb\' 7d878f3d88441d2b3dc371e2a3050f6d_3_3_0 -\'aaa\' 9b50856126a8a6064f11f027d455bf58_1_1_0 -\'aaa\' 9b50856126a8a6064f11f027d455bf58_4_4_0 +bbb 7d878f3d88441d2b3dc371e2a3050f6d_2_2_0 +bbb 7d878f3d88441d2b3dc371e2a3050f6d_3_3_0 +aaa 9b50856126a8a6064f11f027d455bf58_1_1_0 +aaa 9b50856126a8a6064f11f027d455bf58_4_4_0 Parts after OPTIMIZE: -\'bbb\' 7d878f3d88441d2b3dc371e2a3050f6d_2_2_0 -\'bbb\' 7d878f3d88441d2b3dc371e2a3050f6d_3_3_0 -\'aaa\' 9b50856126a8a6064f11f027d455bf58_1_4_1 +bbb 7d878f3d88441d2b3dc371e2a3050f6d_2_2_0 +bbb 7d878f3d88441d2b3dc371e2a3050f6d_3_3_0 +aaa 9b50856126a8a6064f11f027d455bf58_1_4_1 Sum before DROP PARTITION: 15 Sum after DROP PARTITION: diff --git a/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.reference b/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.reference index 313e84db2d7..8b6d0ed1384 100644 --- a/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.reference +++ b/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.reference @@ -10,12 +10,12 @@ Sum after DETACH PARTITION: 0 *** Partitioned by week *** Parts before OPTIMIZE: -\'1999-12-27\' 19991227_0_0_0 -\'2000-01-03\' 20000103_0_0_0 -\'2000-01-03\' 20000103_1_1_0 +1999-12-27 19991227_0_0_0 +2000-01-03 20000103_0_0_0 +2000-01-03 20000103_1_1_0 Parts after OPTIMIZE: -\'1999-12-27\' 19991227_0_0_0 -\'2000-01-03\' 20000103_0_1_1 +1999-12-27 19991227_0_0_0 +2000-01-03 20000103_0_1_1 Sum before DROP PARTITION: 15 Sum after DROP PARTITION: @@ -37,14 +37,14 @@ Sum after DETACH PARTITION: 9 *** Partitioned by String *** Parts before OPTIMIZE: -\'bbb\' 7d878f3d88441d2b3dc371e2a3050f6d_0_0_0 -\'bbb\' 7d878f3d88441d2b3dc371e2a3050f6d_1_1_0 -\'aaa\' 9b50856126a8a6064f11f027d455bf58_0_0_0 -\'aaa\' 9b50856126a8a6064f11f027d455bf58_1_1_0 +bbb 7d878f3d88441d2b3dc371e2a3050f6d_0_0_0 +bbb 7d878f3d88441d2b3dc371e2a3050f6d_1_1_0 +aaa 9b50856126a8a6064f11f027d455bf58_0_0_0 +aaa 9b50856126a8a6064f11f027d455bf58_1_1_0 Parts after OPTIMIZE: -\'bbb\' 7d878f3d88441d2b3dc371e2a3050f6d_0_0_0 -\'bbb\' 7d878f3d88441d2b3dc371e2a3050f6d_1_1_0 -\'aaa\' 9b50856126a8a6064f11f027d455bf58_0_1_1 +bbb 7d878f3d88441d2b3dc371e2a3050f6d_0_0_0 +bbb 7d878f3d88441d2b3dc371e2a3050f6d_1_1_0 +aaa 9b50856126a8a6064f11f027d455bf58_0_1_1 Sum before DROP PARTITION: 15 Sum after DROP PARTITION: From a029034d8c0a60ad760edeb00a43217e05cc3101 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 19 Nov 2018 09:14:36 +0300 Subject: [PATCH 02/22] CLICKHOUSE-4127: Fix ALTER of destination table for the BUFFER engine. --- dbms/src/Storages/StorageBuffer.cpp | 89 +++++++++++-------- ...r_destination_for_storage_buffer.reference | 19 ++++ ...3_alter_destination_for_storage_buffer.sql | 32 +++++++ 3 files changed, 103 insertions(+), 37 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00753_alter_destination_for_storage_buffer.reference create mode 100644 dbms/tests/queries/0_stateless/00753_alter_destination_for_storage_buffer.sql diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 4ec32883e6b..3f14209d141 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -1,7 +1,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -145,7 +147,34 @@ BlockInputStreams StorageBuffer::read( if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); - streams_from_dst = destination->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); + const Block structure_of_destination_table = + allow_materialized ? destination->getSampleBlock() : destination->getSampleBlockNonMaterialized(); + + bool can_read_from_destination = true; + for (const String& column_name : column_names) + { + if (!structure_of_destination_table.has(column_name)) + { + LOG_WARNING(log, "Destination table " << destination_database << "." << destination_table + << " doesn't have column " << column_name << ". Data from destination table is skipped."); + can_read_from_destination = false; + break; + } + auto col = getColumn(column_name); + auto dst_col = structure_of_destination_table.getByName(column_name); + if (!dst_col.type->equals(*col.type)) + { + LOG_WARNING(log, "Destination table " << destination_database << "." << destination_table + << " have different type of column " << column_name << " (" + << col.type->getName() << " != " << dst_col.type->getName() + << "). Data from destination table is skipped."); + can_read_from_destination = false; + break; + } + } + + if (can_read_from_destination) + streams_from_dst = destination->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); } BlockInputStreams streams_from_buffers; @@ -233,6 +262,9 @@ public: if (!block) return; + // Check table structure. + storage.check(block, true); + size_t rows = block.rows(); if (!rows) return; @@ -241,23 +273,8 @@ public: if (!storage.no_destination) { destination = storage.context.tryGetTable(storage.destination_database, storage.destination_table); - - if (destination) - { - if (destination.get() == &storage) - throw Exception("Destination table is myself. Write will cause infinite loop.", ErrorCodes::INFINITE_LOOP); - - /// Check table structure. - try - { - destination->check(block, true); - } - catch (Exception & e) - { - e.addMessage("(when looking at destination table " + storage.destination_database + "." + storage.destination_table + ")"); - throw; - } - } + if (destination.get() == &storage) + throw Exception("Destination table is myself. Write will cause infinite loop.", ErrorCodes::INFINITE_LOOP); } size_t bytes = block.bytes(); @@ -561,48 +578,46 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl * This will support some of the cases (but not all) when the table structure does not match. */ Block structure_of_destination_table = allow_materialized ? table->getSampleBlock() : table->getSampleBlockNonMaterialized(); - Names columns_intersection; - columns_intersection.reserve(block.columns()); + Block block_to_write; for (size_t i : ext::range(0, structure_of_destination_table.columns())) { auto dst_col = structure_of_destination_table.getByPosition(i); if (block.has(dst_col.name)) { - if (!block.getByName(dst_col.name).type->equals(*dst_col.type)) + auto column = block.getByName(dst_col.name); + if (!column.type->equals(*dst_col.type)) { - LOG_ERROR(log, "Destination table " << destination_database << "." << destination_table - << " have different type of column " << dst_col.name << " (" - << block.getByName(dst_col.name).type->getName() << " != " << dst_col.type->getName() - << "). Block of data is discarded."); - return; + LOG_WARNING(log, "Destination table " << destination_database << "." << destination_table + << " have different type of column " << column.name << " (" + << column.type->getName() << " != " << dst_col.type->getName() + << "). Block of data is converted."); + column.column = castColumn(column, dst_col.type, context); + column.type = dst_col.type; } - columns_intersection.push_back(dst_col.name); + block_to_write.insert(column); } } - if (columns_intersection.empty()) + if (block_to_write.columns() == 0) { - LOG_ERROR(log, "Destination table " << destination_database << "." << destination_table << " have no common columns with block in buffer. Block of data is discarded."); + LOG_ERROR(log, "Destination table " << destination_database << "." << destination_table + << " have no common columns with block in buffer. Block of data is discarded."); return; } - if (columns_intersection.size() != block.columns()) + if (block_to_write.columns() != block.columns()) LOG_WARNING(log, "Not all columns from block in buffer exist in destination table " << destination_database << "." << destination_table << ". Some columns are discarded."); auto list_of_columns = std::make_shared(); insert->columns = list_of_columns; - list_of_columns->children.reserve(columns_intersection.size()); - for (const String & column : columns_intersection) - list_of_columns->children.push_back(std::make_shared(column)); + list_of_columns->children.reserve(block_to_write.columns()); + for (const auto& column : block_to_write) + list_of_columns->children.push_back(std::make_shared(column.name)); InterpreterInsertQuery interpreter{insert, context, allow_materialized}; - Block block_to_write; - for (const auto & name : columns_intersection) - block_to_write.insert(block.getByName(name)); - auto block_io = interpreter.execute(); block_io.out->writePrefix(); block_io.out->write(block_to_write); diff --git a/dbms/tests/queries/0_stateless/00753_alter_destination_for_storage_buffer.reference b/dbms/tests/queries/0_stateless/00753_alter_destination_for_storage_buffer.reference new file mode 100644 index 00000000000..00eccfa771d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00753_alter_destination_for_storage_buffer.reference @@ -0,0 +1,19 @@ +init +1 100 +2 200 +- +1 100 +2 200 +3 300 +alt +100 DEFZ +200 DEFZ +- +3 300 +4 400 +opt +100 DEFZ +200 DEFZ +300 DEFZ +400 DEFZ +- diff --git a/dbms/tests/queries/0_stateless/00753_alter_destination_for_storage_buffer.sql b/dbms/tests/queries/0_stateless/00753_alter_destination_for_storage_buffer.sql new file mode 100644 index 00000000000..5c348d6bc61 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00753_alter_destination_for_storage_buffer.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS test.dst; +DROP TABLE IF EXISTS test.buffer; +SET send_logs_level = 'error'; + +CREATE TABLE test.dst (x UInt64, y UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE test.buffer (x UInt64, y UInt64) ENGINE = Buffer(test, dst, 1, 99999, 99999, 1, 1, 99999, 99999); + +INSERT INTO test.buffer VALUES (1, 100); +INSERT INTO test.buffer VALUES (2, 200); +INSERT INTO test.buffer VALUES (3, 300); +SELECT 'init'; +SELECT * FROM test.dst ORDER BY x; +SELECT '-'; +SELECT * FROM test.buffer ORDER BY x; + +ALTER TABLE test.dst DROP COLUMN x, MODIFY COLUMN y String, ADD COLUMN z String DEFAULT 'DEFZ'; + +INSERT INTO test.buffer VALUES (4, 400); +SELECT 'alt'; +SELECT * FROM test.dst ORDER BY y; +SELECT '-'; +SELECT * FROM test.buffer ORDER BY x; + +OPTIMIZE TABLE test.buffer; +SELECT 'opt'; +SELECT * FROM test.dst ORDER BY y; +SELECT '-'; +SELECT * FROM test.buffer ORDER BY x; + +SET send_logs_level = 'warning'; +DROP TABLE IF EXISTS test.dst; +DROP TABLE IF EXISTS test.buffer; From 33eb64269fa6c0e4a4787c720e94aaf4b4996579 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 19 Nov 2018 18:14:44 +0300 Subject: [PATCH 03/22] Update StorageBuffer.cpp --- dbms/src/Storages/StorageBuffer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 3f14209d141..50f914edd4d 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -151,11 +151,11 @@ BlockInputStreams StorageBuffer::read( allow_materialized ? destination->getSampleBlock() : destination->getSampleBlockNonMaterialized(); bool can_read_from_destination = true; - for (const String& column_name : column_names) + for (const String & column_name : column_names) { if (!structure_of_destination_table.has(column_name)) { - LOG_WARNING(log, "Destination table " << destination_database << "." << destination_table + LOG_WARNING(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) << " doesn't have column " << column_name << ". Data from destination table is skipped."); can_read_from_destination = false; break; @@ -164,8 +164,8 @@ BlockInputStreams StorageBuffer::read( auto dst_col = structure_of_destination_table.getByName(column_name); if (!dst_col.type->equals(*col.type)) { - LOG_WARNING(log, "Destination table " << destination_database << "." << destination_table - << " have different type of column " << column_name << " (" + LOG_WARNING(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) + << " has different type of column " << backQuoteIfNeed(column_name) << " (" << col.type->getName() << " != " << dst_col.type->getName() << "). Data from destination table is skipped."); can_read_from_destination = false; From 9a3701cc5099a44e74e31a79462b43d64a388a73 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 19 Nov 2018 18:20:05 +0300 Subject: [PATCH 04/22] Update StorageBuffer.cpp --- dbms/src/Storages/StorageBuffer.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 50f914edd4d..4e2d58fc727 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -565,7 +565,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (!table) { - LOG_ERROR(log, "Destination table " << destination_database << "." << destination_table << " doesn't exist. Block of data is discarded."); + LOG_ERROR(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) << " doesn't exist. Block of data is discarded."); return; } @@ -587,8 +587,8 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto column = block.getByName(dst_col.name); if (!column.type->equals(*dst_col.type)) { - LOG_WARNING(log, "Destination table " << destination_database << "." << destination_table - << " have different type of column " << column.name << " (" + LOG_WARNING(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) + << " have different type of column " << backQuoteIfNeed(column.name) << " (" << column.type->getName() << " != " << dst_col.type->getName() << "). Block of data is converted."); column.column = castColumn(column, dst_col.type, context); @@ -601,14 +601,14 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (block_to_write.columns() == 0) { - LOG_ERROR(log, "Destination table " << destination_database << "." << destination_table - << " have no common columns with block in buffer. Block of data is discarded."); + LOG_ERROR(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) + << " have no common columns with block in buffer. Block of data is discarded."); return; } if (block_to_write.columns() != block.columns()) LOG_WARNING(log, "Not all columns from block in buffer exist in destination table " - << destination_database << "." << destination_table << ". Some columns are discarded."); + << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) << ". Some columns are discarded."); auto list_of_columns = std::make_shared(); insert->columns = list_of_columns; From 72fb78a0f851a2cb713f43a77a12907b0515b283 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 19 Nov 2018 18:20:34 +0300 Subject: [PATCH 05/22] Update StorageBuffer.cpp --- dbms/src/Storages/StorageBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 4e2d58fc727..460b5498840 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -613,7 +613,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto list_of_columns = std::make_shared(); insert->columns = list_of_columns; list_of_columns->children.reserve(block_to_write.columns()); - for (const auto& column : block_to_write) + for (const auto & column : block_to_write) list_of_columns->children.push_back(std::make_shared(column.name)); InterpreterInsertQuery interpreter{insert, context, allow_materialized}; From 3a3dda5ec508af5127c6686a8812acc778ccd5ae Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Mon, 26 Nov 2018 15:41:17 +0300 Subject: [PATCH 06/22] use standard Tuple serialization [#CLICKHOUSE-4054] --- .../Storages/MergeTree/MergeTreePartition.cpp | 18 +++++++++++------- .../00502_custom_partitioning_local.reference | 16 ++++++++-------- ...partitioning_replicated_zookeeper.reference | 16 ++++++++-------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp index d4aad4230b0..b13c9fb6740 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -94,18 +96,20 @@ void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffe } else { - writeChar('(', out); + DataTypes types; + Columns columns; for (size_t i = 0; i < key_size; ++i) { - if (i > 0) - writeCString(", ", out); - - const DataTypePtr & type = storage.partition_key_sample.getByPosition(i).type; + const auto & type = storage.partition_key_sample.getByPosition(i).type; + types.push_back(type); auto column = type->createColumn(); column->insert(value[i]); - type->serializeTextQuoted(*column, 0, out, format_settings); + columns.push_back(std::move(column)); } - writeChar(')', out); + + DataTypeTuple tuple_type(types); + auto tuple_column = ColumnTuple::create(columns); + tuple_type.serializeText(*tuple_column, 0, out, format_settings); } } diff --git a/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.reference b/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.reference index b8370d9e27a..71f0102c20b 100644 --- a/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.reference +++ b/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.reference @@ -22,15 +22,15 @@ Sum after DROP PARTITION: 12 *** Partitioned by a (Date, UInt8) tuple *** Parts before OPTIMIZE: -(\'2000-01-01\', 1) 20000101-1_1_1_0 -(\'2000-01-01\', 1) 20000101-1_5_5_0 -(\'2000-01-01\', 2) 20000101-2_2_2_0 -(\'2000-01-02\', 1) 20000102-1_3_3_0 -(\'2000-01-02\', 1) 20000102-1_4_4_0 +(\'2000-01-01\',1) 20000101-1_1_1_0 +(\'2000-01-01\',1) 20000101-1_5_5_0 +(\'2000-01-01\',2) 20000101-2_2_2_0 +(\'2000-01-02\',1) 20000102-1_3_3_0 +(\'2000-01-02\',1) 20000102-1_4_4_0 Parts after OPTIMIZE: -(\'2000-01-01\', 1) 20000101-1_1_5_1 -(\'2000-01-01\', 2) 20000101-2_2_2_0 -(\'2000-01-02\', 1) 20000102-1_3_4_1 +(\'2000-01-01\',1) 20000101-1_1_5_1 +(\'2000-01-01\',2) 20000101-2_2_2_0 +(\'2000-01-02\',1) 20000102-1_3_4_1 Sum before DETACH PARTITION: 15 Sum after DETACH PARTITION: diff --git a/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.reference b/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.reference index 8b6d0ed1384..689fd252a21 100644 --- a/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.reference +++ b/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.reference @@ -22,15 +22,15 @@ Sum after DROP PARTITION: 12 *** Partitioned by a (Date, UInt8) tuple *** Parts before OPTIMIZE: -(\'2000-01-01\', 1) 20000101-1_0_0_0 -(\'2000-01-01\', 1) 20000101-1_1_1_0 -(\'2000-01-01\', 2) 20000101-2_0_0_0 -(\'2000-01-02\', 1) 20000102-1_0_0_0 -(\'2000-01-02\', 1) 20000102-1_1_1_0 +(\'2000-01-01\',1) 20000101-1_0_0_0 +(\'2000-01-01\',1) 20000101-1_1_1_0 +(\'2000-01-01\',2) 20000101-2_0_0_0 +(\'2000-01-02\',1) 20000102-1_0_0_0 +(\'2000-01-02\',1) 20000102-1_1_1_0 Parts after OPTIMIZE: -(\'2000-01-01\', 1) 20000101-1_0_1_1 -(\'2000-01-01\', 2) 20000101-2_0_0_0 -(\'2000-01-02\', 1) 20000102-1_0_1_1 +(\'2000-01-01\',1) 20000101-1_0_1_1 +(\'2000-01-01\',2) 20000101-2_0_0_0 +(\'2000-01-02\',1) 20000102-1_0_1_1 Sum before DETACH PARTITION: 15 Sum after DETACH PARTITION: From e793a27cc341f30678c135b8c3cc71306564f1fe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Nov 2018 16:57:54 +0300 Subject: [PATCH 07/22] Restored old test util #3665 --- dbms/src/Client/CMakeLists.txt | 2 + dbms/src/Client/tests/CMakeLists.txt | 2 + dbms/src/Client/tests/test_connect.cpp | 59 ++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 dbms/src/Client/tests/CMakeLists.txt create mode 100644 dbms/src/Client/tests/test_connect.cpp diff --git a/dbms/src/Client/CMakeLists.txt b/dbms/src/Client/CMakeLists.txt index 907d9787bf8..cc8b3f8eda8 100644 --- a/dbms/src/Client/CMakeLists.txt +++ b/dbms/src/Client/CMakeLists.txt @@ -4,3 +4,5 @@ #add_library(clickhouse_client ${LINK_MODE} ${clickhouse_client_headers} ${clickhouse_client_sources}) #target_link_libraries (clickhouse_client clickhouse_common_io ${Poco_Net_LIBRARY}) #target_include_directories (clickhouse_client PRIVATE ${DBMS_INCLUDE_DIR}) + +add_subdirectory(tests) diff --git a/dbms/src/Client/tests/CMakeLists.txt b/dbms/src/Client/tests/CMakeLists.txt new file mode 100644 index 00000000000..f4471136a8a --- /dev/null +++ b/dbms/src/Client/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(test-connect test_connect.cpp) +target_link_libraries (test-connect dbms) diff --git a/dbms/src/Client/tests/test_connect.cpp b/dbms/src/Client/tests/test_connect.cpp new file mode 100644 index 00000000000..75eb606cc97 --- /dev/null +++ b/dbms/src/Client/tests/test_connect.cpp @@ -0,0 +1,59 @@ +#include +#include +#include + +#include +#include +#include +#include + + +/** In a loop it connects to the server and immediately breaks the connection. +  * Using the SO_LINGER option, we ensure that the connection is terminated by sending a RST packet (not FIN). +  * This behavior causes a bug in the TCPServer implementation in the Poco library. +  */ +int main(int argc, char ** argv) +try +{ + for (size_t i = 0, num_iters = argc >= 2 ? DB::parse(argv[1]) : 1; i < num_iters; ++i) + { + std::cerr << "."; + + Poco::Net::SocketAddress address("localhost", 9000); + + int fd = socket(PF_INET, SOCK_STREAM, IPPROTO_IP); + + if (fd < 0) + DB::throwFromErrno("Cannot create socket", 0); + + linger linger_value; + linger_value.l_onoff = 1; + linger_value.l_linger = 0; + + if (0 != setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger_value, sizeof(linger_value))) + DB::throwFromErrno("Cannot set linger", 0); + + try + { + int res = connect(fd, address.addr(), address.length()); + + if (res != 0 && errno != EINPROGRESS && errno != EWOULDBLOCK) + { + close(fd); + DB::throwFromErrno("Cannot connect", 0); + } + + close(fd); + } + catch (const Poco::Exception & e) + { + std::cerr << e.displayText() << "\n"; + } + } + + std::cerr << "\n"; +} +catch (const Poco::Exception & e) +{ + std::cerr << e.displayText() << "\n"; +} From 73aa7f2027fe9a41101efc315224052d9d33e521 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Mon, 26 Nov 2018 17:43:40 +0300 Subject: [PATCH 08/22] =?UTF-8?q?Rename=20method=20partition()=20=E2=86=92?= =?UTF-8?q?=20alterPartition()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also fix build with internal shared libraries. --- dbms/programs/CMakeLists.txt | 2 +- dbms/src/Interpreters/InterpreterAlterQuery.cpp | 2 +- dbms/src/Storages/IStorage.h | 2 +- dbms/src/Storages/StorageMaterializedView.cpp | 4 ++-- dbms/src/Storages/StorageMaterializedView.h | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 4 ++-- dbms/src/Storages/StorageMergeTree.h | 2 +- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- dbms/src/Storages/StorageReplicatedMergeTree.h | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index 441b39d9966..9d7c6f2cda1 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -48,7 +48,7 @@ else () link_directories (${LLVM_LIBRARY_DIRS}) endif () add_executable (clickhouse main.cpp) - target_link_libraries (clickhouse PRIVATE clickhouse_common_io) + target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils) target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index c3bab56edcf..ab24d7e1164 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -57,7 +57,7 @@ BlockIO InterpreterAlterQuery::execute() if (!partition_commands.empty()) { partition_commands.validate(*table); - table->partition(query_ptr, partition_commands, context); + table->alterPartition(query_ptr, partition_commands, context); } if (!alter_commands.empty()) diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 65d40d82c9a..81249446746 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -256,7 +256,7 @@ public: /** ALTER tables with regard to its partitions. * Should handle locks for each command on its own. */ - virtual void partition(const ASTPtr & /* query */, const PartitionCommands & /* commands */, const Context & /* context */) + virtual void alterPartition(const ASTPtr & /* query */, const PartitionCommands & /* commands */, const Context & /* context */) { throw Exception("Partition operations are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index e1af2576873..f69efad04ae 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -240,10 +240,10 @@ bool StorageMaterializedView::optimize(const ASTPtr & query, const ASTPtr & part return getTargetTable()->optimize(query, partition, final, deduplicate, context); } -void StorageMaterializedView::partition(const ASTPtr & query, const PartitionCommands &commands, const Context &context) +void StorageMaterializedView::alterPartition(const ASTPtr & query, const PartitionCommands &commands, const Context &context) { checkStatementCanBeForwarded(); - getTargetTable()->partition(query, commands, context); + getTargetTable()->alterPartition(query, commands, context); } void StorageMaterializedView::mutate(const MutationCommands & commands, const Context & context) diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index d1f463151e6..176a95c2d03 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -35,7 +35,7 @@ public: bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; - void partition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override; + void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override; void mutate(const MutationCommands & commands, const Context & context) override; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 7ed871e23c7..50d7af076d2 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -781,7 +781,7 @@ bool StorageMergeTree::optimize( return true; } -void StorageMergeTree::partition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) +void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) { for (const PartitionCommand & command : commands) { @@ -824,7 +824,7 @@ void StorageMergeTree::partition(const ASTPtr & query, const PartitionCommands & break; default: - IStorage::partition(query, commands, context); // should throw an exception. + IStorage::alterPartition(query, commands, context); // should throw an exception. } } } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 04856835fcf..d5e4616d920 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -60,7 +60,7 @@ public: */ bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; - void partition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override; + void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override; void mutate(const MutationCommands & commands, const Context & context) override; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index abb2e46e584..7fe4a72c0eb 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3282,7 +3282,7 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, LOG_DEBUG(log, "ALTER finished"); } -void StorageReplicatedMergeTree::partition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) +void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) { for (const PartitionCommand & command : commands) { @@ -3329,7 +3329,7 @@ void StorageReplicatedMergeTree::partition(const ASTPtr & query, const Partition break; default: - IStorage::partition(query, commands, context); // should throw an exception. + IStorage::alterPartition(query, commands, context); // should throw an exception. } } } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 9bc6d4f08f1..236ca204daf 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -116,7 +116,7 @@ public: void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; - void partition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override; + void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override; void mutate(const MutationCommands & commands, const Context & context) override; From 5a6f9a5ff948f9d033247682ee1d358b627e4db3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 26 Nov 2018 18:18:45 +0300 Subject: [PATCH 09/22] Replace function mysql with function file --- dbms/tests/queries/0_stateless/00758_array_reverse.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00758_array_reverse.sql b/dbms/tests/queries/0_stateless/00758_array_reverse.sql index 1c6afe0995b..8a2417e68b0 100644 --- a/dbms/tests/queries/0_stateless/00758_array_reverse.sql +++ b/dbms/tests/queries/0_stateless/00758_array_reverse.sql @@ -12,4 +12,4 @@ SELECT reverse([]); SELECT reverse([[[[]]]]); SET send_logs_level = 'none'; -SELECT '[RE7', ( SELECT '\0' ) AS riwwq, ( SELECT reverse([( SELECT bitTestAll(NULL) ) , ( SELECT '\0' ) AS ddfweeuy]) ) AS xuvv, '', ( SELECT * FROM mysql() ) AS wqgdswyc, ( SELECT * FROM mysql() ); -- { serverError 42 } +SELECT '[RE7', ( SELECT '\0' ) AS riwwq, ( SELECT reverse([( SELECT bitTestAll(NULL) ) , ( SELECT '\0' ) AS ddfweeuy]) ) AS xuvv, '', ( SELECT * FROM file() ) AS wqgdswyc, ( SELECT * FROM file() ); -- { serverError 42 } From e00ce0bb57d1cd2ba70e86ebd0dbae77d09dec80 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 26 Nov 2018 19:20:40 +0300 Subject: [PATCH 10/22] Every function in its own file, part 7 (#3666) * Every function in its own file, part 7 [#CLICKHOUSE-2] * Every function in its own file, part 7 [#CLICKHOUSE-2] * Every function in its own file, part 7 [#CLICKHOUSE-2] * Every function in its own file, part 7 [#CLICKHOUSE-2] * Every function in its own file, part 7 [#CLICKHOUSE-2] * Fixed build #3666 --- cmake/dbms_generate_function.cmake | 10 - dbms/src/Functions/CMakeLists.txt | 35 - dbms/src/Functions/FunctionBinaryArithmetic.h | 921 +++++++ dbms/src/Functions/FunctionBitTestMany.h | 204 ++ dbms/src/Functions/FunctionUnaryArithmetic.h | 220 ++ dbms/src/Functions/FunctionsArithmetic.h | 2176 ----------------- dbms/src/Functions/FunctionsRound.cpp | 4 - dbms/src/Functions/FunctionsRound.h | 139 +- dbms/src/Functions/abs.cpp | 54 + dbms/src/Functions/bitAnd.cpp | 38 + dbms/src/Functions/bitNot.cpp | 47 + dbms/src/Functions/bitOr.cpp | 38 + dbms/src/Functions/bitRotateLeft.cpp | 41 + dbms/src/Functions/bitRotateRight.cpp | 40 + dbms/src/Functions/bitShiftLeft.cpp | 38 + dbms/src/Functions/bitShiftRight.cpp | 38 + dbms/src/Functions/bitTest.cpp | 31 + dbms/src/Functions/bitTestAll.cpp | 21 + dbms/src/Functions/bitTestAny.cpp | 21 + dbms/src/Functions/bitXor.cpp | 38 + dbms/src/Functions/castTypeToEither.h | 17 + dbms/src/Functions/divide.cpp | 39 + dbms/src/Functions/gcd.cpp | 36 + dbms/src/Functions/greatest.cpp | 61 + dbms/src/Functions/intDiv.cpp | 104 + dbms/src/Functions/intDiv.h | 66 + dbms/src/Functions/intDivOrZero.cpp | 31 + dbms/src/Functions/intExp10.cpp | 48 + dbms/src/Functions/intExp2.cpp | 55 + dbms/src/Functions/lcm.cpp | 36 + dbms/src/Functions/least.cpp | 61 + dbms/src/Functions/minus.cpp | 45 + dbms/src/Functions/modulo.cpp | 102 + dbms/src/Functions/multiply.cpp | 45 + dbms/src/Functions/negate.cpp | 45 + dbms/src/Functions/plus.cpp | 46 + .../Functions/registerFunctionsArithmetic.cpp | 69 + dbms/src/Functions/roundAge.cpp | 38 + dbms/src/Functions/roundDuration.cpp | 47 + dbms/src/Functions/roundToExp2.cpp | 73 + 40 files changed, 2858 insertions(+), 2360 deletions(-) delete mode 100644 cmake/dbms_generate_function.cmake create mode 100644 dbms/src/Functions/FunctionBinaryArithmetic.h create mode 100644 dbms/src/Functions/FunctionBitTestMany.h create mode 100644 dbms/src/Functions/FunctionUnaryArithmetic.h delete mode 100644 dbms/src/Functions/FunctionsArithmetic.h create mode 100644 dbms/src/Functions/abs.cpp create mode 100644 dbms/src/Functions/bitAnd.cpp create mode 100644 dbms/src/Functions/bitNot.cpp create mode 100644 dbms/src/Functions/bitOr.cpp create mode 100644 dbms/src/Functions/bitRotateLeft.cpp create mode 100644 dbms/src/Functions/bitRotateRight.cpp create mode 100644 dbms/src/Functions/bitShiftLeft.cpp create mode 100644 dbms/src/Functions/bitShiftRight.cpp create mode 100644 dbms/src/Functions/bitTest.cpp create mode 100644 dbms/src/Functions/bitTestAll.cpp create mode 100644 dbms/src/Functions/bitTestAny.cpp create mode 100644 dbms/src/Functions/bitXor.cpp create mode 100644 dbms/src/Functions/castTypeToEither.h create mode 100644 dbms/src/Functions/divide.cpp create mode 100644 dbms/src/Functions/gcd.cpp create mode 100644 dbms/src/Functions/greatest.cpp create mode 100644 dbms/src/Functions/intDiv.cpp create mode 100644 dbms/src/Functions/intDiv.h create mode 100644 dbms/src/Functions/intDivOrZero.cpp create mode 100644 dbms/src/Functions/intExp10.cpp create mode 100644 dbms/src/Functions/intExp2.cpp create mode 100644 dbms/src/Functions/lcm.cpp create mode 100644 dbms/src/Functions/least.cpp create mode 100644 dbms/src/Functions/minus.cpp create mode 100644 dbms/src/Functions/modulo.cpp create mode 100644 dbms/src/Functions/multiply.cpp create mode 100644 dbms/src/Functions/negate.cpp create mode 100644 dbms/src/Functions/plus.cpp create mode 100644 dbms/src/Functions/registerFunctionsArithmetic.cpp create mode 100644 dbms/src/Functions/roundAge.cpp create mode 100644 dbms/src/Functions/roundDuration.cpp create mode 100644 dbms/src/Functions/roundToExp2.cpp diff --git a/cmake/dbms_generate_function.cmake b/cmake/dbms_generate_function.cmake deleted file mode 100644 index ae701f798fe..00000000000 --- a/cmake/dbms_generate_function.cmake +++ /dev/null @@ -1,10 +0,0 @@ -function(generate_function_register FUNCTION_AREA) - foreach(FUNCTION IN LISTS ARGN) - configure_file (registerFunction.h.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.h) - configure_file (registerFunction.cpp.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.cpp) - set(REGISTER_HEADERS "${REGISTER_HEADERS}#include \"register${FUNCTION}.h\"\n") - set(REGISTER_FUNCTIONS "${REGISTER_FUNCTIONS} register${FUNCTION}(factory);\n") - endforeach() - - configure_file (registerFunctions_area.cpp.in ${FUNCTIONS_GENERATED_DIR}registerFunctions${FUNCTION_AREA}.cpp) -endfunction() diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt index a6109c70934..1a1d6f06f64 100644 --- a/dbms/src/Functions/CMakeLists.txt +++ b/dbms/src/Functions/CMakeLists.txt @@ -1,43 +1,8 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) -include(${ClickHouse_SOURCE_DIR}/cmake/dbms_generate_function.cmake) - -set (FUNCTIONS_GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated/) - -generate_function_register(Arithmetic - FunctionPlus - FunctionMinus - FunctionMultiply - FunctionDivideFloating - FunctionDivideIntegral - FunctionDivideIntegralOrZero - FunctionModulo - FunctionNegate - FunctionAbs - FunctionBitAnd - FunctionBitOr - FunctionBitXor - FunctionBitNot - FunctionBitShiftLeft - FunctionBitShiftRight - FunctionBitRotateLeft - FunctionBitRotateRight - FunctionLeast - FunctionGreatest - FunctionBitTest - FunctionBitTestAny - FunctionBitTestAll - FunctionGCD - FunctionLCM - FunctionIntExp2 - FunctionIntExp10 -) - add_headers_and_sources(clickhouse_functions .) add_headers_and_sources(clickhouse_functions ./GatherUtils) add_headers_and_sources(clickhouse_functions ./Conditional) -#add_headers_and_sources(clickhouse_functions ${ClickHouse_BINARY_DIR}/dbms/src/Functions) -add_headers_and_sources(clickhouse_functions ${FUNCTIONS_GENERATED_DIR}) list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp) diff --git a/dbms/src/Functions/FunctionBinaryArithmetic.h b/dbms/src/Functions/FunctionBinaryArithmetic.h new file mode 100644 index 00000000000..a668c43b122 --- /dev/null +++ b/dbms/src/Functions/FunctionBinaryArithmetic.h @@ -0,0 +1,921 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if USE_EMBEDDED_COMPILER +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include // Y_IGNORE +#pragma GCC diagnostic pop +#endif + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int LOGICAL_ERROR; + extern const int DECIMAL_OVERFLOW; + extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES; + extern const int ILLEGAL_DIVISION; +} + + +/** Arithmetic operations: +, -, *, /, %, + * intDiv (integer division) + * Bitwise operations: |, &, ^, ~. + * Etc. + */ + +template +struct BinaryOperationImplBase +{ + using ResultType = ResultType_; + + static void NO_INLINE vector_vector(const PaddedPODArray & a, const PaddedPODArray & b, PaddedPODArray & c) + { + size_t size = a.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a[i], b[i]); + } + + static void NO_INLINE vector_constant(const PaddedPODArray & a, B b, PaddedPODArray & c) + { + size_t size = a.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a[i], b); + } + + static void NO_INLINE constant_vector(A a, const PaddedPODArray & b, PaddedPODArray & c) + { + size_t size = b.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a, b[i]); + } + + static ResultType constant_constant(A a, B b) + { + return Op::template apply(a, b); + } +}; + +template +struct BinaryOperationImpl : BinaryOperationImplBase +{ +}; + + +template struct PlusImpl; +template struct MinusImpl; +template struct MultiplyImpl; +template struct DivideFloatingImpl; +template struct DivideIntegralImpl; +template struct DivideIntegralOrZeroImpl; +template struct LeastBaseImpl; +template struct GreatestBaseImpl; +template struct ModuloImpl; + + +template struct NativeType { using Type = T; }; +template <> struct NativeType { using Type = Int32; }; +template <> struct NativeType { using Type = Int64; }; +template <> struct NativeType { using Type = Int128; }; + +/// Binary operations for Decimals need scale args +/// +|- scale one of args (which scale factor is not 1). ScaleR = oneof(Scale1, Scale2); +/// * no agrs scale. ScaleR = Scale1 + Scale2; +/// / first arg scale. ScaleR = Scale1 (scale_a = DecimalType::getScale()). +template typename Operation, typename ResultType_, bool _check_overflow = true> +struct DecimalBinaryOperation +{ + static constexpr bool is_plus_minus = std::is_same_v, PlusImpl> || + std::is_same_v, MinusImpl>; + static constexpr bool is_multiply = std::is_same_v, MultiplyImpl>; + static constexpr bool is_float_division = std::is_same_v, DivideFloatingImpl>; + static constexpr bool is_int_division = std::is_same_v, DivideIntegralImpl> || + std::is_same_v, DivideIntegralOrZeroImpl>; + static constexpr bool is_division = is_float_division || is_int_division; + static constexpr bool is_compare = std::is_same_v, LeastBaseImpl> || + std::is_same_v, GreatestBaseImpl>; + static constexpr bool is_plus_minus_compare = is_plus_minus || is_compare; + static constexpr bool can_overflow = is_plus_minus || is_multiply; + + using ResultType = ResultType_; + using NativeResultType = typename NativeType::Type; + using Op = std::conditional_t, /// substitute divide by intDiv (throw on division by zero) + Operation>; + using ColVecA = std::conditional_t, ColumnDecimal, ColumnVector>; + using ColVecB = std::conditional_t, ColumnDecimal, ColumnVector>; + using ArrayA = typename ColVecA::Container; + using ArrayB = typename ColVecB::Container; + using ArrayC = typename ColumnDecimal::Container; + using SelfNoOverflow = DecimalBinaryOperation; + + static void vector_vector(const ArrayA & a, const ArrayB & b, ArrayC & c, ResultType scale_a, ResultType scale_b, bool check_overflow) + { + if (check_overflow) + vector_vector(a, b, c, scale_a, scale_b); + else + SelfNoOverflow::vector_vector(a, b, c, scale_a, scale_b); + } + + static void vector_constant(const ArrayA & a, B b, ArrayC & c, ResultType scale_a, ResultType scale_b, bool check_overflow) + { + if (check_overflow) + vector_constant(a, b, c, scale_a, scale_b); + else + SelfNoOverflow::vector_constant(a, b, c, scale_a, scale_b); + } + + static void constant_vector(A a, const ArrayB & b, ArrayC & c, ResultType scale_a, ResultType scale_b, bool check_overflow) + { + if (check_overflow) + constant_vector(a, b, c, scale_a, scale_b); + else + SelfNoOverflow::constant_vector(a, b, c, scale_a, scale_b); + } + + static ResultType constant_constant(A a, B b, ResultType scale_a, ResultType scale_b, bool check_overflow) + { + if (check_overflow) + return constant_constant(a, b, scale_a, scale_b); + else + return SelfNoOverflow::constant_constant(a, b, scale_a, scale_b); + } + + static void NO_INLINE vector_vector(const ArrayA & a, const ArrayB & b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = a.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b[i], scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b[i], scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a[i], b[i], scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a[i], b[i]); + } + + static void NO_INLINE vector_constant(const ArrayA & a, B b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = a.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b, scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b, scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a[i], b, scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a[i], b); + } + + static void NO_INLINE constant_vector(A a, const ArrayB & b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = b.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a, b[i], scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a, b[i], scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a, b[i], scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a, b[i]); + } + + static ResultType constant_constant(A a, B b, ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + return applyScaled(a, b, scale_a); + else if (scale_b != 1) + return applyScaled(a, b, scale_b); + } + else if constexpr (is_division && IsDecimalNumber) + return applyScaledDiv(a, b, scale_a); + return apply(a, b); + } + +private: + /// there's implicit type convertion here + static NativeResultType apply(NativeResultType a, NativeResultType b) + { + if constexpr (can_overflow && _check_overflow) + { + NativeResultType res; + if (Op::template apply(a, b, res)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + return res; + } + else + return Op::template apply(a, b); + } + + template + static NativeResultType applyScaled(NativeResultType a, NativeResultType b, NativeResultType scale) + { + if constexpr (is_plus_minus_compare) + { + NativeResultType res; + + if constexpr (_check_overflow) + { + bool overflow = false; + if constexpr (scale_left) + overflow |= common::mulOverflow(a, scale, a); + else + overflow |= common::mulOverflow(b, scale, b); + + if constexpr (can_overflow) + overflow |= Op::template apply(a, b, res); + else + res = Op::template apply(a, b); + + if (overflow) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + } + else + { + if constexpr (scale_left) + a *= scale; + else + b *= scale; + res = Op::template apply(a, b); + } + + return res; + } + } + + static NativeResultType applyScaledDiv(NativeResultType a, NativeResultType b, NativeResultType scale) + { + if constexpr (is_division) + { + if constexpr (_check_overflow) + { + bool overflow = false; + if constexpr (!IsDecimalNumber) + overflow |= common::mulOverflow(scale, scale, scale); + overflow |= common::mulOverflow(a, scale, a); + if (overflow) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + } + else + { + if constexpr (!IsDecimalNumber) + scale *= scale; + a *= scale; + } + + return Op::template apply(a, b); + } + } +}; + + +/// Used to indicate undefined operation +struct InvalidType; + +template struct Case : std::bool_constant { using type = T; }; + +/// Switch, ...> -- select the first Ti for which Ci is true; InvalidType if none. +template using Switch = typename std::disjunction>::type; + +template constexpr bool IsIntegral = false; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; +template <> constexpr bool IsIntegral = true; + +template constexpr bool IsFloatingPoint = false; +template <> constexpr bool IsFloatingPoint = true; +template <> constexpr bool IsFloatingPoint = true; + +template constexpr bool IsDateOrDateTime = false; +template <> constexpr bool IsDateOrDateTime = true; +template <> constexpr bool IsDateOrDateTime = true; + +template constexpr bool UseLeftDecimal = false; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; + +template using DataTypeFromFieldType = std::conditional_t, InvalidType, DataTypeNumber>; + +template