From 6060a1ab575ac12f3f2be4f280c9c1f4ed24b1f7 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Fri, 4 Sep 2020 16:48:51 +0100 Subject: [PATCH 001/114] ALTER TABLE ... DROP|DETACH PART for ReplicatedMergeTree --- src/Parsers/ASTAlterQuery.cpp | 3 +- src/Parsers/ParserAlterQuery.cpp | 23 +++- src/Storages/PartitionCommands.cpp | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 100 ++++++++++++++++-- src/Storages/StorageReplicatedMergeTree.h | 7 +- .../01451_replicated_detach_part.reference | 9 ++ .../01451_replicated_detach_part.sql | 28 +++++ .../01452_replicated_drop_part.reference | 6 ++ .../01452_replicated_drop_part.sql | 25 +++++ 9 files changed, 185 insertions(+), 17 deletions(-) create mode 100644 tests/queries/0_stateless/01451_replicated_detach_part.reference create mode 100644 tests/queries/0_stateless/01451_replicated_detach_part.sql create mode 100644 tests/queries/0_stateless/01452_replicated_drop_part.reference create mode 100644 tests/queries/0_stateless/01452_replicated_drop_part.sql diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 8b23302a05c..72e422412f5 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -166,7 +166,8 @@ void ASTAlterCommand::formatImpl( } else if (type == ASTAlterCommand::DROP_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << (detach ? "DETACH" : "DROP") << " PARTITION " + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str + << (detach ? "DETACH" : "DROP") << (part ? " PART " : " PARTITION ") << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index 3f22aff9cf5..84fb1ac7d05 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -51,13 +51,15 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_modify("MODIFY"); ParserKeyword s_attach_partition("ATTACH PARTITION"); + ParserKeyword s_attach_part("ATTACH PART"); ParserKeyword s_detach_partition("DETACH PARTITION"); + ParserKeyword s_detach_part("DETACH PART"); ParserKeyword s_drop_partition("DROP PARTITION"); + ParserKeyword s_drop_part("DROP PART"); ParserKeyword s_move_partition("MOVE PARTITION"); + ParserKeyword s_move_part("MOVE PART"); ParserKeyword s_drop_detached_partition("DROP DETACHED PARTITION"); ParserKeyword s_drop_detached_part("DROP DETACHED PART"); - ParserKeyword s_attach_part("ATTACH PART"); - ParserKeyword s_move_part("MOVE PART"); ParserKeyword s_fetch_partition("FETCH PARTITION"); ParserKeyword s_replace_partition("REPLACE PARTITION"); ParserKeyword s_freeze("FREEZE"); @@ -149,6 +151,14 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->type = ASTAlterCommand::DROP_PARTITION; } + else if (s_drop_part.ignore(pos, expected)) + { + if (!parser_string_literal.parse(pos, command->partition, expected)) + return false; + + command->type = ASTAlterCommand::DROP_PARTITION; + command->part = true; + } else if (s_drop_detached_partition.ignore(pos, expected)) { if (!parser_partition.parse(pos, command->partition, expected)) @@ -342,6 +352,15 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->type = ASTAlterCommand::DROP_PARTITION; command->detach = true; } + else if (s_detach_part.ignore(pos, expected)) + { + if (!parser_string_literal.parse(pos, command->partition, expected)) + return false; + + command->type = ASTAlterCommand::DROP_PARTITION; + command->part = true; + command->detach = true; + } else if (s_attach_partition.ignore(pos, expected)) { if (!parser_partition.parse(pos, command->partition, expected)) diff --git a/src/Storages/PartitionCommands.cpp b/src/Storages/PartitionCommands.cpp index b21d22e5e1a..76c2af17256 100644 --- a/src/Storages/PartitionCommands.cpp +++ b/src/Storages/PartitionCommands.cpp @@ -21,6 +21,7 @@ std::optional PartitionCommand::parse(const ASTAlterCommand * res.type = DROP_PARTITION; res.partition = command_ast->partition; res.detach = command_ast->detach; + res.part = command_ast->part; return res; } else if (command_ast->type == ASTAlterCommand::DROP_DETACHED_PARTITION) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index cac2219b062..312040cbf16 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3894,8 +3894,13 @@ Pipe StorageReplicatedMergeTree::alterPartition( switch (command.type) { case PartitionCommand::DROP_PARTITION: - checkPartitionCanBeDropped(command.partition); - dropPartition(query, command.partition, command.detach, query_context); + if (command.part) + { + /// TODO(nv) what to check here? it is possible to drop a big partition by dropping small parts... + } + else + checkPartitionCanBeDropped(command.partition); + dropPartition(query, command.partition, command.detach, command.part, query_context); break; case PartitionCommand::DROP_DETACHED_PARTITION: @@ -4017,18 +4022,30 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition(const St } -void StorageReplicatedMergeTree::dropPartition(const ASTPtr &, const ASTPtr & partition, bool detach, const Context & query_context) +void StorageReplicatedMergeTree::dropPartition( + const ASTPtr &, const ASTPtr & partition, bool detach, bool drop_part, const Context & query_context) { assertNotReadonly(); if (!is_leader) - throw Exception("DROP PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); + throw Exception("DROP PART|PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - String partition_id = getPartitionIDFromQuery(partition, query_context); - LogEntry entry; - if (dropPartsInPartition(*zookeeper, partition_id, entry, detach)) + bool did_drop; + + if (drop_part) + { + String part_name = partition->as().value.safeGet(); + did_drop = dropPart(zookeeper, part_name, entry, detach); + } + else + { + String partition_id = getPartitionIDFromQuery(partition, query_context); + did_drop = dropAllPartsInPartition(*zookeeper, partition_id, entry, detach); + } + + if (did_drop) { /// If necessary, wait until the operation is performed on itself or on all replicas. if (query_context.getSettingsRef().replication_alter_partitions_sync != 0) @@ -4041,7 +4058,12 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr &, const ASTPtr & pa } /// Cleaning possibly stored information about parts from /quorum/last_part node in ZooKeeper. - cleanLastPartNode(partition_id); + /// TODO(nv) how is this related to dropPart? Is it? + if (!drop_part) + { + String partition_id = getPartitionIDFromQuery(partition, query_context); + cleanLastPartNode(partition_id); + } } @@ -4062,7 +4084,7 @@ void StorageReplicatedMergeTree::truncate( { LogEntry entry; - if (dropPartsInPartition(*zookeeper, partition_id, entry, false)) + if (dropAllPartsInPartition(*zookeeper, partition_id, entry, false)) waitForAllReplicasToProcessLogEntry(entry); } } @@ -5665,9 +5687,65 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI return true; } +bool StorageReplicatedMergeTree::dropPart( + zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach) +{ + LOG_TRACE(log, "Will try to insert a log entry to DROP_RANGE for part: " + part_name); -bool StorageReplicatedMergeTree::dropPartsInPartition( - zkutil::ZooKeeper & zookeeper, String & partition_id, StorageReplicatedMergeTree::LogEntry & entry, bool detach) + auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); + + while (true) + { + ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper); + + /// TODO(nv) It is possible that part does not exist on replica which executes this statement. + /// Also, it possible for the part to not exist on any replicas, replica which created log entries for the part disappeared. + auto part = data_parts_by_info.find(part_info); + if (part == data_parts_by_info.end()) + throw Exception("Part " + part_name + " not found locally, won't try to drop it.", ErrorCodes::NOT_IMPLEMENTED); + + /// TODO(nv) get ops and commit together w/ log entry? + clearBlocksInPartition(*zookeeper, part_info.partition_id, part_info.min_block, part_info.max_block); + + /// There isn't a lot we can do otherwise. Can't cancel merges because it is possible that a replica already + /// finished the merge. + if (partIsAssignedToBackgroundOperation(*part)) + throw Exception("Part " + part_name + + " is currently participating in a background operation (mutation/merge)" + + ", try again later.", ErrorCodes::PART_IS_TEMPORARILY_LOCKED); + + /// If `part_name` is result of a recent merge and source parts are still available then + /// DROP_RANGE with detach will move this part together with source parts to `detached/` dir. + entry.type = LogEntry::DROP_RANGE; + entry.source_replica = replica_name; + entry.new_part_name = part_name; + entry.detach = detach; + entry.create_time = time(nullptr); + + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/log", merge_pred.getVersion())); /// Make sure no new events were added to the log. + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); /// Just update version. + Coordination::Responses responses; + Coordination::Error rc = zookeeper->tryMulti(ops, responses); + + if (rc == Coordination::Error::ZBADVERSION) + { + LOG_TRACE(log, "A new log entry appeared while trying to commit DROP RANGE. Retry."); + continue; + } + else + zkutil::KeeperMultiException::check(rc, ops, responses); + + String log_znode_path = dynamic_cast(*responses[1]).path_created; + entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + return true; + } +} + +bool StorageReplicatedMergeTree::dropAllPartsInPartition( + zkutil::ZooKeeper & zookeeper, String & partition_id, LogEntry & entry, bool detach) { MergeTreePartInfo drop_range_info; if (!getFakePartCoveringAllPartsInPartition(partition_id, drop_range_info)) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index ad992a10f08..ece2294ac60 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -529,11 +529,12 @@ private: /// Info about how other replicas can access this one. ReplicatedMergeTreeAddress getReplicatedMergeTreeAddress() const; - bool dropPartsInPartition(zkutil::ZooKeeper & zookeeper, String & partition_id, - StorageReplicatedMergeTree::LogEntry & entry, bool detach); + bool dropPart(zkutil::ZooKeeperPtr & zookeeper, String partition_id, LogEntry & entry, bool detach); + bool dropAllPartsInPartition( + zkutil::ZooKeeper & zookeeper, String & partition_id, LogEntry & entry, bool detach); // Partition helpers - void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & query_context); + void dropPartition(const ASTPtr &, const ASTPtr & partition, bool detach, bool drop_part, const Context & query_context); PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, const Context & query_context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & query_context); void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & query_context); diff --git a/tests/queries/0_stateless/01451_replicated_detach_part.reference b/tests/queries/0_stateless/01451_replicated_detach_part.reference new file mode 100644 index 00000000000..b68ee092c91 --- /dev/null +++ b/tests/queries/0_stateless/01451_replicated_detach_part.reference @@ -0,0 +1,9 @@ +0 +1 +2 +0 +2 +all_1_1_0 +0 +1 +2 diff --git a/tests/queries/0_stateless/01451_replicated_detach_part.sql b/tests/queries/0_stateless/01451_replicated_detach_part.sql new file mode 100644 index 00000000000..602fca2d371 --- /dev/null +++ b/tests/queries/0_stateless/01451_replicated_detach_part.sql @@ -0,0 +1,28 @@ +SET replication_alter_partitions_sync = 2; + +DROP TABLE IF EXISTS attach_01451_r1; +DROP TABLE IF EXISTS attach_01451_r2; + +CREATE TABLE attach_01451_r1 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/attach', 'r1') order by tuple() settings max_replicated_merges_in_queue = 0; +CREATE TABLE attach_01451_r2 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/attach', 'r2') order by tuple() settings max_replicated_merges_in_queue = 0; + +INSERT INTO attach_01451_r1 VALUES (0); +INSERT INTO attach_01451_r1 VALUES (1); +INSERT INTO attach_01451_r1 VALUES (2); + +SELECT v FROM attach_01451_r1 ORDER BY v; + +ALTER TABLE attach_01451_r2 DETACH PART 'all_1_1_0'; + +SELECT v FROM attach_01451_r1 ORDER BY v; + +SELECT name FROM system.detached_parts WHERE table = 'attach_01451_r2'; + +ALTER TABLE attach_01451_r2 ATTACH PART 'all_1_1_0'; + +SELECT v FROM attach_01451_r1 ORDER BY v; + +SELECT name FROM system.detached_parts WHERE table = 'attach_01451_r2'; + +DROP TABLE attach_01451_r1; +DROP TABLE attach_01451_r2; diff --git a/tests/queries/0_stateless/01452_replicated_drop_part.reference b/tests/queries/0_stateless/01452_replicated_drop_part.reference new file mode 100644 index 00000000000..6fd8504e46b --- /dev/null +++ b/tests/queries/0_stateless/01452_replicated_drop_part.reference @@ -0,0 +1,6 @@ +0 +1 +2 +0 +2 +all_0_2_1 diff --git a/tests/queries/0_stateless/01452_replicated_drop_part.sql b/tests/queries/0_stateless/01452_replicated_drop_part.sql new file mode 100644 index 00000000000..406019fdaf1 --- /dev/null +++ b/tests/queries/0_stateless/01452_replicated_drop_part.sql @@ -0,0 +1,25 @@ +SET replication_alter_partitions_sync = 2; + +DROP TABLE IF EXISTS attach_r1; +DROP TABLE IF EXISTS attach_r2; + +CREATE TABLE attach_r1 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01452/attach', 'r1') order by tuple() settings max_replicated_merges_in_queue = 0; +CREATE TABLE attach_r2 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01452/attach', 'r2') order by tuple() settings max_replicated_merges_in_queue = 0; + +INSERT INTO attach_r1 VALUES (0); +INSERT INTO attach_r1 VALUES (1); +INSERT INTO attach_r1 VALUES (2); + +SELECT v FROM attach_r1 ORDER BY v; + +ALTER TABLE attach_r2 DROP PART 'all_1_1_0'; + +SELECT v FROM attach_r1 ORDER BY v; + +ALTER TABLE attach_r1 MODIFY SETTING max_replicated_merges_in_queue = 1; +OPTIMIZE TABLE attach_r1 FINAL; + +SELECT name FROM system.parts WHERE table = 'attach_r1' AND active; + +DROP TABLE attach_r1; +DROP TABLE attach_r2; From 0d28fdd1161511ac9d1eb56c6291f7fba58a2d4b Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 7 Sep 2020 13:58:34 +0100 Subject: [PATCH 002/114] Sync replicas between operations --- tests/queries/0_stateless/01451_replicated_detach_part.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01451_replicated_detach_part.sql b/tests/queries/0_stateless/01451_replicated_detach_part.sql index 602fca2d371..b414bb09e69 100644 --- a/tests/queries/0_stateless/01451_replicated_detach_part.sql +++ b/tests/queries/0_stateless/01451_replicated_detach_part.sql @@ -12,6 +12,7 @@ INSERT INTO attach_01451_r1 VALUES (2); SELECT v FROM attach_01451_r1 ORDER BY v; +SYSTEM SYNC REPLICA attach_01451_r2; ALTER TABLE attach_01451_r2 DETACH PART 'all_1_1_0'; SELECT v FROM attach_01451_r1 ORDER BY v; @@ -20,6 +21,7 @@ SELECT name FROM system.detached_parts WHERE table = 'attach_01451_r2'; ALTER TABLE attach_01451_r2 ATTACH PART 'all_1_1_0'; +SYSTEM SYNC REPLICA attach_01451_r1; SELECT v FROM attach_01451_r1 ORDER BY v; SELECT name FROM system.detached_parts WHERE table = 'attach_01451_r2'; From 18763747705f214918cc1c4b0d8623dde9e1144d Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 7 Sep 2020 16:05:19 +0100 Subject: [PATCH 003/114] ALTER TABLE ... DROP|DETACH PART for MergeTree --- src/Storages/StorageMergeTree.cpp | 32 +++++++++++++++---- src/Storages/StorageMergeTree.h | 2 +- .../01451_detach_drop_part.reference | 12 +++++++ .../0_stateless/01451_detach_drop_part.sql | 31 ++++++++++++++++++ 4 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/01451_detach_drop_part.reference create mode 100644 tests/queries/0_stateless/01451_detach_drop_part.sql diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 8b233fb2408..9ea83b3bf73 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1066,8 +1066,13 @@ Pipe StorageMergeTree::alterPartition( switch (command.type) { case PartitionCommand::DROP_PARTITION: - checkPartitionCanBeDropped(command.partition); - dropPartition(command.partition, command.detach, query_context); + if (command.part) + { + /// TODO(nv) what would be a good check here? + } + else + checkPartitionCanBeDropped(command.partition); + dropPartition(command.partition, command.detach, command.part, query_context); break; case PartitionCommand::DROP_DETACHED_PARTITION: @@ -1165,7 +1170,7 @@ ActionLock StorageMergeTree::stopMergesAndWait() } -void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, const Context & context) +void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool drop_part, const Context & context) { { /// Asks to complete merges and does not allow them to start. @@ -1173,10 +1178,23 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons auto merge_blocker = stopMergesAndWait(); auto metadata_snapshot = getInMemoryMetadataPtr(); - String partition_id = getPartitionIDFromQuery(partition, context); + + MergeTreeData::DataPartsVector parts_to_remove; /// TODO: should we include PreComitted parts like in Replicated case? - auto parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); + if (drop_part) + { + String part_name = partition->as().value.safeGet(); + auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Committed}); + + if (part) + parts_to_remove.push_back(part); + } else + { + String partition_id = getPartitionIDFromQuery(partition, context); + parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); + } + // TODO should we throw an exception if parts_to_remove is empty? removePartsFromWorkingSet(parts_to_remove, true); @@ -1191,9 +1209,9 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } if (detach) - LOG_INFO(log, "Detached {} parts inside partition ID {}.", parts_to_remove.size(), partition_id); + LOG_INFO(log, "Detached {} parts.", parts_to_remove.size()); else - LOG_INFO(log, "Removed {} parts inside partition ID {}.", parts_to_remove.size(), partition_id); + LOG_INFO(log, "Removed {} parts.", parts_to_remove.size()); } clearOldPartsFromFilesystem(); diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 5662f9e0088..2f82f88e7ca 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -150,7 +150,7 @@ private: void clearOldMutations(bool truncate = false); // Partition helpers - void dropPartition(const ASTPtr & partition, bool detach, const Context & context); + void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, const Context & context); PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); diff --git a/tests/queries/0_stateless/01451_detach_drop_part.reference b/tests/queries/0_stateless/01451_detach_drop_part.reference new file mode 100644 index 00000000000..48e639c9125 --- /dev/null +++ b/tests/queries/0_stateless/01451_detach_drop_part.reference @@ -0,0 +1,12 @@ +0 +1 +2 +0 +2 +all_2_2_0 +0 +1 +2 +-- drop part -- +0 +2 diff --git a/tests/queries/0_stateless/01451_detach_drop_part.sql b/tests/queries/0_stateless/01451_detach_drop_part.sql new file mode 100644 index 00000000000..92adaef14d9 --- /dev/null +++ b/tests/queries/0_stateless/01451_detach_drop_part.sql @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS attach_01451_mt; + +CREATE TABLE attach_01451_mt (v UInt8) ENGINE = MergeTree() order by tuple(); + +INSERT INTO attach_01451_mt VALUES (0); +INSERT INTO attach_01451_mt VALUES (1); +INSERT INTO attach_01451_mt VALUES (2); + +SELECT v FROM attach_01451_mt ORDER BY v; + +ALTER TABLE attach_01451_mt DETACH PART 'all_2_2_0'; + +SELECT v FROM attach_01451_mt ORDER BY v; + +SELECT name FROM system.detached_parts WHERE table = 'attach_01451_mt'; + +ALTER TABLE attach_01451_mt ATTACH PART 'all_2_2_0'; + +SELECT v FROM attach_01451_mt ORDER BY v; + +SELECT name FROM system.detached_parts WHERE table = 'attach_01451_mt'; + +SELECT '-- drop part --'; + +ALTER TABLE attach_01451_mt DROP PART 'all_4_4_0'; + +ALTER TABLE attach_01451_mt ATTACH PART 'all_4_4_0'; -- { serverError 233 } + +SELECT v FROM attach_01451_mt ORDER BY v; + +DROP TABLE attach_01451_mt; From 97d0b5ab235aa9e7c5c63ac83eba58e8a62136de Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 7 Sep 2020 16:19:47 +0100 Subject: [PATCH 004/114] Remove unused query argument --- src/Interpreters/InterpreterAlterQuery.cpp | 2 +- src/Storages/IStorage.cpp | 1 - src/Storages/IStorage.h | 1 - src/Storages/StorageMaterializedView.cpp | 4 ++-- src/Storages/StorageMaterializedView.h | 2 +- src/Storages/StorageMergeTree.cpp | 3 +-- src/Storages/StorageMergeTree.h | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 6 ++---- src/Storages/StorageReplicatedMergeTree.h | 3 +-- 9 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index c0ed5d8a36c..bf55e372066 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -87,7 +87,7 @@ BlockIO InterpreterAlterQuery::execute() if (!partition_commands.empty()) { table->checkAlterPartitionIsPossible(partition_commands, metadata_snapshot, context.getSettingsRef()); - auto partition_commands_pipe = table->alterPartition(query_ptr, metadata_snapshot, partition_commands, context); + auto partition_commands_pipe = table->alterPartition(metadata_snapshot, partition_commands, context); if (!partition_commands_pipe.empty()) res.pipeline.init(std::move(partition_commands_pipe)); } diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 494b6a19c02..1eea364190e 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -92,7 +92,6 @@ Pipe IStorage::read( } Pipe IStorage::alterPartition( - const ASTPtr & /* query */, const StorageMetadataPtr & /* metadata_snapshot */, const PartitionCommands & /* commands */, const Context & /* context */) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index bdbb2eb9d80..d874663ae11 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -348,7 +348,6 @@ public: * Should handle locks for each command on its own. */ virtual Pipe alterPartition( - const ASTPtr & /* query */, const StorageMetadataPtr & /* metadata_snapshot */, const PartitionCommands & /* commands */, const Context & /* context */); diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index a2e3fae0951..fd2cec95647 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -249,10 +249,10 @@ void StorageMaterializedView::checkAlterIsPossible(const AlterCommands & command } Pipe StorageMaterializedView::alterPartition( - const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context) + const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context) { checkStatementCanBeForwarded(); - return getTargetTable()->alterPartition(query, metadata_snapshot, commands, context); + return getTargetTable()->alterPartition(metadata_snapshot, commands, context); } void StorageMaterializedView::checkAlterPartitionIsPossible( diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 1ee4246c7f1..4faa3b69fe7 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -51,7 +51,7 @@ public: void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const override; - Pipe alterPartition(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context) override; + Pipe alterPartition(const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context) override; void checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & metadata_snapshot, const Settings & settings) const override; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 9ea83b3bf73..8164bce0f4d 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1054,7 +1054,6 @@ bool StorageMergeTree::optimize( } Pipe StorageMergeTree::alterPartition( - const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & query_context) @@ -1130,7 +1129,7 @@ Pipe StorageMergeTree::alterPartition( break; default: - IStorage::alterPartition(query, metadata_snapshot, commands, query_context); // should throw an exception. + IStorage::alterPartition(metadata_snapshot, commands, query_context); // should throw an exception. } for (auto & command_result : current_command_results) diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 2f82f88e7ca..8c970e56f13 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -62,7 +62,6 @@ public: const Context & context) override; Pipe alterPartition( - const ASTPtr & query, const StorageMetadataPtr & /* metadata_snapshot */, const PartitionCommands & commands, const Context & context) override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 312040cbf16..e8007440f2f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3882,7 +3882,6 @@ void StorageReplicatedMergeTree::alter( } Pipe StorageReplicatedMergeTree::alterPartition( - const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & query_context) @@ -3900,7 +3899,7 @@ Pipe StorageReplicatedMergeTree::alterPartition( } else checkPartitionCanBeDropped(command.partition); - dropPartition(query, command.partition, command.detach, command.part, query_context); + dropPartition(command.partition, command.detach, command.part, query_context); break; case PartitionCommand::DROP_DETACHED_PARTITION: @@ -4022,8 +4021,7 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition(const St } -void StorageReplicatedMergeTree::dropPartition( - const ASTPtr &, const ASTPtr & partition, bool detach, bool drop_part, const Context & query_context) +void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool drop_part, const Context & query_context) { assertNotReadonly(); if (!is_leader) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index ece2294ac60..58c82511a27 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -112,7 +112,6 @@ public: void alter(const AlterCommands & params, const Context & query_context, TableLockHolder & table_lock_holder) override; Pipe alterPartition( - const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & query_context) override; @@ -534,7 +533,7 @@ private: zkutil::ZooKeeper & zookeeper, String & partition_id, LogEntry & entry, bool detach); // Partition helpers - void dropPartition(const ASTPtr &, const ASTPtr & partition, bool detach, bool drop_part, const Context & query_context); + void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, const Context & query_context); PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, const Context & query_context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & query_context); void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & query_context); From 1007d2676197a770ddd352a1e9e86699e44f1a7e Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 7 Sep 2020 17:23:59 +0100 Subject: [PATCH 005/114] Simplify ALTER TABLE ... DETACH|DROP PART tests --- .../01451_detach_drop_part.reference | 4 ++ .../0_stateless/01451_detach_drop_part.sql | 41 +++++++++------- ...1451_replicated_detach_drop_part.reference | 16 +++++++ .../01451_replicated_detach_drop_part.sql | 47 +++++++++++++++++++ .../01451_replicated_detach_part.reference | 9 ---- .../01451_replicated_detach_part.sql | 30 ------------ .../01452_replicated_drop_part.reference | 6 --- .../01452_replicated_drop_part.sql | 25 ---------- 8 files changed, 92 insertions(+), 86 deletions(-) create mode 100644 tests/queries/0_stateless/01451_replicated_detach_drop_part.reference create mode 100644 tests/queries/0_stateless/01451_replicated_detach_drop_part.sql delete mode 100644 tests/queries/0_stateless/01451_replicated_detach_part.reference delete mode 100644 tests/queries/0_stateless/01451_replicated_detach_part.sql delete mode 100644 tests/queries/0_stateless/01452_replicated_drop_part.reference delete mode 100644 tests/queries/0_stateless/01452_replicated_drop_part.sql diff --git a/tests/queries/0_stateless/01451_detach_drop_part.reference b/tests/queries/0_stateless/01451_detach_drop_part.reference index 48e639c9125..bc4f1b6be80 100644 --- a/tests/queries/0_stateless/01451_detach_drop_part.reference +++ b/tests/queries/0_stateless/01451_detach_drop_part.reference @@ -10,3 +10,7 @@ all_2_2_0 -- drop part -- 0 2 +-- resume merges -- +0 +2 +all_1_3_1 diff --git a/tests/queries/0_stateless/01451_detach_drop_part.sql b/tests/queries/0_stateless/01451_detach_drop_part.sql index 92adaef14d9..47e61f2d924 100644 --- a/tests/queries/0_stateless/01451_detach_drop_part.sql +++ b/tests/queries/0_stateless/01451_detach_drop_part.sql @@ -1,31 +1,40 @@ -DROP TABLE IF EXISTS attach_01451_mt; +DROP TABLE IF EXISTS mt; -CREATE TABLE attach_01451_mt (v UInt8) ENGINE = MergeTree() order by tuple(); +CREATE TABLE mt (v UInt8) ENGINE = MergeTree() order by tuple(); +SYSTEM STOP MERGES; -INSERT INTO attach_01451_mt VALUES (0); -INSERT INTO attach_01451_mt VALUES (1); -INSERT INTO attach_01451_mt VALUES (2); +INSERT INTO mt VALUES (0); +INSERT INTO mt VALUES (1); +INSERT INTO mt VALUES (2); -SELECT v FROM attach_01451_mt ORDER BY v; +SELECT v FROM mt ORDER BY v; -ALTER TABLE attach_01451_mt DETACH PART 'all_2_2_0'; +ALTER TABLE mt DETACH PART 'all_2_2_0'; -SELECT v FROM attach_01451_mt ORDER BY v; +SELECT v FROM mt ORDER BY v; -SELECT name FROM system.detached_parts WHERE table = 'attach_01451_mt'; +SELECT name FROM system.detached_parts WHERE table = 'mt'; -ALTER TABLE attach_01451_mt ATTACH PART 'all_2_2_0'; +ALTER TABLE mt ATTACH PART 'all_2_2_0'; -SELECT v FROM attach_01451_mt ORDER BY v; +SELECT v FROM mt ORDER BY v; -SELECT name FROM system.detached_parts WHERE table = 'attach_01451_mt'; +SELECT name FROM system.detached_parts WHERE table = 'mt'; SELECT '-- drop part --'; -ALTER TABLE attach_01451_mt DROP PART 'all_4_4_0'; +ALTER TABLE mt DROP PART 'all_4_4_0'; -ALTER TABLE attach_01451_mt ATTACH PART 'all_4_4_0'; -- { serverError 233 } +ALTER TABLE mt ATTACH PART 'all_4_4_0'; -- { serverError 233 } -SELECT v FROM attach_01451_mt ORDER BY v; +SELECT v FROM mt ORDER BY v; -DROP TABLE attach_01451_mt; +SELECT '-- resume merges --'; +SYSTEM START MERGES; +OPTIMIZE TABLE mt FINAL; + +SELECT v FROM mt ORDER BY v; + +SELECT name FROM system.parts WHERE table = 'mt' AND active; + +DROP TABLE mt; diff --git a/tests/queries/0_stateless/01451_replicated_detach_drop_part.reference b/tests/queries/0_stateless/01451_replicated_detach_drop_part.reference new file mode 100644 index 00000000000..109012cba70 --- /dev/null +++ b/tests/queries/0_stateless/01451_replicated_detach_drop_part.reference @@ -0,0 +1,16 @@ +0 +1 +2 +0 +2 +all_1_1_0 +0 +1 +2 +-- drop part -- +0 +2 +-- resume merges -- +0 +2 +all_0_2_1 diff --git a/tests/queries/0_stateless/01451_replicated_detach_drop_part.sql b/tests/queries/0_stateless/01451_replicated_detach_drop_part.sql new file mode 100644 index 00000000000..1209f11b68e --- /dev/null +++ b/tests/queries/0_stateless/01451_replicated_detach_drop_part.sql @@ -0,0 +1,47 @@ +SET replication_alter_partitions_sync = 2; + +DROP TABLE IF EXISTS replica1; +DROP TABLE IF EXISTS replica2; + +CREATE TABLE replica1 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/attach', 'r1') order by tuple() settings max_replicated_merges_in_queue = 0; +CREATE TABLE replica2 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/attach', 'r2') order by tuple() settings max_replicated_merges_in_queue = 0; + +INSERT INTO replica1 VALUES (0); +INSERT INTO replica1 VALUES (1); +INSERT INTO replica1 VALUES (2); + +SELECT v FROM replica1 ORDER BY v; + +SYSTEM SYNC REPLICA replica2; +ALTER TABLE replica2 DETACH PART 'all_1_1_0'; + +SELECT v FROM replica1 ORDER BY v; + +SELECT name FROM system.detached_parts WHERE table = 'replica2'; + +ALTER TABLE replica2 ATTACH PART 'all_1_1_0'; + +SYSTEM SYNC REPLICA replica1; +SELECT v FROM replica1 ORDER BY v; + +SELECT name FROM system.detached_parts WHERE table = 'replica2'; + +SELECT '-- drop part --'; + +ALTER TABLE replica1 DROP PART 'all_3_3_0'; + +ALTER TABLE replica1 ATTACH PART 'all_3_3_0'; -- { serverError 233 } + +SELECT v FROM replica1 ORDER BY v; + +SELECT '-- resume merges --'; + +ALTER TABLE replica1 MODIFY SETTING max_replicated_merges_in_queue = 1; +OPTIMIZE TABLE replica1 FINAL; + +SELECT v FROM replica1 ORDER BY v; + +SELECT name FROM system.parts WHERE table = 'replica2' AND active; + +DROP TABLE replica1; +DROP TABLE replica2; diff --git a/tests/queries/0_stateless/01451_replicated_detach_part.reference b/tests/queries/0_stateless/01451_replicated_detach_part.reference deleted file mode 100644 index b68ee092c91..00000000000 --- a/tests/queries/0_stateless/01451_replicated_detach_part.reference +++ /dev/null @@ -1,9 +0,0 @@ -0 -1 -2 -0 -2 -all_1_1_0 -0 -1 -2 diff --git a/tests/queries/0_stateless/01451_replicated_detach_part.sql b/tests/queries/0_stateless/01451_replicated_detach_part.sql deleted file mode 100644 index b414bb09e69..00000000000 --- a/tests/queries/0_stateless/01451_replicated_detach_part.sql +++ /dev/null @@ -1,30 +0,0 @@ -SET replication_alter_partitions_sync = 2; - -DROP TABLE IF EXISTS attach_01451_r1; -DROP TABLE IF EXISTS attach_01451_r2; - -CREATE TABLE attach_01451_r1 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/attach', 'r1') order by tuple() settings max_replicated_merges_in_queue = 0; -CREATE TABLE attach_01451_r2 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/attach', 'r2') order by tuple() settings max_replicated_merges_in_queue = 0; - -INSERT INTO attach_01451_r1 VALUES (0); -INSERT INTO attach_01451_r1 VALUES (1); -INSERT INTO attach_01451_r1 VALUES (2); - -SELECT v FROM attach_01451_r1 ORDER BY v; - -SYSTEM SYNC REPLICA attach_01451_r2; -ALTER TABLE attach_01451_r2 DETACH PART 'all_1_1_0'; - -SELECT v FROM attach_01451_r1 ORDER BY v; - -SELECT name FROM system.detached_parts WHERE table = 'attach_01451_r2'; - -ALTER TABLE attach_01451_r2 ATTACH PART 'all_1_1_0'; - -SYSTEM SYNC REPLICA attach_01451_r1; -SELECT v FROM attach_01451_r1 ORDER BY v; - -SELECT name FROM system.detached_parts WHERE table = 'attach_01451_r2'; - -DROP TABLE attach_01451_r1; -DROP TABLE attach_01451_r2; diff --git a/tests/queries/0_stateless/01452_replicated_drop_part.reference b/tests/queries/0_stateless/01452_replicated_drop_part.reference deleted file mode 100644 index 6fd8504e46b..00000000000 --- a/tests/queries/0_stateless/01452_replicated_drop_part.reference +++ /dev/null @@ -1,6 +0,0 @@ -0 -1 -2 -0 -2 -all_0_2_1 diff --git a/tests/queries/0_stateless/01452_replicated_drop_part.sql b/tests/queries/0_stateless/01452_replicated_drop_part.sql deleted file mode 100644 index 406019fdaf1..00000000000 --- a/tests/queries/0_stateless/01452_replicated_drop_part.sql +++ /dev/null @@ -1,25 +0,0 @@ -SET replication_alter_partitions_sync = 2; - -DROP TABLE IF EXISTS attach_r1; -DROP TABLE IF EXISTS attach_r2; - -CREATE TABLE attach_r1 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01452/attach', 'r1') order by tuple() settings max_replicated_merges_in_queue = 0; -CREATE TABLE attach_r2 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01452/attach', 'r2') order by tuple() settings max_replicated_merges_in_queue = 0; - -INSERT INTO attach_r1 VALUES (0); -INSERT INTO attach_r1 VALUES (1); -INSERT INTO attach_r1 VALUES (2); - -SELECT v FROM attach_r1 ORDER BY v; - -ALTER TABLE attach_r2 DROP PART 'all_1_1_0'; - -SELECT v FROM attach_r1 ORDER BY v; - -ALTER TABLE attach_r1 MODIFY SETTING max_replicated_merges_in_queue = 1; -OPTIMIZE TABLE attach_r1 FINAL; - -SELECT name FROM system.parts WHERE table = 'attach_r1' AND active; - -DROP TABLE attach_r1; -DROP TABLE attach_r2; From a62e96c7aa03e556465f9d8418fe3de76720f9f0 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 7 Sep 2020 18:31:07 +0100 Subject: [PATCH 006/114] nit: fix argument names for clang-10 --- src/Storages/StorageReplicatedMergeTree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 58c82511a27..3ce946a54c0 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -528,7 +528,7 @@ private: /// Info about how other replicas can access this one. ReplicatedMergeTreeAddress getReplicatedMergeTreeAddress() const; - bool dropPart(zkutil::ZooKeeperPtr & zookeeper, String partition_id, LogEntry & entry, bool detach); + bool dropPart(zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach); bool dropAllPartsInPartition( zkutil::ZooKeeper & zookeeper, String & partition_id, LogEntry & entry, bool detach); From 193b572a0581b621069684e437daf93980e3fcbc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 17 Sep 2020 15:11:23 +0300 Subject: [PATCH 007/114] Add MaterializingStep. --- .../QueryPlan/MaterializingStep.cpp | 39 +++++++++++++++++++ src/Processors/QueryPlan/MaterializingStep.h | 18 +++++++++ src/Processors/ya.make | 1 + 3 files changed, 58 insertions(+) create mode 100644 src/Processors/QueryPlan/MaterializingStep.cpp create mode 100644 src/Processors/QueryPlan/MaterializingStep.h diff --git a/src/Processors/QueryPlan/MaterializingStep.cpp b/src/Processors/QueryPlan/MaterializingStep.cpp new file mode 100644 index 00000000000..f5313369020 --- /dev/null +++ b/src/Processors/QueryPlan/MaterializingStep.cpp @@ -0,0 +1,39 @@ +#include +#include +#include + +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +MaterializingStep::MaterializingStep(const DataStream & input_stream_) + : ITransformingStep(input_stream_, materializeBlock(input_stream_.header), getTraits()) +{ +} + +void MaterializingStep::transformPipeline(QueryPipeline & pipeline) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header); + }); +} + +} diff --git a/src/Processors/QueryPlan/MaterializingStep.h b/src/Processors/QueryPlan/MaterializingStep.h new file mode 100644 index 00000000000..c1ffcaeb775 --- /dev/null +++ b/src/Processors/QueryPlan/MaterializingStep.h @@ -0,0 +1,18 @@ +#pragma once +#include + +namespace DB +{ + +/// Convert one block structure to another. See ConvertingTransform. +class MaterializingStep : public ITransformingStep +{ +public: + explicit MaterializingStep(const DataStream & input_stream_); + + String getName() const override { return "Materializing"; } + + void transformPipeline(QueryPipeline & pipeline) override; +}; + +} diff --git a/src/Processors/ya.make b/src/Processors/ya.make index c69d274a70e..cd8857926bb 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -105,6 +105,7 @@ SRCS( QueryPlan/ITransformingStep.cpp QueryPlan/LimitByStep.cpp QueryPlan/LimitStep.cpp + QueryPlan/MaterializingStep.cpp QueryPlan/MergeSortingStep.cpp QueryPlan/MergingAggregatedStep.cpp QueryPlan/MergingSortedStep.cpp From 0bf4e8e6e912eda3f9d7131cf4581020e69c16e9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 17 Sep 2020 16:22:24 +0300 Subject: [PATCH 008/114] Support reading with QueryPlan for StorageView. --- src/Processors/QueryPipeline.h | 2 + src/Processors/QueryPlan/MaterializingStep.h | 2 +- .../QueryPlan/SettingQuotaAndLimitsStep.cpp | 53 +++++++++++++++++++ .../QueryPlan/SettingQuotaAndLimitsStep.h | 42 +++++++++++++++ src/Processors/ya.make | 1 + src/Storages/StorageView.cpp | 52 ++++++++++++++++++ src/Storages/StorageView.h | 13 +++++ 7 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp create mode 100644 src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 45b410ab323..40aabf43ecb 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -99,6 +99,8 @@ public: void addInterpreterContext(std::shared_ptr context) { pipe.addInterpreterContext(std::move(context)); } void addStorageHolder(StoragePtr storage) { pipe.addStorageHolder(std::move(storage)); } void addQueryPlan(std::unique_ptr plan) { pipe.addQueryPlan(std::move(plan)); } + void setLimits(const StreamLocalLimits & limits) { pipe.setLimits(limits); } + void setQuota(const std::shared_ptr & quota) { pipe.setQuota(quota); }; /// For compatibility with IBlockInputStream. void setProgressCallback(const ProgressCallback & callback); diff --git a/src/Processors/QueryPlan/MaterializingStep.h b/src/Processors/QueryPlan/MaterializingStep.h index c1ffcaeb775..72b3133dfe4 100644 --- a/src/Processors/QueryPlan/MaterializingStep.h +++ b/src/Processors/QueryPlan/MaterializingStep.h @@ -4,7 +4,7 @@ namespace DB { -/// Convert one block structure to another. See ConvertingTransform. +/// Materialize constants. See MaterializingTransform. class MaterializingStep : public ITransformingStep { public: diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp new file mode 100644 index 00000000000..73cd459fa5d --- /dev/null +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp @@ -0,0 +1,53 @@ +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +SettingQuotaAndLimitsStep::SettingQuotaAndLimitsStep( + const DataStream & input_stream_, + StoragePtr storage_, + TableLockHolder table_lock_, + StreamLocalLimits limits_, + std::shared_ptr quota_, + std::shared_ptr context_) + : ITransformingStep(input_stream_, input_stream_.header, getTraits()) + , storage(std::move(storage_)) + , table_lock(std::move(table_lock_)) + , limits(std::move(limits_)) + , quota(std::move(quota_)) + , context(std::move(context_)) +{ +} + +void SettingQuotaAndLimitsStep::transformPipeline(QueryPipeline & pipeline) +{ + /// Table lock is stored inside pipeline here. + pipeline.addTableLock(table_lock); + + pipeline.setLimits(limits); + + if (quota) + pipeline.setQuota(quota); + + pipeline.addInterpreterContext(std::move(context)); + pipeline.addStorageHolder(std::move(storage)); +} + +} diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h new file mode 100644 index 00000000000..538d3c35b9d --- /dev/null +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h @@ -0,0 +1,42 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +class IStorage; +using StoragePtr = std::shared_ptr; + +struct StorageInMemoryMetadata; +using StorageMetadataPtr = std::shared_ptr; + +class EnabledQuota; + +/// Add limits, quota, table_lock and other stuff to pipeline. +/// Doesn't change DataStream. +class SettingQuotaAndLimitsStep : public ITransformingStep +{ +public: + SettingQuotaAndLimitsStep( + const DataStream & input_stream_, + StoragePtr storage_, + TableLockHolder table_lock_, + StreamLocalLimits limits_, + std::shared_ptr quota_, + std::shared_ptr context_); + + String getName() const override { return "SettingQuotaAndLimits"; } + + void transformPipeline(QueryPipeline & pipeline) override; + +private: + StoragePtr storage; + TableLockHolder table_lock; + StreamLocalLimits limits; + std::shared_ptr quota; + std::shared_ptr context; +}; + +} diff --git a/src/Processors/ya.make b/src/Processors/ya.make index cd8857926bb..08de142479b 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -116,6 +116,7 @@ SRCS( QueryPlan/ReadFromStorageStep.cpp QueryPlan/ReadNothingStep.cpp QueryPlan/RollupStep.cpp + QueryPlan/SettingQuotaAndLimitsStep.cpp QueryPlan/TotalsHavingStep.cpp QueryPlan/UnionStep.cpp ResizeProcessor.cpp diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 4b7733c1cd2..a7cba22bebf 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -16,6 +16,9 @@ #include #include #include +#include +#include +#include namespace DB { @@ -89,6 +92,55 @@ Pipe StorageView::read( return QueryPipeline::getPipe(std::move(pipeline)); } +void StorageView::read( + QueryPlan & query_plan, + TableLockHolder table_lock, + StorageMetadataPtr metadata_snapshot, + StreamLocalLimits & limits, + std::shared_ptr quota, + const Names & column_names, + const SelectQueryInfo & query_info, + std::shared_ptr context, + QueryProcessingStage::Enum /*processed_stage*/, + const size_t /*max_block_size*/, + const unsigned /*num_streams*/) +{ + ASTPtr current_inner_query = metadata_snapshot->getSelectQuery().inner_query; + + if (query_info.view_query) + { + if (!query_info.view_query->as()) + throw Exception("Unexpected optimized VIEW query", ErrorCodes::LOGICAL_ERROR); + current_inner_query = query_info.view_query->clone(); + } + + InterpreterSelectWithUnionQuery interpreter(current_inner_query, *context, {}, column_names); + interpreter.buildQueryPlan(query_plan); + + /// It's expected that the columns read from storage are not constant. + /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. + auto materializing = std::make_unique(query_plan.getCurrentDataStream()); + materializing->setStepDescription("Materialize constants after VIEW subquery"); + query_plan.addStep(std::move(materializing)); + + /// And also convert to expected structure. + auto header = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + auto converting = std::make_unique(query_plan.getCurrentDataStream(), header); + converting->setStepDescription("Convert VIEW subquery result to VIEW table structure"); + query_plan.addStep(std::move(converting)); + + /// Extend lifetime of context, table lock, storage. Set limits and quota. + auto adding_limits_and_quota = std::make_unique( + query_plan.getCurrentDataStream(), + shared_from_this(), + std::move(table_lock), + limits, + std::move(quota), + std::move(context)); + adding_limits_and_quota->setStepDescription("Set limits and quota for VIEW subquery"); + query_plan.addStep(std::move(adding_limits_and_quota)); +} + static ASTTableExpression * getFirstTableExpression(ASTSelectQuery & select_query) { auto * select_element = select_query.tables()->children[0]->as(); diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index 682c7424b98..79155209ff8 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -30,6 +30,19 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + TableLockHolder table_lock, + StorageMetadataPtr metadata_snapshot, + StreamLocalLimits & limits, + std::shared_ptr quota, + const Names & column_names, + const SelectQueryInfo & query_info, + std::shared_ptr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name, const StorageMetadataPtr & metadata_snapshot) const { replaceWithSubquery(select_query, metadata_snapshot->getSelectQuery().inner_query->clone(), view_name); From e013acb6b66e0986e2a33d5d6060ab0bd1c27dac Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 17 Sep 2020 16:26:34 +0300 Subject: [PATCH 009/114] Fix build. --- src/Processors/QueryPipeline.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 40aabf43ecb..03547170286 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -100,7 +100,7 @@ public: void addStorageHolder(StoragePtr storage) { pipe.addStorageHolder(std::move(storage)); } void addQueryPlan(std::unique_ptr plan) { pipe.addQueryPlan(std::move(plan)); } void setLimits(const StreamLocalLimits & limits) { pipe.setLimits(limits); } - void setQuota(const std::shared_ptr & quota) { pipe.setQuota(quota); }; + void setQuota(const std::shared_ptr & quota) { pipe.setQuota(quota); } /// For compatibility with IBlockInputStream. void setProgressCallback(const ProgressCallback & callback); From c498b2b3ddb5c39afe93569aed0c9dee813f967c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 17 Sep 2020 16:55:48 +0300 Subject: [PATCH 010/114] Added perftest. --- tests/performance/push_down_limit.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 0dcd9335a52..339709d552e 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,4 +1,8 @@ + CREATE VIEW numbers_view AS SELECT number from numbers_mt(100000000) order by number desc + select number from (select number from numbers(10000000) order by -number) limit 10 select number from (select number from numbers_mt(100000000) order by -number) limit 10 + + select number from numbers_view limit 100 From cbedd44a60a6899d4a0c2e8f48ae502b1e2079b7 Mon Sep 17 00:00:00 2001 From: "philip.han" Date: Fri, 18 Sep 2020 17:43:07 +0900 Subject: [PATCH 011/114] Fix indexOf() to use BloomFilter --- .../MergeTreeIndexConditionBloomFilter.cpp | 33 +++++++++++++++---- .../MergeTreeIndexConditionBloomFilter.h | 2 ++ tests/performance/bloom_filter.xml | 10 ++++-- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index d4736444242..3a7f7408f7c 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -222,9 +222,21 @@ bool MergeTreeIndexConditionBloomFilter::traverseAtomAST(const ASTPtr & node, Bl } } + return traverseFunction(node, block_with_constants, out); +} + +bool MergeTreeIndexConditionBloomFilter::traverseFunction(const ASTPtr & node, Block & block_with_constants, RPNElement & out) +{ + bool maybe_useful = false; + if (const auto * function = node->as()) { const ASTs & arguments = function->arguments->children; + for (auto arg : arguments) + { + if (traverseFunction(arg, block_with_constants, out)) + maybe_useful = true; + } if (arguments.size() != 2) return false; @@ -232,20 +244,29 @@ bool MergeTreeIndexConditionBloomFilter::traverseAtomAST(const ASTPtr & node, Bl if (functionIsInOrGlobalInOperator(function->name)) { if (const auto & prepared_set = getPreparedSet(arguments[1])) - return traverseASTIn(function->name, arguments[0], prepared_set, out); + { + if (traverseASTIn(function->name, arguments[0], prepared_set, out)) + maybe_useful = true; + } } - else if (function->name == "equals" || function->name == "notEquals" || function->name == "has") + else if (function->name == "equals" || function->name == "notEquals" || function->name == "has" || function->name == "indexOf") { Field const_value; DataTypePtr const_type; if (KeyCondition::getConstant(arguments[1], block_with_constants, const_value, const_type)) - return traverseASTEquals(function->name, arguments[0], const_type, const_value, out); + { + if (traverseASTEquals(function->name, arguments[0], const_type, const_value, out)) + maybe_useful = true; + } else if (KeyCondition::getConstant(arguments[0], block_with_constants, const_value, const_type)) - return traverseASTEquals(function->name, arguments[1], const_type, const_value, out); + { + if (traverseASTEquals(function->name, arguments[1], const_type, const_value, out)) + maybe_useful = true; + } } } - return false; + return maybe_useful; } bool MergeTreeIndexConditionBloomFilter::traverseASTIn( @@ -311,7 +332,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTEquals( const DataTypePtr & index_type = header.getByPosition(position).type; const auto * array_type = typeid_cast(index_type.get()); - if (function_name == "has") + if (function_name == "has" || function_name == "indexOf") { out.function = RPNElement::FUNCTION_HAS; diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h index 41b7416253f..40299cb7a43 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h @@ -67,6 +67,8 @@ private: bool traverseAtomAST(const ASTPtr & node, Block & block_with_constants, RPNElement & out); + bool traverseFunction(const ASTPtr & node, Block & block_with_constants, RPNElement & out); + bool traverseASTIn(const String & function_name, const ASTPtr & key_ast, const SetPtr & prepared_set, RPNElement & out); bool traverseASTIn( diff --git a/tests/performance/bloom_filter.xml b/tests/performance/bloom_filter.xml index 3d9096afb03..d1881cde56e 100644 --- a/tests/performance/bloom_filter.xml +++ b/tests/performance/bloom_filter.xml @@ -1,10 +1,16 @@ - - DROP TABLE IF EXISTS test_bf CREATE TABLE test_bf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 8192) ENGINE = MergeTree() ORDER BY id + CREATE TABLE test_bf_indexOf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SYSTEM STOP MERGES + INSERT INTO test_bf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) + INSERT INTO test_bf_indexOf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) + + SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0 + SELECT count() FROM test_bf_indexOf WHERE ary[indexOf(ary, '1')] = '1' + SYSTEM START MERGES + DROP TABLE IF EXISTS test_bf_indexOf DROP TABLE IF EXISTS test_bf From 50674a320fc083122c58a1023a11bc50dadfe570 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 18 Sep 2020 14:39:07 +0300 Subject: [PATCH 012/114] Refactor IStorage::read with query plan. --- src/Interpreters/InterpreterSelectQuery.cpp | 18 +++++- src/Processors/QueryPipeline.h | 1 + .../QueryPlan/ReadFromStorageStep.cpp | 55 +++-------------- .../QueryPlan/ReadFromStorageStep.h | 21 +------ .../QueryPlan/SettingQuotaAndLimitsStep.cpp | 17 +++++- .../QueryPlan/SettingQuotaAndLimitsStep.h | 4 +- src/Storages/IStorage.cpp | 11 +--- src/Storages/IStorage.h | 18 +++--- src/Storages/StorageView.cpp | 60 +++---------------- src/Storages/StorageView.h | 7 +-- 10 files changed, 66 insertions(+), 146 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index cd2c16813b4..073198c3a27 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include @@ -1456,8 +1456,20 @@ void InterpreterSelectQuery::executeFetchColumns( if (!options.ignore_quota && (options.to_stage == QueryProcessingStage::Complete)) quota = context->getQuota(); - storage->read(query_plan, table_lock, metadata_snapshot, limits, leaf_limits, std::move(quota), - required_columns, query_info, context, processing_stage, max_block_size, max_streams); + storage->read(query_plan, required_columns, metadata_snapshot, + query_info, *context, processing_stage, max_block_size, max_streams); + + /// Extend lifetime of context, table lock, storage. Set limits and quota. + auto adding_limits_and_quota = std::make_unique( + query_plan.getCurrentDataStream(), + storage, + std::move(table_lock), + limits, + leaf_limits, + std::move(quota), + std::move(context)); + adding_limits_and_quota->setStepDescription("Set limits and quota after reading from storage"); + query_plan.addStep(std::move(adding_limits_and_quota)); } else throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 03547170286..f923df43d76 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -100,6 +100,7 @@ public: void addStorageHolder(StoragePtr storage) { pipe.addStorageHolder(std::move(storage)); } void addQueryPlan(std::unique_ptr plan) { pipe.addQueryPlan(std::move(plan)); } void setLimits(const StreamLocalLimits & limits) { pipe.setLimits(limits); } + void setLeafLimits(const SizeLimits & limits) { pipe.setLeafLimits(limits); } void setQuota(const std::shared_ptr & quota) { pipe.setQuota(quota); } /// For compatibility with IBlockInputStream. diff --git a/src/Processors/QueryPlan/ReadFromStorageStep.cpp b/src/Processors/QueryPlan/ReadFromStorageStep.cpp index b085c177ad4..a88a396193b 100644 --- a/src/Processors/QueryPlan/ReadFromStorageStep.cpp +++ b/src/Processors/QueryPlan/ReadFromStorageStep.cpp @@ -12,35 +12,19 @@ namespace DB { ReadFromStorageStep::ReadFromStorageStep( - TableLockHolder table_lock_, - StorageMetadataPtr metadata_snapshot_, - StreamLocalLimits & limits_, - SizeLimits & leaf_limits_, - std::shared_ptr quota_, - StoragePtr storage_, - const Names & required_columns_, - const SelectQueryInfo & query_info_, - std::shared_ptr context_, - QueryProcessingStage::Enum processing_stage_, - size_t max_block_size_, - size_t max_streams_) - : table_lock(std::move(table_lock_)) - , metadata_snapshot(std::move(metadata_snapshot_)) - , limits(limits_) - , leaf_limits(leaf_limits_) - , quota(std::move(quota_)) - , storage(std::move(storage_)) - , required_columns(required_columns_) - , query_info(query_info_) - , context(std::move(context_)) - , processing_stage(processing_stage_) - , max_block_size(max_block_size_) - , max_streams(max_streams_) + StoragePtr storage, + const Names & required_columns, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processing_stage, + size_t max_block_size, + size_t max_streams) { /// Note: we read from storage in constructor of step because we don't know real header before reading. /// It will be fixed when storage return QueryPlanStep itself. - Pipe pipe = storage->read(required_columns, metadata_snapshot, query_info, *context, processing_stage, max_block_size, max_streams); + Pipe pipe = storage->read(required_columns, metadata_snapshot, query_info, context, processing_stage, max_block_size, max_streams); if (pipe.empty()) { @@ -83,29 +67,8 @@ ReadFromStorageStep::ReadFromStorageStep( pipeline = std::make_unique(); QueryPipelineProcessorsCollector collector(*pipeline, this); - /// Table lock is stored inside pipeline here. - pipeline->addTableLock(table_lock); - - pipe.setLimits(limits); - - /** - * Leaf size limits should be applied only for local processing of distributed queries. - * Such limits allow to control the read stage on leaf nodes and exclude the merging stage. - * Consider the case when distributed query needs to read from multiple shards. Then leaf - * limits will be applied on the shards only (including the root node) but will be ignored - * on the results merging stage. - */ - if (!storage->isRemote()) - pipe.setLeafLimits(leaf_limits); - - if (quota) - pipe.setQuota(quota); - pipeline->init(std::move(pipe)); - pipeline->addInterpreterContext(std::move(context)); - pipeline->addStorageHolder(std::move(storage)); - processors = collector.detachProcessors(); output_stream = DataStream{.header = pipeline->getHeader(), .has_single_port = pipeline->getNumStreams() == 1}; diff --git a/src/Processors/QueryPlan/ReadFromStorageStep.h b/src/Processors/QueryPlan/ReadFromStorageStep.h index 98cde63a863..59276d13081 100644 --- a/src/Processors/QueryPlan/ReadFromStorageStep.h +++ b/src/Processors/QueryPlan/ReadFromStorageStep.h @@ -23,15 +23,11 @@ class ReadFromStorageStep : public IQueryPlanStep { public: ReadFromStorageStep( - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - SizeLimits & leaf_limits, - std::shared_ptr quota, StoragePtr storage, const Names & required_columns, + const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, - std::shared_ptr context, + const Context & context, QueryProcessingStage::Enum processing_stage, size_t max_block_size, size_t max_streams); @@ -45,19 +41,6 @@ public: void describePipeline(FormatSettings & settings) const override; private: - TableLockHolder table_lock; - StorageMetadataPtr metadata_snapshot; - StreamLocalLimits limits; - SizeLimits leaf_limits; - std::shared_ptr quota; - - StoragePtr storage; - const Names & required_columns; - const SelectQueryInfo & query_info; - std::shared_ptr context; - QueryProcessingStage::Enum processing_stage; - size_t max_block_size; - size_t max_streams; QueryPipelinePtr pipeline; Processors processors; diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp index 73cd459fa5d..2a03d1fd82f 100644 --- a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -24,13 +25,15 @@ SettingQuotaAndLimitsStep::SettingQuotaAndLimitsStep( const DataStream & input_stream_, StoragePtr storage_, TableLockHolder table_lock_, - StreamLocalLimits limits_, + StreamLocalLimits & limits_, + SizeLimits & leaf_limits_, std::shared_ptr quota_, std::shared_ptr context_) : ITransformingStep(input_stream_, input_stream_.header, getTraits()) , storage(std::move(storage_)) , table_lock(std::move(table_lock_)) - , limits(std::move(limits_)) + , limits(limits_) + , leaf_limits(leaf_limits_) , quota(std::move(quota_)) , context(std::move(context_)) { @@ -43,6 +46,16 @@ void SettingQuotaAndLimitsStep::transformPipeline(QueryPipeline & pipeline) pipeline.setLimits(limits); + /** + * Leaf size limits should be applied only for local processing of distributed queries. + * Such limits allow to control the read stage on leaf nodes and exclude the merging stage. + * Consider the case when distributed query needs to read from multiple shards. Then leaf + * limits will be applied on the shards only (including the root node) but will be ignored + * on the results merging stage. + */ + if (!storage->isRemote()) + pipeline.setLeafLimits(leaf_limits); + if (quota) pipeline.setQuota(quota); diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h index 538d3c35b9d..7ec4cfa91c6 100644 --- a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h @@ -23,7 +23,8 @@ public: const DataStream & input_stream_, StoragePtr storage_, TableLockHolder table_lock_, - StreamLocalLimits limits_, + StreamLocalLimits & limits_, + SizeLimits & leaf_limits_, std::shared_ptr quota_, std::shared_ptr context_); @@ -35,6 +36,7 @@ private: StoragePtr storage; TableLockHolder table_lock; StreamLocalLimits limits; + SizeLimits leaf_limits; std::shared_ptr quota; std::shared_ptr context; }; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 50b36ced19c..73dabca0871 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -94,21 +94,16 @@ Pipe IStorage::read( void IStorage::read( QueryPlan & query_plan, - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - SizeLimits & leaf_limits, - std::shared_ptr quota, const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, - std::shared_ptr context, + const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { auto read_step = std::make_unique( - std::move(table_lock), std::move(metadata_snapshot), limits, leaf_limits, std::move(quota), shared_from_this(), - column_names, query_info, std::move(context), processed_stage, max_block_size, num_streams); + shared_from_this(), column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); read_step->setStepDescription("Read from " + getName()); query_plan.addStep(std::move(read_step)); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index dbd18c9558e..58a39497e38 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -285,17 +285,13 @@ public: /// Default implementation creates ReadFromStorageStep and uses usual read. virtual void read( QueryPlan & query_plan, - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - SizeLimits & leaf_limits, - std::shared_ptr quota, - const Names & column_names, - const SelectQueryInfo & query_info, - std::shared_ptr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - unsigned num_streams); + const Names & /*column_names*/, + const StorageMetadataPtr & /*metadata_snapshot*/, + const SelectQueryInfo & /*query_info*/, + const Context & /*context*/, + QueryProcessingStage::Enum /*processed_stage*/, + size_t /*max_block_size*/, + unsigned /*num_streams*/); /** Writes the data to a table. * Receives a description of the query, which can contain information about the data write method. diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index a7cba22bebf..e71228f2a23 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -55,52 +55,21 @@ Pipe StorageView::read( const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum /*processed_stage*/, - const size_t /*max_block_size*/, - const unsigned /*num_streams*/) + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) { - Pipes pipes; - - ASTPtr current_inner_query = metadata_snapshot->getSelectQuery().inner_query; - - if (query_info.view_query) - { - if (!query_info.view_query->as()) - throw Exception("Unexpected optimized VIEW query", ErrorCodes::LOGICAL_ERROR); - current_inner_query = query_info.view_query->clone(); - } - - InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names); - - auto pipeline = interpreter.execute().pipeline; - - /// It's expected that the columns read from storage are not constant. - /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. - pipeline.addSimpleTransform([](const Block & header) - { - return std::make_shared(header); - }); - - /// And also convert to expected structure. - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, metadata_snapshot->getSampleBlockForColumns( - column_names, getVirtuals(), getStorageID()), ConvertingTransform::MatchColumnsMode::Name); - }); - - return QueryPipeline::getPipe(std::move(pipeline)); + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); } void StorageView::read( QueryPlan & query_plan, - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - std::shared_ptr quota, const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, - std::shared_ptr context, + const Context & context, QueryProcessingStage::Enum /*processed_stage*/, const size_t /*max_block_size*/, const unsigned /*num_streams*/) @@ -114,7 +83,7 @@ void StorageView::read( current_inner_query = query_info.view_query->clone(); } - InterpreterSelectWithUnionQuery interpreter(current_inner_query, *context, {}, column_names); + InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names); interpreter.buildQueryPlan(query_plan); /// It's expected that the columns read from storage are not constant. @@ -128,17 +97,6 @@ void StorageView::read( auto converting = std::make_unique(query_plan.getCurrentDataStream(), header); converting->setStepDescription("Convert VIEW subquery result to VIEW table structure"); query_plan.addStep(std::move(converting)); - - /// Extend lifetime of context, table lock, storage. Set limits and quota. - auto adding_limits_and_quota = std::make_unique( - query_plan.getCurrentDataStream(), - shared_from_this(), - std::move(table_lock), - limits, - std::move(quota), - std::move(context)); - adding_limits_and_quota->setStepDescription("Set limits and quota for VIEW subquery"); - query_plan.addStep(std::move(adding_limits_and_quota)); } static ASTTableExpression * getFirstTableExpression(ASTSelectQuery & select_query) diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index 79155209ff8..1b43888baf3 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -32,13 +32,10 @@ public: void read( QueryPlan & query_plan, - TableLockHolder table_lock, - StorageMetadataPtr metadata_snapshot, - StreamLocalLimits & limits, - std::shared_ptr quota, const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, - std::shared_ptr context, + const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; From 2ffc2b692765b6c1883b1afb6386f1875379a1b2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 18 Sep 2020 15:10:59 +0300 Subject: [PATCH 013/114] Refactor IStorage::read with query plan. --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 073198c3a27..e63cf1fd78c 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1467,7 +1467,7 @@ void InterpreterSelectQuery::executeFetchColumns( limits, leaf_limits, std::move(quota), - std::move(context)); + context); adding_limits_and_quota->setStepDescription("Set limits and quota after reading from storage"); query_plan.addStep(std::move(adding_limits_and_quota)); } From b26f11c00caa3af96071fdb90f0bf039598b1ff6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 18 Sep 2020 17:16:53 +0300 Subject: [PATCH 014/114] Support StorageDistributed::read for QueryPlan. --- .../ClusterProxy/IStreamFactory.h | 7 +- .../ClusterProxy/SelectStreamFactory.cpp | 54 +++++------ .../ClusterProxy/SelectStreamFactory.h | 4 +- .../ClusterProxy/executeQuery.cpp | 48 +++++++++- src/Interpreters/ClusterProxy/executeQuery.h | 4 +- src/Interpreters/InterpreterSelectQuery.cpp | 52 +++++++++++ .../QueryPlan/ReadFromPreparedSource.cpp | 4 +- .../QueryPlan/ReadFromPreparedSource.h | 16 +++- .../QueryPlan/ReadFromStorageStep.cpp | 89 ------------------- .../QueryPlan/ReadFromStorageStep.h | 49 ---------- src/Processors/QueryPlan/UnionStep.cpp | 2 +- src/Processors/QueryPlan/UnionStep.h | 2 +- src/Processors/ya.make | 1 - src/Storages/IStorage.cpp | 8 +- src/Storages/StorageDistributed.cpp | 3 +- src/Storages/StorageDistributed.h | 10 +++ 16 files changed, 166 insertions(+), 187 deletions(-) delete mode 100644 src/Processors/QueryPlan/ReadFromStorageStep.cpp delete mode 100644 src/Processors/QueryPlan/ReadFromStorageStep.h diff --git a/src/Interpreters/ClusterProxy/IStreamFactory.h b/src/Interpreters/ClusterProxy/IStreamFactory.h index 8829dc38c93..80be585d15e 100644 --- a/src/Interpreters/ClusterProxy/IStreamFactory.h +++ b/src/Interpreters/ClusterProxy/IStreamFactory.h @@ -16,6 +16,9 @@ struct SelectQueryInfo; class Pipe; using Pipes = std::vector; +class QueryPlan; +using QueryPlanPtr = std::unique_ptr; + namespace ClusterProxy { @@ -31,7 +34,9 @@ public: const String & query, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, const SelectQueryInfo & query_info, - Pipes & res) = 0; + std::vector & res, + Pipes & remote_pipes, + Pipes & delayed_pipes) = 0; }; } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index ed7bd2cf71f..e1b008da66b 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -14,6 +14,8 @@ #include #include #include +#include + namespace ProfileEvents { @@ -69,38 +71,25 @@ SelectStreamFactory::SelectStreamFactory( namespace { -auto createLocalPipe( - const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage) +std::unique_ptr createLocalPlan( + const ASTPtr & query_ast, + const Block & header, + const Context & context, + QueryProcessingStage::Enum processed_stage) { checkStackSize(); - InterpreterSelectQuery interpreter(query_ast, context, SelectQueryOptions(processed_stage)); auto query_plan = std::make_unique(); + InterpreterSelectQuery interpreter(query_ast, context, SelectQueryOptions(processed_stage)); interpreter.buildQueryPlan(*query_plan); - auto pipeline = std::move(*query_plan->buildQueryPipeline()); - /// Avoid going it out-of-scope for EXPLAIN - pipeline.addQueryPlan(std::move(query_plan)); - - pipeline.addSimpleTransform([&](const Block & source_header) - { - return std::make_shared( - source_header, header, ConvertingTransform::MatchColumnsMode::Name, true); - }); - - /** Materialization is needed, since from remote servers the constants come materialized. - * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, - * And this is not allowed, since all code is based on the assumption that in the block stream all types are the same. - */ - - /* Now we don't need to materialize constants, because RemoteBlockInputStream will ignore constant and take it from header. - * So, streams from different threads will always have the same header. - */ - /// return std::make_shared(stream); - - pipeline.setMaxThreads(1); - return QueryPipeline::getPipe(std::move(pipeline)); + /// Convert header structure to expected. + /// Also we ignore constants from result and replace it with constants from header. + /// It is needed for functions like `now64()` or `randConstant()` because their values may be different. + auto converting = std::make_unique(query_plan->getCurrentDataStream(), header, true); + converting->setStepDescription("Convert block structure for query from local replica"); + query_plan->addStep(std::move(converting)); } String formattedAST(const ASTPtr & ast) @@ -119,7 +108,9 @@ void SelectStreamFactory::createForShard( const String &, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, const SelectQueryInfo &, - Pipes & pipes) + std::vector & plans, + Pipes & remote_pipes, + Pipes & delayed_pipes) { bool add_agg_info = processed_stage == QueryProcessingStage::WithMergeableState; bool add_totals = false; @@ -136,7 +127,7 @@ void SelectStreamFactory::createForShard( auto emplace_local_stream = [&]() { - pipes.emplace_back(createLocalPipe(modified_query_ast, header, context, processed_stage)); + plans.emplace_back(createLocalPlan(modified_query_ast, header, context, processed_stage)); }; String modified_query = formattedAST(modified_query_ast); @@ -149,7 +140,7 @@ void SelectStreamFactory::createForShard( if (!table_func_ptr) remote_query_executor->setMainTable(main_table); - pipes.emplace_back(createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes)); + remote_pipes.emplace_back(createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes)); }; const auto & settings = context.getSettingsRef(); @@ -276,7 +267,10 @@ void SelectStreamFactory::createForShard( } if (try_results.empty() || local_delay < max_remote_delay) - return createLocalPipe(modified_query_ast, header, context, stage); + { + auto plan = createLocalPlan(modified_query_ast, header, context, stage); + return QueryPipeline::getPipe(std::move(*plan->buildQueryPipeline())); + } else { std::vector connections; @@ -291,7 +285,7 @@ void SelectStreamFactory::createForShard( } }; - pipes.emplace_back(createDelayedPipe(header, lazily_create_stream, add_totals, add_extremes)); + delayed_pipes.emplace_back(createDelayedPipe(header, lazily_create_stream, add_totals, add_extremes)); } else emplace_remote_stream(); diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 80f72fd0024..c3dfc150d6d 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -39,7 +39,9 @@ public: const String & query, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, const SelectQueryInfo & query_info, - Pipes & pipes) override; + std::vector & res, + Pipes & remote_pipes, + Pipes & delayed_pipes) override; private: const Block header; diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 1ebd3009ff7..4618e6122be 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include namespace DB @@ -74,13 +77,16 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin return new_context; } -Pipe executeQuery( +void executeQuery( + QueryPlan & query_plan, IStreamFactory & stream_factory, const ClusterPtr & cluster, Poco::Logger * log, const ASTPtr & query_ast, const Context & context, const Settings & settings, const SelectQueryInfo & query_info) { assert(log); - Pipes res; + std::vector plans; + Pipes remote_pipes; + Pipes delayed_pipes; const std::string query = queryToString(query_ast); @@ -104,9 +110,43 @@ Pipe executeQuery( throttler = user_level_throttler; for (const auto & shard_info : cluster->getShardsInfo()) - stream_factory.createForShard(shard_info, query, query_ast, new_context, throttler, query_info, res); + stream_factory.createForShard(shard_info, query, query_ast, new_context, throttler, query_info, plans, remote_pipes, delayed_pipes); - return Pipe::unitePipes(std::move(res)); + if (!remote_pipes.empty()) + { + auto plan = std::make_unique(); + auto read_from_remote = std::make_unique(Pipe::unitePipes(std::move(remote_pipes))); + read_from_remote->setStepDescription("Read from remote replica"); + plan->addStep(std::move(read_from_remote)); + plans.emplace_back(std::move(plan)); + } + + if (!delayed_pipes.empty()) + { + auto plan = std::make_unique(); + auto read_from_remote = std::make_unique(Pipe::unitePipes(std::move(delayed_pipes))); + read_from_remote->setStepDescription("Read from delayed local replica"); + plan->addStep(std::move(read_from_remote)); + plans.emplace_back(std::move(plan)); + } + + if (plans.empty()) + return; + + if (plans.size() == 1) + { + query_plan = std::move(*plans.front()); + return; + } + + DataStreams input_streams; + input_streams.reserve(plans.size()); + for (auto & plan : plans) + input_streams.emplace_back(plan->getCurrentDataStream()); + + auto header = input_streams.front().header; + auto union_step = std::make_unique(std::move(input_streams), header); + query_plan.unitePlans(std::move(union_step), std::move(plans)); } } diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index f0d9539770d..1adc689c905 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -12,6 +12,7 @@ class Cluster; struct SelectQueryInfo; class Pipe; +class QueryPlan; namespace ClusterProxy { @@ -25,7 +26,8 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin /// Execute a distributed query, creating a vector of BlockInputStreams, from which the result can be read. /// `stream_factory` object encapsulates the logic of creating streams for a different type of query /// (currently SELECT, DESCRIBE). -Pipe executeQuery( +void executeQuery( + QueryPlan & query_plan, IStreamFactory & stream_factory, const ClusterPtr & cluster, Poco::Logger * log, const ASTPtr & query_ast, const Context & context, const Settings & settings, const SelectQueryInfo & query_info); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index e63cf1fd78c..30da2e7d6f2 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -31,9 +31,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -1102,6 +1104,48 @@ static StreamLocalLimits getLimitsForStorage(const Settings & settings, const Se return limits; } +static void addEmptySource(QueryPlan & query_plan, const Block & header, SelectQueryInfo & query_info) +{ + Pipe pipe(std::make_shared(header)); + + if (query_info.prewhere_info) + { + if (query_info.prewhere_info->alias_actions) + { + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, query_info.prewhere_info->alias_actions); + }); + } + + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, + query_info.prewhere_info->prewhere_actions, + query_info.prewhere_info->prewhere_column_name, + query_info.prewhere_info->remove_prewhere_column); + }); + + // To remove additional columns + // In some cases, we did not read any marks so that the pipeline.streams is empty + // Thus, some columns in prewhere are not removed as expected + // This leads to mismatched header in distributed table + if (query_info.prewhere_info->remove_columns_actions) + { + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, query_info.prewhere_info->remove_columns_actions); + }); + } + } + + auto read_from_pipe = std::make_unique(std::move(pipe)); + read_from_pipe->setStepDescription("Read from NullSource"); + query_plan.addStep(std::move(read_from_pipe)); +} + void InterpreterSelectQuery::executeFetchColumns( QueryProcessingStage::Enum processing_stage, QueryPlan & query_plan, const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere) @@ -1459,6 +1503,14 @@ void InterpreterSelectQuery::executeFetchColumns( storage->read(query_plan, required_columns, metadata_snapshot, query_info, *context, processing_stage, max_block_size, max_streams); + /// Create step which reads from empty source if storage has no data. + if (!query_plan.isInitialized()) + { + auto header = metadata_snapshot->getSampleBlockForColumns( + required_columns, storage->getVirtuals(), storage->getStorageID()); + addEmptySource(query_plan, header, query_info); + } + /// Extend lifetime of context, table lock, storage. Set limits and quota. auto adding_limits_and_quota = std::make_unique( query_plan.getCurrentDataStream(), diff --git a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp index 6f0d1693ce0..dcfe609e070 100644 --- a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp +++ b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp @@ -14,7 +14,9 @@ ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_); + explicit ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_ = nullptr); - String getName() const override { return "ReadNothing"; } + String getName() const override { return "ReadFromPreparedSource"; } void initializePipeline(QueryPipeline & pipeline) override; @@ -20,4 +20,16 @@ private: std::shared_ptr context; }; +class ReadFromStorageStep : public ReadFromPreparedSource +{ +public: + ReadFromStorageStep(Pipe pipe, String storage_name) + : ReadFromPreparedSource(std::move(pipe)) + { + setStepDescription(storage_name); + } + + String getName() const override { return "ReadFromStorage"; } +}; + } diff --git a/src/Processors/QueryPlan/ReadFromStorageStep.cpp b/src/Processors/QueryPlan/ReadFromStorageStep.cpp deleted file mode 100644 index a88a396193b..00000000000 --- a/src/Processors/QueryPlan/ReadFromStorageStep.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -ReadFromStorageStep::ReadFromStorageStep( - StoragePtr storage, - const Names & required_columns, - const StorageMetadataPtr & metadata_snapshot, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum processing_stage, - size_t max_block_size, - size_t max_streams) -{ - /// Note: we read from storage in constructor of step because we don't know real header before reading. - /// It will be fixed when storage return QueryPlanStep itself. - - Pipe pipe = storage->read(required_columns, metadata_snapshot, query_info, context, processing_stage, max_block_size, max_streams); - - if (pipe.empty()) - { - pipe = Pipe(std::make_shared(metadata_snapshot->getSampleBlockForColumns(required_columns, storage->getVirtuals(), storage->getStorageID()))); - - if (query_info.prewhere_info) - { - if (query_info.prewhere_info->alias_actions) - { - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, query_info.prewhere_info->alias_actions); - }); - } - - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, - query_info.prewhere_info->prewhere_actions, - query_info.prewhere_info->prewhere_column_name, - query_info.prewhere_info->remove_prewhere_column); - }); - - // To remove additional columns - // In some cases, we did not read any marks so that the pipeline.streams is empty - // Thus, some columns in prewhere are not removed as expected - // This leads to mismatched header in distributed table - if (query_info.prewhere_info->remove_columns_actions) - { - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, query_info.prewhere_info->remove_columns_actions); - }); - } - } - } - - pipeline = std::make_unique(); - QueryPipelineProcessorsCollector collector(*pipeline, this); - - pipeline->init(std::move(pipe)); - - processors = collector.detachProcessors(); - - output_stream = DataStream{.header = pipeline->getHeader(), .has_single_port = pipeline->getNumStreams() == 1}; -} - -ReadFromStorageStep::~ReadFromStorageStep() = default; - -QueryPipelinePtr ReadFromStorageStep::updatePipeline(QueryPipelines) -{ - return std::move(pipeline); -} - -void ReadFromStorageStep::describePipeline(FormatSettings & settings) const -{ - IQueryPlanStep::describePipeline(processors, settings); -} - -} diff --git a/src/Processors/QueryPlan/ReadFromStorageStep.h b/src/Processors/QueryPlan/ReadFromStorageStep.h deleted file mode 100644 index 59276d13081..00000000000 --- a/src/Processors/QueryPlan/ReadFromStorageStep.h +++ /dev/null @@ -1,49 +0,0 @@ -#include -#include -#include -#include - -namespace DB -{ - -class IStorage; -using StoragePtr = std::shared_ptr; - -struct StorageInMemoryMetadata; -using StorageMetadataPtr = std::shared_ptr; - -struct SelectQueryInfo; - -struct PrewhereInfo; - -class EnabledQuota; - -/// Reads from storage. -class ReadFromStorageStep : public IQueryPlanStep -{ -public: - ReadFromStorageStep( - StoragePtr storage, - const Names & required_columns, - const StorageMetadataPtr & metadata_snapshot, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum processing_stage, - size_t max_block_size, - size_t max_streams); - - ~ReadFromStorageStep() override; - - String getName() const override { return "ReadFromStorage"; } - - QueryPipelinePtr updatePipeline(QueryPipelines) override; - - void describePipeline(FormatSettings & settings) const override; - -private: - - QueryPipelinePtr pipeline; - Processors processors; -}; - -} diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index 1e74046b071..630ff53f47d 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -30,7 +30,7 @@ QueryPipelinePtr UnionStep::updatePipeline(QueryPipelines pipelines) return pipeline; } - *pipeline = QueryPipeline::unitePipelines(std::move(pipelines), output_stream->header ,max_threads); + *pipeline = QueryPipeline::unitePipelines(std::move(pipelines), output_stream->header, max_threads); processors = collector.detachProcessors(); return pipeline; diff --git a/src/Processors/QueryPlan/UnionStep.h b/src/Processors/QueryPlan/UnionStep.h index 9e00e24279b..e2e1f2c9efa 100644 --- a/src/Processors/QueryPlan/UnionStep.h +++ b/src/Processors/QueryPlan/UnionStep.h @@ -9,7 +9,7 @@ class UnionStep : public IQueryPlanStep { public: /// max_threads is used to limit the number of threads for result pipeline. - UnionStep(DataStreams input_streams_, Block result_header, size_t max_threads_); + UnionStep(DataStreams input_streams_, Block result_header, size_t max_threads_ = 0); String getName() const override { return "Union"; } diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 08de142479b..cd18ea3deb0 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -113,7 +113,6 @@ SRCS( QueryPlan/PartialSortingStep.cpp QueryPlan/QueryPlan.cpp QueryPlan/ReadFromPreparedSource.cpp - QueryPlan/ReadFromStorageStep.cpp QueryPlan/ReadNothingStep.cpp QueryPlan/RollupStep.cpp QueryPlan/SettingQuotaAndLimitsStep.cpp diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 73dabca0871..fab3018911f 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include @@ -102,10 +102,8 @@ void IStorage::read( size_t max_block_size, unsigned num_streams) { - auto read_step = std::make_unique( - shared_from_this(), column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - - read_step->setStepDescription("Read from " + getName()); + auto pipe = read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + auto read_step = std::make_unique(std::move(pipe), getName()); query_plan.addStep(std::move(read_step)); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 596681002ab..268d1b9287f 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -494,6 +494,7 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(const Con } Pipe StorageDistributed::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -537,7 +538,7 @@ Pipe StorageDistributed::read( : ClusterProxy::SelectStreamFactory( header, processed_stage, StorageID{remote_database, remote_table}, scalars, has_virtual_shard_num_column, context.getExternalTables()); - return ClusterProxy::executeQuery(select_stream_factory, cluster, log, + ClusterProxy::executeQuery(query_plan, select_stream_factory, cluster, log, modified_query_ast, context, context.getSettingsRef(), query_info); } diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 7e4e9f56ab4..c4e213d2276 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -77,6 +77,16 @@ public: size_t max_block_size, unsigned num_streams) override; + Pipe StorageDistributed::read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t /*max_block_size*/, + const unsigned /*num_streams*/) + bool supportsParallelInsert() const override { return true; } BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; From 322e63971d4592ec7769975f6830a1343e66b687 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 18 Sep 2020 21:47:32 +0300 Subject: [PATCH 015/114] Update MergeTreeIndexConditionBloomFilter.cpp --- src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 3a7f7408f7c..b78ed3adf45 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -232,7 +232,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseFunction(const ASTPtr & node, B if (const auto * function = node->as()) { const ASTs & arguments = function->arguments->children; - for (auto arg : arguments) + for (const auto & arg : arguments) { if (traverseFunction(arg, block_with_constants, out)) maybe_useful = true; From 3122455b4d1c26dfe5832aaa99429fa677d21197 Mon Sep 17 00:00:00 2001 From: "philip.han" Date: Sun, 20 Sep 2020 19:04:33 +0900 Subject: [PATCH 016/114] BloomFilter is used just in specific cases of indexof() --- .../MergeTreeIndexConditionBloomFilter.cpp | 73 +++++++++++++++---- .../MergeTreeIndexConditionBloomFilter.h | 4 +- .../00945_bloom_filter_index.reference | 36 +++++++++ .../0_stateless/00945_bloom_filter_index.sql | 51 +++++++++++++ 4 files changed, 148 insertions(+), 16 deletions(-) mode change 100755 => 100644 tests/queries/0_stateless/00945_bloom_filter_index.sql diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index b78ed3adf45..69d63d0e80d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -222,10 +223,10 @@ bool MergeTreeIndexConditionBloomFilter::traverseAtomAST(const ASTPtr & node, Bl } } - return traverseFunction(node, block_with_constants, out); + return traverseFunction(node, block_with_constants, out, nullptr); } -bool MergeTreeIndexConditionBloomFilter::traverseFunction(const ASTPtr & node, Block & block_with_constants, RPNElement & out) +bool MergeTreeIndexConditionBloomFilter::traverseFunction(const ASTPtr & node, Block & block_with_constants, RPNElement & out, const ASTPtr & parent) { bool maybe_useful = false; @@ -234,7 +235,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseFunction(const ASTPtr & node, B const ASTs & arguments = function->arguments->children; for (const auto & arg : arguments) { - if (traverseFunction(arg, block_with_constants, out)) + if (traverseFunction(arg, block_with_constants, out, node)) maybe_useful = true; } @@ -255,12 +256,12 @@ bool MergeTreeIndexConditionBloomFilter::traverseFunction(const ASTPtr & node, B DataTypePtr const_type; if (KeyCondition::getConstant(arguments[1], block_with_constants, const_value, const_type)) { - if (traverseASTEquals(function->name, arguments[0], const_type, const_value, out)) + if (traverseASTEquals(function->name, arguments[0], const_type, const_value, out, parent)) maybe_useful = true; } else if (KeyCondition::getConstant(arguments[0], block_with_constants, const_value, const_type)) { - if (traverseASTEquals(function->name, arguments[1], const_type, const_value, out)) + if (traverseASTEquals(function->name, arguments[1], const_type, const_value, out, parent)) maybe_useful = true; } } @@ -323,8 +324,50 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTIn( return false; } +static bool indexOfCanUseBloomFilter(const ASTPtr & parent) +{ + if (!parent) + return true; + + if (const auto * function = parent->as()) + { + if (function->name == "arrayElement") + { + return true; + } + else if (function->name == "equals" + || function->name == "greater" || function->name == "greaterOrEquals" + || function->name == "less" || function->name == "lessOrEquals") + { + if (function->arguments->children.size() != 2) + return false; + + if (const ASTLiteral * left = function->arguments->children[0]->as()) + { + if (function->name == "equals" && left->value.get() != 0) + return true; + else if (function->name == "less" && left->value.get() >= 0) + return true; + else if (function->name == "lessOrEquals" && left->value.get() > 0) + return true; + } + else if (const ASTLiteral * right = function->arguments->children[1]->as()) + { + if (function->name == "equals" && right->value.get() != 0) + return true; + else if (function->name == "greater" && right->value.get() >= 0) + return true; + else if (function->name == "greaterOrEquals" && right->value.get() > 0) + return true; + } + } + } + + return false; +} + bool MergeTreeIndexConditionBloomFilter::traverseASTEquals( - const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out) + const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out, const ASTPtr & parent) { if (header.has(key_ast->getColumnName())) { @@ -334,19 +377,21 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTEquals( if (function_name == "has" || function_name == "indexOf") { - out.function = RPNElement::FUNCTION_HAS; - if (!array_type) - throw Exception("First argument for function has must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("First argument for function " + function_name + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - const DataTypePtr actual_type = BloomFilter::getPrimitiveType(array_type->getNestedType()); - Field converted_field = convertFieldToType(value_field, *actual_type, value_type.get()); - out.predicate.emplace_back(std::make_pair(position, BloomFilterHash::hashWithField(actual_type.get(), converted_field))); + if (function_name == "has" || indexOfCanUseBloomFilter(parent)) + { + out.function = RPNElement::FUNCTION_HAS; + const DataTypePtr actual_type = BloomFilter::getPrimitiveType(array_type->getNestedType()); + Field converted_field = convertFieldToType(value_field, *actual_type, value_type.get()); + out.predicate.emplace_back(std::make_pair(position, BloomFilterHash::hashWithField(actual_type.get(), converted_field))); + } } else { if (array_type) - throw Exception("An array type of bloom_filter supports only has() function.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("An array type of bloom_filter supports only has() and indexOf() function.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); out.function = function_name == "equals" ? RPNElement::FUNCTION_EQUALS : RPNElement::FUNCTION_NOT_EQUALS; const DataTypePtr actual_type = BloomFilter::getPrimitiveType(index_type); @@ -374,7 +419,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTEquals( const DataTypes & subtypes = value_tuple_data_type->getElements(); for (size_t index = 0; index < tuple.size(); ++index) - match_with_subtype |= traverseASTEquals(function_name, arguments[index], subtypes[index], tuple[index], out); + match_with_subtype |= traverseASTEquals(function_name, arguments[index], subtypes[index], tuple[index], out, key_ast); return match_with_subtype; } diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h index 40299cb7a43..34fb45c86a5 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h @@ -67,7 +67,7 @@ private: bool traverseAtomAST(const ASTPtr & node, Block & block_with_constants, RPNElement & out); - bool traverseFunction(const ASTPtr & node, Block & block_with_constants, RPNElement & out); + bool traverseFunction(const ASTPtr & node, Block & block_with_constants, RPNElement & out, const ASTPtr & parent); bool traverseASTIn(const String & function_name, const ASTPtr & key_ast, const SetPtr & prepared_set, RPNElement & out); @@ -75,7 +75,7 @@ private: const String & function_name, const ASTPtr & key_ast, const DataTypePtr & type, const ColumnPtr & column, RPNElement & out); bool traverseASTEquals( - const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out); + const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out, const ASTPtr & parent); }; } diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.reference b/tests/queries/0_stateless/00945_bloom_filter_index.reference index a00ae5f2d5b..184aafdd568 100755 --- a/tests/queries/0_stateless/00945_bloom_filter_index.reference +++ b/tests/queries/0_stateless/00945_bloom_filter_index.reference @@ -178,3 +178,39 @@ 5000 5000 5000 +2 +1 +2 +1 +1 +2 +1 +2 +1 +2 +2 +1 +1 +2 +2 +1 +2 +1 +2 +2 +1 +1 +2 +1 +2 +1 +1 +1 +2 +1 +2 +1 +1 +1 value1 +1 value2 +2 value3 diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql old mode 100755 new mode 100644 index 083caba710b..82321a75c67 --- a/tests/queries/0_stateless/00945_bloom_filter_index.sql +++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql @@ -302,3 +302,54 @@ CREATE TABLE bloom_filter_array_offsets_i (order_key int, i Array(int), INDEX id INSERT INTO bloom_filter_array_offsets_i SELECT number AS i, if(i%2, [99999], []) FROM system.numbers LIMIT 10000; SELECT count() FROM bloom_filter_array_offsets_i WHERE has(i, 99999); DROP TABLE IF EXISTS bloom_filter_array_offsets_i; + +DROP TABLE IF EXISTS test_bf_indexOf; +CREATE TABLE test_bf_indexOf ( `id` int, `ary` Array(LowCardinality(Nullable(String))), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 1; +INSERT INTO test_bf_indexOf VALUES (1, ['value1', 'value2']); +INSERT INTO test_bf_indexOf VALUES (2, ['value3']); + +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') = 0 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') = 1 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value2') = 0 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value2') = 2 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') = 0 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') = 1 ORDER BY id FORMAT TSV; + +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') != 0 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') != 1 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value2') != 0 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value2') != 2 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') != 0 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') != 1 ORDER BY id FORMAT TSV; + +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') = 2 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value3') = 2 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') = 1 OR indexOf(ary, 'value3') = 1 ORDER BY id FORMAT TSV; + +SELECT id FROM test_bf_indexOf WHERE not(indexOf(ary, 'value1')) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE not(indexOf(ary, 'value1') == 0) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE not(indexOf(ary, 'value1') == 1) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE not(indexOf(ary, 'value1') == 2) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') in (0) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') in (1) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') in (2) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') not in (0) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') not in (1) ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') not in (2) ORDER BY id FORMAT TSV; + +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') > 0 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE 0 < indexOf(ary, 'value1') ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') >= 0 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE 0 <= indexOf(ary, 'value1') ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') > 1 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE 1 < indexOf(ary, 'value1') ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') >= 1 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE 1 <= indexOf(ary, 'value1') ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE indexOf(ary, 'value1') >= 2 ORDER BY id FORMAT TSV; +SELECT id FROM test_bf_indexOf WHERE 2 <= indexOf(ary, 'value1') ORDER BY id FORMAT TSV; + +SELECT id, ary[indexOf(ary, 'value1')] FROM test_bf_indexOf WHERE ary[indexOf(ary, 'value1')] = 'value1' ORDER BY id FORMAT TSV; +SELECT id, ary[indexOf(ary, 'value2')] FROM test_bf_indexOf WHERE ary[indexOf(ary, 'value2')] = 'value2' ORDER BY id FORMAT TSV; +SELECT id, ary[indexOf(ary, 'value3')] FROM test_bf_indexOf WHERE ary[indexOf(ary, 'value3')] = 'value3' ORDER BY id FORMAT TSV; + +DROP TABLE IF EXISTS test_bf_indexOf; From 2045bc9f8c2f2e64d9669fd5c78a72e0fbacb89b Mon Sep 17 00:00:00 2001 From: "philip.han" Date: Sun, 20 Sep 2020 19:06:46 +0900 Subject: [PATCH 017/114] Fix performance_test for bloom_filter --- tests/performance/bloom_filter.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/performance/bloom_filter.xml b/tests/performance/bloom_filter.xml index d1881cde56e..00bdd594738 100644 --- a/tests/performance/bloom_filter.xml +++ b/tests/performance/bloom_filter.xml @@ -1,12 +1,13 @@ DROP TABLE IF EXISTS test_bf CREATE TABLE test_bf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 8192) ENGINE = MergeTree() ORDER BY id - CREATE TABLE test_bf_indexOf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id + CREATE TABLE test_bf_indexOf (`id` int, `ary` Array(LowCardinality(Nullable(String))), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SYSTEM STOP MERGES INSERT INTO test_bf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) INSERT INTO test_bf_indexOf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) + SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') = 2 SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0 SELECT count() FROM test_bf_indexOf WHERE ary[indexOf(ary, '1')] = '1' From 576ffadb170747665dc3a7b9d1b6918e58358096 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 25 Sep 2020 16:19:26 +0300 Subject: [PATCH 018/114] Fix explain for ISourceStep. --- .../ClusterProxy/SelectStreamFactory.cpp | 2 ++ .../ClusterProxy/SelectStreamFactory.h | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 6 +++--- src/Interpreters/InterpreterSelectQuery.h | 2 ++ src/Processors/QueryPlan/ISourceStep.cpp | 3 ++- src/Processors/QueryPlan/ISourceStep.h | 2 +- .../QueryPlan/ReadFromPreparedSource.cpp | 5 ++++- src/Processors/QueryPlan/ReadFromPreparedSource.h | 4 ++-- src/Storages/IStorage.cpp | 13 +++++++++++-- src/Storages/StorageDistributed.cpp | 14 ++++++++++++++ src/Storages/StorageDistributed.h | 2 +- 11 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index e1b008da66b..2e6587a32f6 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -90,6 +90,8 @@ std::unique_ptr createLocalPlan( auto converting = std::make_unique(query_plan->getCurrentDataStream(), header, true); converting->setStepDescription("Convert block structure for query from local replica"); query_plan->addStep(std::move(converting)); + + return query_plan; } String formattedAST(const ASTPtr & ast) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index c3dfc150d6d..9b57b92ed50 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -39,7 +39,7 @@ public: const String & query, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, const SelectQueryInfo & query_info, - std::vector & res, + std::vector & plans, Pipes & remote_pipes, Pipes & delayed_pipes) override; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 228ccfae39e..5a6bb5770cf 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1107,9 +1107,9 @@ static StreamLocalLimits getLimitsForStorage(const Settings & settings, const Se return limits; } -static void addEmptySource(QueryPlan & query_plan, const Block & header, SelectQueryInfo & query_info) +void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, const Block & source_header, const SelectQueryInfo & query_info) { - Pipe pipe(std::make_shared(header)); + Pipe pipe(std::make_shared(source_header)); if (query_info.prewhere_info) { @@ -1511,7 +1511,7 @@ void InterpreterSelectQuery::executeFetchColumns( { auto header = metadata_snapshot->getSampleBlockForColumns( required_columns, storage->getVirtuals(), storage->getStorageID()); - addEmptySource(query_plan, header, query_info); + addEmptySourceToQueryPlan(query_plan, header, query_info); } /// Extend lifetime of context, table lock, storage. Set limits and quota. diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 455b1a1e623..5ed938af9f3 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -94,6 +94,8 @@ public: const SelectQueryInfo & getQueryInfo() const { return query_info; } + static void addEmptySourceToQueryPlan(QueryPlan & query_plan, const Block & source_header, const SelectQueryInfo & query_info); + private: InterpreterSelectQuery( const ASTPtr & query_ptr_, diff --git a/src/Processors/QueryPlan/ISourceStep.cpp b/src/Processors/QueryPlan/ISourceStep.cpp index cf68104f18c..1796f033896 100644 --- a/src/Processors/QueryPlan/ISourceStep.cpp +++ b/src/Processors/QueryPlan/ISourceStep.cpp @@ -14,7 +14,8 @@ QueryPipelinePtr ISourceStep::updatePipeline(QueryPipelines) auto pipeline = std::make_unique(); QueryPipelineProcessorsCollector collector(*pipeline, this); initializePipeline(*pipeline); - processors = collector.detachProcessors(); + auto added_processors = collector.detachProcessors(); + processors.insert(processors.end(), added_processors.begin(), added_processors.end()); return pipeline; } diff --git a/src/Processors/QueryPlan/ISourceStep.h b/src/Processors/QueryPlan/ISourceStep.h index 54bc19957f4..fdb3dd566cb 100644 --- a/src/Processors/QueryPlan/ISourceStep.h +++ b/src/Processors/QueryPlan/ISourceStep.h @@ -16,7 +16,7 @@ public: void describePipeline(FormatSettings & settings) const override; -private: +protected: /// We collect processors got after pipeline transformation. Processors processors; }; diff --git a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp index dcfe609e070..2d4b759b637 100644 --- a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp +++ b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp @@ -5,7 +5,7 @@ namespace DB { ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_) - : ISourceStep(DataStream{.header = pipe_.getHeader(), .has_single_port = true}) + : ISourceStep(DataStream{.header = pipe_.getHeader()}) , pipe(std::move(pipe_)) , context(std::move(context_)) { @@ -13,6 +13,9 @@ ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr #include #include +#include namespace DB @@ -103,8 +104,16 @@ void IStorage::read( unsigned num_streams) { auto pipe = read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - auto read_step = std::make_unique(std::move(pipe), getName()); - query_plan.addStep(std::move(read_step)); + if (pipe.empty()) + { + auto header = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + InterpreterSelectQuery::addEmptySourceToQueryPlan(query_plan, header, query_info); + } + else + { + auto read_step = std::make_unique(std::move(pipe), getName()); + query_plan.addStep(std::move(read_step)); + } } Pipe IStorage::alterPartition( diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index b27b7794932..6e7d7ea41bd 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -495,6 +495,20 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(const Con } Pipe StorageDistributed::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); +} + +void StorageDistributed::read( QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 3d02d196c7e..df0ca05e1c6 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -77,7 +77,7 @@ public: size_t max_block_size, unsigned num_streams) override; - Pipe StorageDistributed::read( + void read( QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, From 5ac6bc071dc5066cd4d9759769b03898e6a3ec06 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 29 Sep 2020 19:21:58 +0300 Subject: [PATCH 019/114] QueryPlan for StorageBuffer and StorageMaterializedView read. --- src/Processors/QueryPlan/AddingMissedStep.cpp | 45 +++++ src/Processors/QueryPlan/AddingMissedStep.h | 28 ++++ .../QueryPlan/SettingQuotaAndLimitsStep.cpp | 7 +- src/Processors/ya.make | 1 + src/Storages/StorageBuffer.cpp | 158 ++++++++++++------ src/Storages/StorageBuffer.h | 10 ++ src/Storages/StorageMaterializedView.cpp | 20 ++- src/Storages/StorageMaterializedView.h | 10 ++ ...84_shard_distributed_group_by_no_merge.sql | 4 +- 9 files changed, 226 insertions(+), 57 deletions(-) create mode 100644 src/Processors/QueryPlan/AddingMissedStep.cpp create mode 100644 src/Processors/QueryPlan/AddingMissedStep.h diff --git a/src/Processors/QueryPlan/AddingMissedStep.cpp b/src/Processors/QueryPlan/AddingMissedStep.cpp new file mode 100644 index 00000000000..f2d06c033b1 --- /dev/null +++ b/src/Processors/QueryPlan/AddingMissedStep.cpp @@ -0,0 +1,45 @@ +#include +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +AddingMissedStep::AddingMissedStep( + const DataStream & input_stream_, + Block result_header_, + const ColumnDefaults & column_defaults_, + const Context & context_) + : ITransformingStep(input_stream_, result_header_, getTraits()) + , column_defaults(column_defaults_) + , context(context_) +{ + updateDistinctColumns(output_stream->header, output_stream->distinct_columns); +} + +void AddingMissedStep::transformPipeline(QueryPipeline & pipeline) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, output_stream->header, column_defaults, context); + }); +} + +} diff --git a/src/Processors/QueryPlan/AddingMissedStep.h b/src/Processors/QueryPlan/AddingMissedStep.h new file mode 100644 index 00000000000..77075a410a5 --- /dev/null +++ b/src/Processors/QueryPlan/AddingMissedStep.h @@ -0,0 +1,28 @@ +#pragma once +#include + +namespace DB +{ + +struct ColumnDefault; +using ColumnDefaults = std::unordered_map; + +/// Convert one block structure to another. See ConvertingTransform. +class AddingMissedStep : public ITransformingStep +{ +public: + AddingMissedStep(const DataStream & input_stream_, + Block result_header_, + const ColumnDefaults & column_defaults_, + const Context & context_); + + String getName() const override { return "AddingMissed"; } + + void transformPipeline(QueryPipeline & pipeline) override; + +private: + const ColumnDefaults column_defaults; + const Context & context; +}; + +} diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp index 2a03d1fd82f..5b05ad77d6c 100644 --- a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp @@ -59,8 +59,11 @@ void SettingQuotaAndLimitsStep::transformPipeline(QueryPipeline & pipeline) if (quota) pipeline.setQuota(quota); - pipeline.addInterpreterContext(std::move(context)); - pipeline.addStorageHolder(std::move(storage)); + if (context) + pipeline.addInterpreterContext(std::move(context)); + + if (storage) + pipeline.addStorageHolder(std::move(storage)); } } diff --git a/src/Processors/ya.make b/src/Processors/ya.make index cd18ea3deb0..b5afc1ada3a 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -89,6 +89,7 @@ SRCS( printPipeline.cpp QueryPipeline.cpp QueryPlan/AddingDelayedSourceStep.cpp + QueryPlan/AddingMissedStep.cpp QueryPlan/AggregatingStep.cpp QueryPlan/ArrayJoinStep.cpp QueryPlan/ConvertingStep.cpp diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 14f188275e5..162463a4e33 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -22,10 +22,13 @@ #include #include #include -#include +#include #include #include #include +#include +#include +#include namespace ProfileEvents @@ -147,6 +150,21 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage(const Context Pipe StorageBuffer::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); +} + +void StorageBuffer::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -155,8 +173,6 @@ Pipe StorageBuffer::read( size_t max_block_size, unsigned num_streams) { - Pipe pipe_from_dst; - if (destination_id) { auto destination = DatabaseCatalog::instance().getTable(destination_id, context); @@ -182,8 +198,8 @@ Pipe StorageBuffer::read( query_info.input_order_info = query_info.order_optimizer->getInputOrder(destination, destination_metadata_snapshot); /// The destination table has the same structure of the requested columns and we can simply read blocks from there. - pipe_from_dst = destination->read( - column_names, destination_metadata_snapshot, query_info, + destination->read( + query_plan, column_names, destination_metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); } else @@ -217,25 +233,45 @@ Pipe StorageBuffer::read( } else { - pipe_from_dst = destination->read( - columns_intersection, destination_metadata_snapshot, query_info, - context, processed_stage, max_block_size, num_streams); + destination->read( + query_plan, columns_intersection, destination_metadata_snapshot, query_info, + context, processed_stage, max_block_size, num_streams); - pipe_from_dst.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, header_after_adding_defaults, + auto adding_missed = std::make_unique( + query_plan.getCurrentDataStream(), + header_after_adding_defaults, metadata_snapshot->getColumns().getDefaults(), context); - }); - pipe_from_dst.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared( - stream_header, header, ConvertingTransform::MatchColumnsMode::Name); - }); + adding_missed->setStepDescription("Add columns missing in destination table"); + query_plan.addStep(std::move(adding_missed)); + + auto converting = std::make_unique( + query_plan.getCurrentDataStream(), + header); + + converting->setStepDescription("Convert destination table columns to Buffer table structure"); + query_plan.addStep(std::move(converting)); } } - pipe_from_dst.addTableLock(destination_lock); + if (query_plan.isInitialized()) + { + StreamLocalLimits limits; + SizeLimits leaf_limits; + + /// Add table lock for destination table. + auto adding_limits_and_quota = std::make_unique( + query_plan.getCurrentDataStream(), + destination, + std::move(destination_lock), + limits, + leaf_limits, + nullptr, + nullptr); + + adding_limits_and_quota->setStepDescription("Lock destination table for Buffer"); + query_plan.addStep(std::move(adding_limits_and_quota)); + } } Pipe pipe_from_buffers; @@ -248,49 +284,73 @@ Pipe StorageBuffer::read( pipe_from_buffers = Pipe::unitePipes(std::move(pipes_from_buffers)); } - /// Convert pipes from table to structure from buffer. - if (!pipe_from_buffers.empty() && !pipe_from_dst.empty() - && !blocksHaveEqualStructure(pipe_from_buffers.getHeader(), pipe_from_dst.getHeader())) - { - pipe_from_dst.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, - pipe_from_buffers.getHeader(), - ConvertingTransform::MatchColumnsMode::Name); - }); - } + if (pipe_from_buffers.empty()) + return; + + QueryPlan buffers_plan; /** If the sources from the table were processed before some non-initial stage of query execution, * then sources from the buffers must also be wrapped in the processing pipeline before the same stage. */ if (processed_stage > QueryProcessingStage::FetchColumns) - pipe_from_buffers = QueryPipeline::getPipe( - InterpreterSelectQuery(query_info.query, context, std::move(pipe_from_buffers), - SelectQueryOptions(processed_stage)).execute().pipeline); - - if (query_info.prewhere_info) { - pipe_from_buffers.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, query_info.prewhere_info->prewhere_actions, - query_info.prewhere_info->prewhere_column_name, query_info.prewhere_info->remove_prewhere_column); - }); - - if (query_info.prewhere_info->alias_actions) + auto interpreter = InterpreterSelectQuery( + query_info.query, context, std::move(pipe_from_buffers), + SelectQueryOptions(processed_stage)); + interpreter.buildQueryPlan(buffers_plan); + } + else + { + if (query_info.prewhere_info) { pipe_from_buffers.addSimpleTransform([&](const Block & header) { - return std::make_shared(header, query_info.prewhere_info->alias_actions); + return std::make_shared( + header, query_info.prewhere_info->prewhere_actions, + query_info.prewhere_info->prewhere_column_name, query_info.prewhere_info->remove_prewhere_column); }); + + if (query_info.prewhere_info->alias_actions) + { + pipe_from_buffers.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, query_info.prewhere_info->alias_actions); + }); + } } + + auto read_from_buffers = std::make_unique(std::move(pipe_from_buffers)); + read_from_buffers->setStepDescription("Read from buffers of Buffer table"); + buffers_plan.addStep(std::move(read_from_buffers)); } - Pipes pipes; - pipes.emplace_back(std::move(pipe_from_dst)); - pipes.emplace_back(std::move(pipe_from_buffers)); - return Pipe::unitePipes(std::move(pipes)); + if (!query_plan.isInitialized()) + { + query_plan = std::move(buffers_plan); + return; + } + + auto result_header = buffers_plan.getCurrentDataStream().header; + + /// Convert structure from table to structure from buffer. + if (!blocksHaveEqualStructure(query_plan.getCurrentDataStream().header, result_header)) + { + auto converting = std::make_unique(query_plan.getCurrentDataStream(), result_header); + query_plan.addStep(std::move(converting)); + } + + DataStreams input_streams; + input_streams.emplace_back(query_plan.getCurrentDataStream()); + input_streams.emplace_back(buffers_plan.getCurrentDataStream()); + + std::vector> plans; + plans.emplace_back(std::make_unique(std::move(query_plan))); + plans.emplace_back(std::make_unique(std::move(buffers_plan))); + query_plan = QueryPlan(); + + auto union_step = std::make_unique(std::move(input_streams), result_header); + union_step->setStepDescription("Unite sources from Buffer table"); + query_plan.unitePlans(std::move(union_step), std::move(plans)); } diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index b18b574ec6c..406e6a51fdb 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -65,6 +65,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + bool supportsParallelInsert() const override { return true; } BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index a2e3fae0951..69669c0b680 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -107,6 +107,21 @@ QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage(cons } Pipe StorageMaterializedView::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); +} + +void StorageMaterializedView::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & /*metadata_snapshot*/, const SelectQueryInfo & query_info, @@ -122,10 +137,7 @@ Pipe StorageMaterializedView::read( if (query_info.order_optimizer) query_info.input_order_info = query_info.order_optimizer->getInputOrder(storage, metadata_snapshot); - Pipe pipe = storage->read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - pipe.addTableLock(lock); - - return pipe; + storage->read(query_plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); } BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 1ee4246c7f1..b7e60649601 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -80,6 +80,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + Strings getDataPaths() const override; private: diff --git a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql index e7174c5b56b..cb572cc542f 100644 --- a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql +++ b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql @@ -10,9 +10,9 @@ SET max_threads=1; SET optimize_move_functions_out_of_any=0; SELECT 'LIMIT'; -SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) LIMIT 1 SETTINGS distributed_group_by_no_merge=2; +SELECT * FROM (SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) LIMIT 1 ) ORDER BY shard_num SETTINGS distributed_group_by_no_merge=2; SELECT 'OFFSET'; -SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) LIMIT 1, 1 SETTINGS distributed_group_by_no_merge=2; +SELECT * FROM (SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) LIMIT 1, 1) ORDER BY shard_num SETTINGS distributed_group_by_no_merge=2; SELECT 'ALIAS'; SELECT dummy AS d FROM remote('127.0.0.{2,3}', system.one) ORDER BY d SETTINGS distributed_group_by_no_merge=2; From fcd40d041fc011e24650c199d1f1f93149db2b61 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 30 Sep 2020 15:54:33 +0300 Subject: [PATCH 020/114] Update test. --- tests/queries/0_stateless/01508_explain_header.reference | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01508_explain_header.reference b/tests/queries/0_stateless/01508_explain_header.reference index 50216432e14..a1f0b44c224 100644 --- a/tests/queries/0_stateless/01508_explain_header.reference +++ b/tests/queries/0_stateless/01508_explain_header.reference @@ -3,5 +3,7 @@ Header: x UInt8 Expression (Before ORDER BY and SELECT) Header: _dummy UInt8 1 UInt8 - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) Header: dummy UInt8 + ReadFromStorage (SystemOne) + Header: dummy UInt8 From 49773d6761570ecd97544ed4b980af8914f52828 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 30 Sep 2020 18:53:12 +0300 Subject: [PATCH 021/114] Update test. --- .../0_stateless/00184_shard_distributed_group_by_no_merge.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql index cb572cc542f..9912e083777 100644 --- a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql +++ b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql @@ -10,9 +10,9 @@ SET max_threads=1; SET optimize_move_functions_out_of_any=0; SELECT 'LIMIT'; -SELECT * FROM (SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) LIMIT 1 ) ORDER BY shard_num SETTINGS distributed_group_by_no_merge=2; +SELECT * FROM (SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one)) ORDER BY shard_num LIMIT 1 SETTINGS distributed_group_by_no_merge=2; SELECT 'OFFSET'; -SELECT * FROM (SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one) LIMIT 1, 1) ORDER BY shard_num SETTINGS distributed_group_by_no_merge=2; +SELECT * FROM (SELECT any(_shard_num) shard_num, count(), uniq(dummy) FROM remote('127.0.0.{2,3}', system.one)) ORDER BY shard_num LIMIT 1, 1 SETTINGS distributed_group_by_no_merge=2; SELECT 'ALIAS'; SELECT dummy AS d FROM remote('127.0.0.{2,3}', system.one) ORDER BY d SETTINGS distributed_group_by_no_merge=2; From cb221528787f04b861b024452231c3649accbc83 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 30 Sep 2020 18:54:16 +0300 Subject: [PATCH 022/114] Fix build. --- src/Processors/QueryPlan/AddingMissedStep.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/AddingMissedStep.h b/src/Processors/QueryPlan/AddingMissedStep.h index 77075a410a5..8e3b8148b6a 100644 --- a/src/Processors/QueryPlan/AddingMissedStep.h +++ b/src/Processors/QueryPlan/AddingMissedStep.h @@ -1,12 +1,10 @@ #pragma once #include +#include namespace DB { -struct ColumnDefault; -using ColumnDefaults = std::unordered_map; - /// Convert one block structure to another. See ConvertingTransform. class AddingMissedStep : public ITransformingStep { @@ -21,7 +19,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; private: - const ColumnDefaults column_defaults; + ColumnDefaults column_defaults; const Context & context; }; From 7197a0c803f8ee65d5241b76fe09533e31b3f684 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 1 Oct 2020 11:38:50 +0100 Subject: [PATCH 023/114] Possibly fix the issue with quorum --- src/Storages/StorageReplicatedMergeTree.cpp | 72 ++++++++++++++++++++- src/Storages/StorageReplicatedMergeTree.h | 2 + 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e8007440f2f..23cca6a818e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3107,6 +3107,60 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) } } +void StorageReplicatedMergeTree::updateLastPartNodeIfMatches(const String & partition_id, const String & old_part_name, const String & new_part_name) +{ + auto zookeeper = getZooKeeper(); + + const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + + while (true) + { + Coordination::Stat added_parts_stat; + String old_added_parts = zookeeper->get(quorum_last_part_path, &added_parts_stat); + + ReplicatedMergeTreeQuorumAddedParts parts_with_quorum(format_version); + + if (!old_added_parts.empty()) + parts_with_quorum.fromString(old_added_parts); + + if (!parts_with_quorum.added_parts.count(partition_id)) + { + /// There is no information about partition at all. + break; + } + + /// Part for which last quorum was reached in partition_id. + auto quorum_part_info = MergeTreePartInfo::fromPartName(parts_with_quorum.added_parts.at(partition_id), format_version); + auto old_part_info = MergeTreePartInfo::fromPartName(old_part_name, format_version); + + /// Update last part for which quorum was reached. + if (old_part_info.contains(quorum_part_info)) + parts_with_quorum.added_parts.emplace(partition_id, new_part_name); + + /// Serialize and try update. + String new_added_parts = parts_with_quorum.toString(); + + auto code = zookeeper->trySet(quorum_last_part_path, new_added_parts, added_parts_stat.version); + + if (code == Coordination::Error::ZOK) + { + break; + } + else if (code == Coordination::Error::ZNONODE) + { + /// Node is deleted. It is impossible, but it is Ok. + break; + } + else if (code == Coordination::Error::ZBADVERSION) + { + /// Node was updated meanwhile. We must re-read it and repeat all the actions. + continue; + } + else + throw Coordination::Exception(code, quorum_last_part_path); + } +} + bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot, const String & source_replica_path, bool to_detached, size_t quorum) @@ -4055,9 +4109,23 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool de } } + bool drop_entire_partition = !drop_part; + /// Cleaning possibly stored information about parts from /quorum/last_part node in ZooKeeper. - /// TODO(nv) how is this related to dropPart? Is it? - if (!drop_part) + if (drop_part) + { + auto part_info = MergeTreePartInfo::fromPartName(partition->as().value.safeGet(), format_version); + auto data_parts_vec = getDataPartsVectorInPartition(DataPartState::Committed, part_info.partition_id); + std::sort(data_parts_vec.begin(), data_parts_vec.end(), LessDataPart()); + + auto prev_part = std::upper_bound(data_parts_vec.begin(), data_parts_vec.end(), part_info, LessDataPart()); + if (prev_part != data_parts_vec.end()) + updateLastPartNodeIfMatches(part_info.partition_id, part_info.getPartName(), (*prev_part)->info.getPartName()); + else if (data_parts_vec.empty()) + drop_entire_partition = true; + } + + if (drop_entire_partition) { String partition_id = getPartitionIDFromQuery(partition, query_context); cleanLastPartNode(partition_id); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 3ce946a54c0..4fb57f6a10b 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -490,6 +490,8 @@ private: /// Deletes info from quorum/last_part node for particular partition_id. void cleanLastPartNode(const String & partition_id); + void updateLastPartNodeIfMatches(const String & partition_id, const String & old_part_name, const String & new_part_name); + /// Creates new block number if block with such block_id does not exist std::optional allocateBlockNumber( const String & partition_id, zkutil::ZooKeeperPtr & zookeeper, From ec64def384bf23288450e6a7a4b005d748a46245 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 1 Oct 2020 20:34:22 +0300 Subject: [PATCH 024/114] Use QueryPlan while reading from MergeTree. --- src/Processors/QueryPipeline.cpp | 6 + src/Processors/QueryPipeline.h | 5 + .../QueryPlan/AddingConstColumnStep.cpp | 41 +++ .../QueryPlan/AddingConstColumnStep.h | 22 ++ src/Processors/QueryPlan/MergingFinal.cpp | 159 ++++++++++ src/Processors/QueryPlan/MergingFinal.h | 35 +++ src/Processors/QueryPlan/QueryPlan.h | 5 +- src/Processors/QueryPlan/ReverseRowsStep.cpp | 37 +++ src/Processors/QueryPlan/ReverseRowsStep.h | 18 ++ .../Transforms/AddingConstColumnTransform.h | 33 +- src/Processors/ya.make | 3 + src/Storages/IStorage.h | 1 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 284 ++++++++---------- .../MergeTree/MergeTreeDataSelectExecutor.h | 10 +- .../MergeTree/StorageFromMergeTreeDataPart.h | 9 +- src/Storages/StorageMerge.cpp | 9 +- src/Storages/StorageMergeTree.cpp | 20 +- src/Storages/StorageMergeTree.h | 10 + src/Storages/StorageReplicatedMergeTree.cpp | 23 +- src/Storages/StorageReplicatedMergeTree.h | 10 + 20 files changed, 546 insertions(+), 194 deletions(-) create mode 100644 src/Processors/QueryPlan/AddingConstColumnStep.cpp create mode 100644 src/Processors/QueryPlan/AddingConstColumnStep.h create mode 100644 src/Processors/QueryPlan/MergingFinal.cpp create mode 100644 src/Processors/QueryPlan/MergingFinal.h create mode 100644 src/Processors/QueryPlan/ReverseRowsStep.cpp create mode 100644 src/Processors/QueryPlan/ReverseRowsStep.h diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index 4cbb4d9edb7..8280923b1e3 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -96,6 +96,12 @@ void QueryPipeline::addTransform(ProcessorPtr transform) pipe.addTransform(std::move(transform)); } +void QueryPipeline::transform(const Transformer & transformer) +{ + checkInitializedAndNotCompleted(); + pipe.transform(transformer); +} + void QueryPipeline::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter) { checkInitializedAndNotCompleted(); diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index ca6ec76bb0b..d91cfe89840 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -53,6 +53,11 @@ public: void addSimpleTransform(const Pipe::ProcessorGetterWithStreamKind & getter); /// Add transform with getNumStreams() input ports. void addTransform(ProcessorPtr transform); + + using Transformer = std::function; + /// Transform pipeline in general way. + void transform(const Transformer & transformer); + /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port. void addTotalsHavingTransform(ProcessorPtr transform); /// Add transform which calculates extremes. This transform adds extremes port and doesn't change inputs number. diff --git a/src/Processors/QueryPlan/AddingConstColumnStep.cpp b/src/Processors/QueryPlan/AddingConstColumnStep.cpp new file mode 100644 index 00000000000..27c7720e58e --- /dev/null +++ b/src/Processors/QueryPlan/AddingConstColumnStep.cpp @@ -0,0 +1,41 @@ +#include +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +AddingConstColumnStep::AddingConstColumnStep(const DataStream & input_stream_, ColumnWithTypeAndName column_) + : ITransformingStep(input_stream_, + AddingConstColumnTransform::transformHeader(input_stream_.header, column_), + getTraits()) + , column(std::move(column_)) +{ +} + +void AddingConstColumnStep::transformPipeline(QueryPipeline & pipeline) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, column); + }); +} + +} diff --git a/src/Processors/QueryPlan/AddingConstColumnStep.h b/src/Processors/QueryPlan/AddingConstColumnStep.h new file mode 100644 index 00000000000..baa63873f21 --- /dev/null +++ b/src/Processors/QueryPlan/AddingConstColumnStep.h @@ -0,0 +1,22 @@ +#pragma once +#include + +namespace DB +{ + +/// Adds a materialized const column with a specified value. +class AddingConstColumnStep : public ITransformingStep +{ +public: + AddingConstColumnStep(const DataStream & input_stream_, ColumnWithTypeAndName column_); + + String getName() const override { return "AddingConstColumn"; } + + void transformPipeline(QueryPipeline & pipeline) override; + +private: + ColumnWithTypeAndName column; +}; + +} + diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp new file mode 100644 index 00000000000..cd9d6abb4c3 --- /dev/null +++ b/src/Processors/QueryPlan/MergingFinal.cpp @@ -0,0 +1,159 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = false, + .preserves_sorting = false, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +MergingFinal::MergingFinal( + const DataStream & input_stream, + size_t num_output_streams_, + SortDescription sort_description_, + MergeTreeData::MergingParams params_, + Names partition_key_columns_, + size_t max_block_size_) + : ITransformingStep(input_stream, input_stream.header, getTraits()) + , num_output_streams(num_output_streams_) + , sort_description(std::move(sort_description_)) + , merging_params(std::move(params_)) + , partition_key_columns(std::move(partition_key_columns_)) + , max_block_size(max_block_size_) +{ + /// TODO: check input_stream is partially sorted (each port) by the same description. +// output_stream->sort_description = sort_description; +// output_stream->sort_mode = DataStream::SortMode::Stream; +} + +void MergingFinal::transformPipeline(QueryPipeline & pipeline) +{ + const auto & header = pipeline.getHeader(); + size_t num_outputs = pipeline.getNumStreams(); + + auto get_merging_processor = [&]() -> MergingTransformPtr + { + switch (merging_params.mode) + { + case MergeTreeData::MergingParams::Ordinary: + { + return std::make_shared(header, num_outputs, + sort_description, max_block_size); + } + + case MergeTreeData::MergingParams::Collapsing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.sign_column, true, max_block_size); + + case MergeTreeData::MergingParams::Summing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.columns_to_sum, partition_key_columns, max_block_size); + + case MergeTreeData::MergingParams::Aggregating: + return std::make_shared(header, num_outputs, + sort_description, max_block_size); + + case MergeTreeData::MergingParams::Replacing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.version_column, max_block_size); + + case MergeTreeData::MergingParams::VersionedCollapsing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.sign_column, max_block_size); + + case MergeTreeData::MergingParams::Graphite: + throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); + } + + __builtin_unreachable(); + }; + + if (num_outputs <= 1 || sort_description.empty()) + { + pipeline.addTransform(get_merging_processor()); + return; + } + + ColumnNumbers key_columns; + key_columns.reserve(sort_description.size()); + + for (auto & desc : sort_description) + { + if (!desc.column_name.empty()) + key_columns.push_back(header.getPositionByName(desc.column_name)); + else + key_columns.emplace_back(desc.column_number); + } + + pipeline.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, num_outputs, key_columns); + }); + + pipeline.transform([&](OutputPortRawPtrs ports) + { + Processors processors; + std::vector output_ports; + processors.reserve(ports.size() + num_outputs); + output_ports.reserve(ports.size()); + + for (auto & port : ports) + { + auto copier = std::make_shared(header, num_outputs); + connect(*port, copier->getInputPort()); + output_ports.emplace_back(copier->getOutputs().begin()); + processors.emplace_back(std::move(copier)); + } + + for (size_t i = 0; i < num_outputs; ++i) + { + auto merge = get_merging_processor(); + merge->setSelectorPosition(i); + auto input = merge->getInputs().begin(); + + /// Connect i-th merge with i-th input port of every copier. + for (size_t j = 0; j < ports.size(); ++j) + { + connect(*output_ports[j], *input); + ++output_ports[j]; + ++input; + } + + processors.emplace_back(std::move(merge)); + } + + return processors; + }); +} + +void MergingFinal::describeActions(FormatSettings & settings) const +{ + String prefix(settings.offset, ' '); + settings.out << prefix << "Sort description: "; + dumpSortDescription(sort_description, input_streams.front().header, settings.out); + settings.out << '\n'; +} + +} diff --git a/src/Processors/QueryPlan/MergingFinal.h b/src/Processors/QueryPlan/MergingFinal.h new file mode 100644 index 00000000000..c01f5c7f9a1 --- /dev/null +++ b/src/Processors/QueryPlan/MergingFinal.h @@ -0,0 +1,35 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +/// Merge streams of data into single sorted stream. +class MergingFinal : public ITransformingStep +{ +public: + explicit MergingFinal( + const DataStream & input_stream, + size_t num_output_streams_, + SortDescription sort_description_, + MergeTreeData::MergingParams params_, + Names partition_key_columns_, + size_t max_block_size_); + + String getName() const override { return "MergingFinal"; } + + void transformPipeline(QueryPipeline & pipeline) override; + + void describeActions(FormatSettings & settings) const override; + +private: + size_t num_output_streams; + SortDescription sort_description; + MergeTreeData::MergingParams merging_params; + Names partition_key_columns; + size_t max_block_size; +}; + +} diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 6296eac7502..4f558e04c55 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -17,6 +17,9 @@ using QueryPipelinePtr = std::unique_ptr; class Context; class WriteBuffer; +class QueryPlan; +using QueryPlanPtr = std::unique_ptr; + /// A tree of query steps. /// The goal of QueryPlan is to build QueryPipeline. /// QueryPlan let delay pipeline creation which is helpful for pipeline-level optimisations. @@ -28,7 +31,7 @@ public: QueryPlan(QueryPlan &&); QueryPlan & operator=(QueryPlan &&); - void unitePlans(QueryPlanStepPtr step, std::vector> plans); + void unitePlans(QueryPlanStepPtr step, std::vector plans); void addStep(QueryPlanStepPtr step); bool isInitialized() const { return root != nullptr; } /// Tree is not empty diff --git a/src/Processors/QueryPlan/ReverseRowsStep.cpp b/src/Processors/QueryPlan/ReverseRowsStep.cpp new file mode 100644 index 00000000000..32e16937611 --- /dev/null +++ b/src/Processors/QueryPlan/ReverseRowsStep.cpp @@ -0,0 +1,37 @@ +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = true, + .returns_single_stream = false, + .preserves_number_of_streams = true, + .preserves_sorting = false, + }, + { + .preserves_number_of_rows = true, + } + }; +} + +ReverseRowsStep::ReverseRowsStep(const DataStream & input_stream_) + : ITransformingStep(input_stream_, input_stream_.header, getTraits()) +{ +} + +void ReverseRowsStep::transformPipeline(QueryPipeline & pipeline) +{ + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header); + }); +} + +} diff --git a/src/Processors/QueryPlan/ReverseRowsStep.h b/src/Processors/QueryPlan/ReverseRowsStep.h new file mode 100644 index 00000000000..955d022cde0 --- /dev/null +++ b/src/Processors/QueryPlan/ReverseRowsStep.h @@ -0,0 +1,18 @@ +#pragma once +#include + +namespace DB +{ + +/// Reverse rows in chunk. +class ReverseRowsStep : public ITransformingStep +{ +public: + ReverseRowsStep(const DataStream & input_stream_); + + String getName() const override { return "ReverseRows"; } + + void transformPipeline(QueryPipeline & pipeline) override; +}; + +} diff --git a/src/Processors/Transforms/AddingConstColumnTransform.h b/src/Processors/Transforms/AddingConstColumnTransform.h index 26d70d27ca7..15e9addd924 100644 --- a/src/Processors/Transforms/AddingConstColumnTransform.h +++ b/src/Processors/Transforms/AddingConstColumnTransform.h @@ -4,33 +4,40 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + /// Adds a materialized const column to the chunk with a specified value. -template class AddingConstColumnTransform : public ISimpleTransform { public: - AddingConstColumnTransform(const Block & header, DataTypePtr data_type_, T value_, const String & column_name_) - : ISimpleTransform(header, addColumn(header, data_type_, column_name_), false) - , data_type(std::move(data_type_)), value(value_) {} + AddingConstColumnTransform(const Block & header, ColumnWithTypeAndName column_) + : ISimpleTransform(header, transformHeader(header, column_), false) + , column(std::move(column_)) + { + if (!column.column || !isColumnConst(*column.column) || !column.column->empty()) + throw Exception("AddingConstColumnTransform expected empty const column", ErrorCodes::LOGICAL_ERROR); + } String getName() const override { return "AddingConstColumnTransform"; } + static Block transformHeader(Block header, ColumnWithTypeAndName & column_) + { + header.insert(column_); + return header; + } + protected: void transform(Chunk & chunk) override { auto num_rows = chunk.getNumRows(); - chunk.addColumn(data_type->createColumnConst(num_rows, value)->convertToFullColumnIfConst()); + chunk.addColumn(column.column->cloneResized(num_rows)->convertToFullColumnIfConst()); } private: - static Block addColumn(Block header, const DataTypePtr & data_type, const String & column_name) - { - header.insert({data_type->createColumn(), data_type, column_name}); - return header; - } - - DataTypePtr data_type; - T value; + ColumnWithTypeAndName column; }; } diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 8c3709f2d34..972e4995747 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -90,6 +90,7 @@ SRCS( Port.cpp printPipeline.cpp QueryPipeline.cpp + QueryPlan/AddingConstColumnStep.cpp QueryPlan/AddingDelayedSourceStep.cpp QueryPlan/AddingMissedStep.cpp QueryPlan/AggregatingStep.cpp @@ -111,12 +112,14 @@ SRCS( QueryPlan/MaterializingStep.cpp QueryPlan/MergeSortingStep.cpp QueryPlan/MergingAggregatedStep.cpp + QueryPlan/MergingFinal.cpp QueryPlan/MergingSortedStep.cpp QueryPlan/OffsetStep.cpp QueryPlan/PartialSortingStep.cpp QueryPlan/QueryPlan.cpp QueryPlan/ReadFromPreparedSource.cpp QueryPlan/ReadNothingStep.cpp + QueryPlan/ReverseRowsStep.cpp QueryPlan/RollupStep.cpp QueryPlan/SettingQuotaAndLimitsStep.cpp QueryPlan/TotalsHavingStep.cpp diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 8df832a9511..2260587b432 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -49,6 +49,7 @@ using Processors = std::vector; class Pipe; class QueryPlan; +using QueryPlanPtr = std::unique_ptr; class StoragePolicy; using StoragePolicyPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8b5b337bcec..72940e2754b 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -20,6 +20,16 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /// Allow to use __uint128_t as a template parameter for boost::rational. // https://stackoverflow.com/questions/41198673/uint128-t-not-working-with-clang-and-libstdc @@ -46,20 +56,6 @@ namespace std #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include namespace ProfileEvents @@ -151,7 +147,7 @@ static RelativeSize convertAbsoluteSampleSizeToRelative(const ASTPtr & node, siz } -Pipe MergeTreeDataSelectExecutor::read( +QueryPlanPtr MergeTreeDataSelectExecutor::read( const Names & column_names_to_return, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -166,7 +162,7 @@ Pipe MergeTreeDataSelectExecutor::read( max_block_numbers_to_read); } -Pipe MergeTreeDataSelectExecutor::readFromParts( +QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( MergeTreeData::DataPartsVector parts, const Names & column_names_to_return, const StorageMetadataPtr & metadata_snapshot, @@ -688,7 +684,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges); ProfileEvents::increment(ProfileEvents::SelectedMarks, sum_marks); - Pipe res; + QueryPlanPtr plan; /// Projection, that needed to drop columns, which have appeared by execution /// of some extra expressions, and to allow execute the same expressions later. @@ -709,7 +705,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( std::sort(column_names_to_read.begin(), column_names_to_read.end()); column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); - res = spreadMarkRangesAmongStreamsFinal( + plan = spreadMarkRangesAmongStreamsFinal( std::move(parts_with_ranges), num_streams, column_names_to_read, @@ -731,7 +727,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActions(false); - res = spreadMarkRangesAmongStreamsWithOrder( + plan = spreadMarkRangesAmongStreamsWithOrder( std::move(parts_with_ranges), num_streams, column_names_to_read, @@ -747,7 +743,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( } else { - res = spreadMarkRangesAmongStreams( + plan = spreadMarkRangesAmongStreams( std::move(parts_with_ranges), num_streams, column_names_to_read, @@ -762,40 +758,47 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( if (use_sampling) { - res.addSimpleTransform([&filter_expression, &filter_function](const Block & header) - { - return std::make_shared( - header, filter_expression, filter_function->getColumnName(), false); - }); + auto sampling_step = std::make_unique( + plan->getCurrentDataStream(), + filter_expression, + filter_function->getColumnName(), + false); + + sampling_step->setStepDescription("Sampling"); + plan->addStep(std::move(sampling_step)); } if (result_projection) { - res.addSimpleTransform([&result_projection](const Block & header) - { - return std::make_shared(header, result_projection); - }); + auto projection_step = std::make_unique(plan->getCurrentDataStream(), result_projection); + projection_step->setStepDescription("Remove unused columns after reading from storage"); + plan->addStep(std::move(projection_step)); } /// By the way, if a distributed query or query to a Merge table is made, then the `_sample_factor` column can have different values. if (sample_factor_column_queried) { - res.addSimpleTransform([used_sample_factor](const Block & header) - { - return std::make_shared>( - header, std::make_shared(), used_sample_factor, "_sample_factor"); - }); + ColumnWithTypeAndName column; + column.name = "_sample_factor"; + column.type = std::make_shared(); + column.column = column.type->createColumnConst(0, Field(used_sample_factor)); + + auto adding_column = std::make_unique(plan->getCurrentDataStream(), std::move(column)); + adding_column->setStepDescription("Add _sample_factor column"); + plan->addStep(std::move(adding_column)); } if (query_info.prewhere_info && query_info.prewhere_info->remove_columns_actions) { - res.addSimpleTransform([&query_info](const Block & header) - { - return std::make_shared(header, query_info.prewhere_info->remove_columns_actions); - }); + auto expression_step = std::make_unique( + plan->getCurrentDataStream(), + query_info.prewhere_info->remove_columns_actions); + + expression_step->setStepDescription("Remove unused columns after PREWHERE"); + plan->addStep(std::move(expression_step)); } - return res; + return plan; } namespace @@ -820,8 +823,20 @@ size_t roundRowsOrBytesToMarks( } +static QueryPlanPtr createPlanFromPipe(Pipe pipe, const std::string & description = "") +{ + auto plan = std::make_unique(); -Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( + std::string storage_name = "MergeTree"; + if (!description.empty()) + storage_name += ' ' + description; + + auto step = std::make_unique(std::move(pipe), storage_name); + plan->addStep(std::move(step)); + return plan; +} + +QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -915,7 +930,7 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( res.emplace_back(std::move(source)); } - return Pipe::unitePipes(std::move(res)); + return createPlanFromPipe(Pipe::unitePipes(std::move(res))); } else { @@ -939,19 +954,18 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( if (pipe.numOutputPorts() > 1) pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts())); - return pipe; + return createPlanFromPipe(std::move(pipe)); } } -static ExpressionActionsPtr createProjection(const Pipe & pipe, const MergeTreeData & data) +static ExpressionActionsPtr createProjection(const Block & header, const MergeTreeData & data) { - const auto & header = pipe.getHeader(); auto projection = std::make_shared(header.getNamesAndTypesList(), data.global_context); projection->add(ExpressionAction::project(header.getNames())); return projection; } -Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( +QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -1053,6 +1067,8 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( const size_t min_marks_per_stream = (sum_marks - 1) / num_streams + 1; bool need_preliminary_merge = (parts.size() > settings.read_in_order_two_level_merge_threshold); + std::vector plans; + for (size_t i = 0; i < num_streams && !parts.empty(); ++i) { size_t need_marks = min_marks_per_stream; @@ -1151,17 +1167,15 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( } } - auto pipe = Pipe::unitePipes(std::move(pipes)); + auto plan = createPlanFromPipe(Pipe::unitePipes(std::move(pipes)), " with order"); if (input_order_info->direction != 1) { - pipe.addSimpleTransform([](const Block & header) - { - return std::make_shared(header); - }); + auto reverse_step = std::make_unique(plan->getCurrentDataStream()); + plan->addStep(std::move(reverse_step)); } - if (pipe.numOutputPorts() > 1 && need_preliminary_merge) + if (need_preliminary_merge) { SortDescription sort_description; for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j) @@ -1169,24 +1183,45 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( input_order_info->direction, 1); /// Drop temporary columns, added by 'sorting_key_prefix_expr' - out_projection = createProjection(pipe, data); - pipe.addSimpleTransform([sorting_key_prefix_expr](const Block & header) - { - return std::make_shared(header, sorting_key_prefix_expr); - }); + out_projection = createProjection(plan->getCurrentDataStream().header, data); - pipe.addTransform(std::make_shared( - pipe.getHeader(), pipe.numOutputPorts(), sort_description, max_block_size)); + auto expression_step = std::make_unique( + plan->getCurrentDataStream(), + sorting_key_prefix_expr); + + expression_step->setStepDescription("Calculate sorting key prefix"); + plan->addStep(std::move(expression_step)); + + auto merging_sorted = std::make_unique( + plan->getCurrentDataStream(), + sort_description, + max_block_size); + + merging_sorted->setStepDescription("Merge sorting mark ranges"); + plan->addStep(std::move(merging_sorted)); } - res.emplace_back(std::move(pipe)); + plans.emplace_back(std::move(plan)); } - return Pipe::unitePipes(std::move(res)); + if (plans.size() == 1) + return std::move(plans.front()); + + DataStreams input_streams; + for (const auto & plan : plans) + input_streams.emplace_back(plan->getCurrentDataStream()); + + const auto & common_header = plans.front()->getCurrentDataStream().header; + auto union_step = std::make_unique(std::move(input_streams), common_header); + + auto plan = std::make_unique(); + plan->unitePlans(std::move(union_step), std::move(plans)); + + return plan; } -Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( +QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -1224,7 +1259,7 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( if (sum_marks > max_marks_to_use_cache) use_uncompressed_cache = false; - Pipe pipe; + QueryPlanPtr plan; { Pipes pipes; @@ -1241,17 +1276,21 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( pipes.emplace_back(std::move(source_processor)); } - pipe = Pipe::unitePipes(std::move(pipes)); + auto pipe = Pipe::unitePipes(std::move(pipes)); + + /// Drop temporary columns, added by 'sorting_key_expr' + if (!out_projection) + out_projection = createProjection(pipe.getHeader(), data); + + plan = createPlanFromPipe(std::move(pipe), "with final"); } - /// Drop temporary columns, added by 'sorting_key_expr' - if (!out_projection) - out_projection = createProjection(pipe, data); + auto expression_step = std::make_unique( + plan->getCurrentDataStream(), + metadata_snapshot->getSortingKey().expression); - pipe.addSimpleTransform([&metadata_snapshot](const Block & header) - { - return std::make_shared(header, metadata_snapshot->getSortingKey().expression); - }); + expression_step->setStepDescription("Calculate sorting key expression"); + plan->addStep(std::move(expression_step)); Names sort_columns = metadata_snapshot->getSortingKeyColumns(); SortDescription sort_description; @@ -1260,108 +1299,25 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( Names partition_key_columns = metadata_snapshot->getPartitionKey().column_names; - Block header = pipe.getHeader(); + const auto & header = plan->getCurrentDataStream().header; for (size_t i = 0; i < sort_columns_size; ++i) sort_description.emplace_back(header.getPositionByName(sort_columns[i]), 1, 1); - auto get_merging_processor = [&]() -> MergingTransformPtr - { - switch (data.merging_params.mode) - { - case MergeTreeData::MergingParams::Ordinary: - { - return std::make_shared(header, pipe.numOutputPorts(), - sort_description, max_block_size); - } - - case MergeTreeData::MergingParams::Collapsing: - return std::make_shared(header, pipe.numOutputPorts(), - sort_description, data.merging_params.sign_column, true, max_block_size); - - case MergeTreeData::MergingParams::Summing: - return std::make_shared(header, pipe.numOutputPorts(), - sort_description, data.merging_params.columns_to_sum, partition_key_columns, max_block_size); - - case MergeTreeData::MergingParams::Aggregating: - return std::make_shared(header, pipe.numOutputPorts(), - sort_description, max_block_size); - - case MergeTreeData::MergingParams::Replacing: - return std::make_shared(header, pipe.numOutputPorts(), - sort_description, data.merging_params.version_column, max_block_size); - - case MergeTreeData::MergingParams::VersionedCollapsing: - return std::make_shared(header, pipe.numOutputPorts(), - sort_description, data.merging_params.sign_column, max_block_size); - - case MergeTreeData::MergingParams::Graphite: - throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); - } - - __builtin_unreachable(); - }; - if (num_streams > settings.max_final_threads) num_streams = settings.max_final_threads; - if (num_streams <= 1 || sort_description.empty()) - { - pipe.addTransform(get_merging_processor()); - return pipe; - } + auto final_step = std::make_unique( + plan->getCurrentDataStream(), + num_streams, + sort_description, + data.merging_params, + partition_key_columns, + max_block_size); - ColumnNumbers key_columns; - key_columns.reserve(sort_description.size()); + final_step->setStepDescription("Merge rows for FINAL"); + plan->addStep(std::move(final_step)); - for (auto & desc : sort_description) - { - if (!desc.column_name.empty()) - key_columns.push_back(header.getPositionByName(desc.column_name)); - else - key_columns.emplace_back(desc.column_number); - } - - pipe.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, num_streams, key_columns); - }); - - pipe.transform([&](OutputPortRawPtrs ports) - { - Processors processors; - std::vector output_ports; - processors.reserve(ports.size() + num_streams); - output_ports.reserve(ports.size()); - - for (auto & port : ports) - { - auto copier = std::make_shared(header, num_streams); - connect(*port, copier->getInputPort()); - output_ports.emplace_back(copier->getOutputs().begin()); - processors.emplace_back(std::move(copier)); - } - - for (size_t i = 0; i < num_streams; ++i) - { - auto merge = get_merging_processor(); - merge->setSelectorPosition(i); - auto input = merge->getInputs().begin(); - - /// Connect i-th merge with i-th input port of every copier. - for (size_t j = 0; j < ports.size(); ++j) - { - connect(*output_ports[j], *input); - ++output_ports[j]; - ++input; - } - - processors.emplace_back(std::move(merge)); - } - - return processors; - }); - - return pipe; + return plan; } /// Calculates a set of mark ranges, that could possibly contain keys, required by condition. diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 5894d6e044b..7fd0e580b79 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -24,7 +24,7 @@ public: */ using PartitionIdToMaxBlock = std::unordered_map; - Pipe read( + QueryPlanPtr read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -33,7 +33,7 @@ public: unsigned num_streams, const PartitionIdToMaxBlock * max_block_numbers_to_read = nullptr) const; - Pipe readFromParts( + QueryPlanPtr readFromParts( MergeTreeData::DataPartsVector parts, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -48,7 +48,7 @@ private: Poco::Logger * log; - Pipe spreadMarkRangesAmongStreams( + QueryPlanPtr spreadMarkRangesAmongStreams( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -61,7 +61,7 @@ private: const MergeTreeReaderSettings & reader_settings) const; /// out_projection - save projection only with columns, requested to read - Pipe spreadMarkRangesAmongStreamsWithOrder( + QueryPlanPtr spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, size_t num_streams, const Names & column_names, @@ -75,7 +75,7 @@ private: const MergeTreeReaderSettings & reader_settings, ExpressionActionsPtr & out_projection) const; - Pipe spreadMarkRangesAmongStreamsFinal( + QueryPlanPtr spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, size_t num_streams, const Names & column_names, diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index c13f540ad34..049b4a99084 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -27,8 +29,11 @@ public: size_t max_block_size, unsigned num_streams) override { - return MergeTreeDataSelectExecutor(part->storage) - .readFromParts({part}, column_names, metadata_snapshot, query_info, context, max_block_size, num_streams); + QueryPlan query_plan = + std::move(*MergeTreeDataSelectExecutor(part->storage) + .readFromParts({part}, column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)); + + return QueryPipeline::getPipe(std::move(*query_plan.buildQueryPipeline())); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 42ac9cb6a5b..e660c27dd6b 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -319,10 +319,13 @@ Pipe StorageMerge::createSources( if (has_table_virtual_column) { - pipe.addSimpleTransform([name = table_name](const Block & stream_header) + ColumnWithTypeAndName column; + column.name = "_table"; + column.type = std::make_shared(); + column.column = column.type->createColumnConst(0, Field(table_name)); + pipe.addSimpleTransform([&](const Block & stream_header) { - return std::make_shared>( - stream_header, std::make_shared(), name, "_table"); + return std::make_shared(stream_header, column); }); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 55fb42b550e..35b147f22f7 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -171,17 +171,31 @@ StorageMergeTree::~StorageMergeTree() shutdown(); } -Pipe StorageMergeTree::read( +void StorageMergeTree::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum /*processed_stage*/, + size_t max_block_size, + unsigned num_streams) +{ + query_plan = std::move(*reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)); +} + +Pipe StorageMergeTree::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, const size_t max_block_size, const unsigned num_streams) { - return reader.read(column_names, metadata_snapshot, query_info, - context, max_block_size, num_streams); + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); } std::optional StorageMergeTree::totalRows() const diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 5662f9e0088..c890d5f3fc2 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -46,6 +46,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & /*metadata_snapshot*/, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + std::optional totalRows() const override; std::optional totalBytes() const override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9613bd5111d..8e2ecf1f189 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3514,7 +3514,9 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg return max_added_blocks; } -Pipe StorageReplicatedMergeTree::read( + +void StorageReplicatedMergeTree::read( + QueryPlan & query_plan, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -3531,10 +3533,25 @@ Pipe StorageReplicatedMergeTree::read( if (context.getSettingsRef().select_sequential_consistency) { auto max_added_blocks = getMaxAddedBlocks(); - return reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams, &max_added_blocks); + query_plan = std::move(*reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams, &max_added_blocks)); + return; } - return reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams); + query_plan = std::move(*reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)); +} + +Pipe StorageReplicatedMergeTree::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index d851082d5c2..2cea4817627 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -96,6 +96,16 @@ public: size_t max_block_size, unsigned num_streams) override; + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageMetadataPtr & /*metadata_snapshot*/, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + std::optional totalRows() const override; std::optional totalBytes() const override; From f02ad3dc8ecf6f3228faa667364823ef993ca356 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 1 Oct 2020 21:02:22 +0300 Subject: [PATCH 025/114] Use QueryPlan while reading from MergeTree. --- src/Processors/QueryPlan/MergingFinal.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp index cd9d6abb4c3..cde4d99ea47 100644 --- a/src/Processors/QueryPlan/MergingFinal.cpp +++ b/src/Processors/QueryPlan/MergingFinal.cpp @@ -90,7 +90,7 @@ void MergingFinal::transformPipeline(QueryPipeline & pipeline) __builtin_unreachable(); }; - if (num_outputs <= 1 || sort_description.empty()) + if (num_output_streams <= 1 || sort_description.empty()) { pipeline.addTransform(get_merging_processor()); return; @@ -109,25 +109,25 @@ void MergingFinal::transformPipeline(QueryPipeline & pipeline) pipeline.addSimpleTransform([&](const Block & stream_header) { - return std::make_shared(stream_header, num_outputs, key_columns); + return std::make_shared(stream_header, num_output_streams, key_columns); }); pipeline.transform([&](OutputPortRawPtrs ports) { Processors processors; std::vector output_ports; - processors.reserve(ports.size() + num_outputs); + processors.reserve(ports.size() + num_output_streams); output_ports.reserve(ports.size()); for (auto & port : ports) { - auto copier = std::make_shared(header, num_outputs); + auto copier = std::make_shared(header, num_output_streams); connect(*port, copier->getInputPort()); output_ports.emplace_back(copier->getOutputs().begin()); processors.emplace_back(std::move(copier)); } - for (size_t i = 0; i < num_outputs; ++i) + for (size_t i = 0; i < num_output_streams; ++i) { auto merge = get_merging_processor(); merge->setSelectorPosition(i); From 513402814069d1c2a300359825bba0eb7ba52850 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 1 Oct 2020 21:19:38 +0300 Subject: [PATCH 026/114] Fix style. --- src/Processors/QueryPlan/MergingFinal.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp index cde4d99ea47..9a33ce7709e 100644 --- a/src/Processors/QueryPlan/MergingFinal.cpp +++ b/src/Processors/QueryPlan/MergingFinal.cpp @@ -13,6 +13,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + static ITransformingStep::Traits getTraits() { return ITransformingStep::Traits From ea131989befee7e638a294e98551f7d60bc33d3a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 1 Oct 2020 21:47:20 +0300 Subject: [PATCH 027/114] Try fix test. --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 72940e2754b..3129fa81be9 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -540,7 +540,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( if (no_data) { LOG_DEBUG(log, "Sampling yields no data."); - return {}; + return std::make_unique(); } LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); @@ -678,7 +678,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( LOG_DEBUG(log, "Selected {} parts by partition key, {} parts by primary key, {} marks by primary key, {} marks to read from {} ranges", parts.size(), parts_with_ranges.size(), sum_marks_pk.load(std::memory_order_relaxed), sum_marks, sum_ranges); if (parts_with_ranges.empty()) - return {}; + return std::make_unique(); ProfileEvents::increment(ProfileEvents::SelectedParts, parts_with_ranges.size()); ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges); @@ -756,6 +756,9 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( reader_settings); } + if (!plan) + return std::make_unique(); + if (use_sampling) { auto sampling_step = std::make_unique( From 5ea4dc0850189a76e531883bdeabb9c22a32175f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 2 Oct 2020 14:25:16 +0300 Subject: [PATCH 028/114] Try fix tests. --- src/Storages/MergeTree/StorageFromMergeTreeDataPart.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 049b4a99084..769e82155b9 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -33,6 +33,9 @@ public: std::move(*MergeTreeDataSelectExecutor(part->storage) .readFromParts({part}, column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)); + if (!query_plan.isInitialized()) + return {}; + return QueryPipeline::getPipe(std::move(*query_plan.buildQueryPipeline())); } From bef96faa1a062af6ae6c9968bc859850f54dd2db Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 2 Oct 2020 15:38:33 +0300 Subject: [PATCH 029/114] Tru fix tests. --- src/Interpreters/InterpreterSelectQuery.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 5a6bb5770cf..f8e896b3448 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1382,7 +1382,7 @@ void InterpreterSelectQuery::executeFetchColumns( ErrorCodes::TOO_MANY_COLUMNS); /// General limit for the number of threads. - query_plan.setMaxThreads(settings.max_threads); + size_t max_threads_execute_query = settings.max_threads; /** With distributed query processing, almost no computations are done in the threads, * but wait and receive data from remote servers. @@ -1395,8 +1395,7 @@ void InterpreterSelectQuery::executeFetchColumns( if (storage && storage->isRemote()) { is_remote = true; - max_streams = settings.max_distributed_connections; - query_plan.setMaxThreads(max_streams); + max_threads_execute_query = max_streams = settings.max_distributed_connections; } UInt64 max_block_size = settings.max_block_size; @@ -1421,8 +1420,7 @@ void InterpreterSelectQuery::executeFetchColumns( && limit_length + limit_offset < max_block_size) { max_block_size = std::max(UInt64(1), limit_length + limit_offset); - max_streams = 1; - query_plan.setMaxThreads(max_streams); + max_threads_execute_query = max_streams = 1; } if (!max_block_size) @@ -1529,6 +1527,8 @@ void InterpreterSelectQuery::executeFetchColumns( else throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); + query_plan.setMaxThreads(max_threads_execute_query); + /// Aliases in table declaration. if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions) { From 11c0c2864fb539d138b4c96ea85e63b58921a383 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 2 Oct 2020 17:23:06 +0300 Subject: [PATCH 030/114] Fix build. --- src/Processors/QueryPlan/MergingFinal.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp index 9a33ce7709e..c1dec231f11 100644 --- a/src/Processors/QueryPlan/MergingFinal.cpp +++ b/src/Processors/QueryPlan/MergingFinal.cpp @@ -119,9 +119,9 @@ void MergingFinal::transformPipeline(QueryPipeline & pipeline) pipeline.transform([&](OutputPortRawPtrs ports) { - Processors processors; + Processors transforms; std::vector output_ports; - processors.reserve(ports.size() + num_output_streams); + transforms.reserve(ports.size() + num_output_streams); output_ports.reserve(ports.size()); for (auto & port : ports) @@ -129,7 +129,7 @@ void MergingFinal::transformPipeline(QueryPipeline & pipeline) auto copier = std::make_shared(header, num_output_streams); connect(*port, copier->getInputPort()); output_ports.emplace_back(copier->getOutputs().begin()); - processors.emplace_back(std::move(copier)); + transforms.emplace_back(std::move(copier)); } for (size_t i = 0; i < num_output_streams; ++i) @@ -146,10 +146,10 @@ void MergingFinal::transformPipeline(QueryPipeline & pipeline) ++input; } - processors.emplace_back(std::move(merge)); + transforms.emplace_back(std::move(merge)); } - return processors; + return transforms; }); } From 5bbae0953a748402aceab3e09566b3a7defaf8e1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Oct 2020 09:53:51 +0300 Subject: [PATCH 031/114] Fix order of resource destruction in SettingQuotaAndLimitsStep. --- src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp | 8 +++++--- src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp index 5b05ad77d6c..588dd7599a1 100644 --- a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.cpp @@ -30,20 +30,18 @@ SettingQuotaAndLimitsStep::SettingQuotaAndLimitsStep( std::shared_ptr quota_, std::shared_ptr context_) : ITransformingStep(input_stream_, input_stream_.header, getTraits()) + , context(std::move(context_)) , storage(std::move(storage_)) , table_lock(std::move(table_lock_)) , limits(limits_) , leaf_limits(leaf_limits_) , quota(std::move(quota_)) - , context(std::move(context_)) { } void SettingQuotaAndLimitsStep::transformPipeline(QueryPipeline & pipeline) { /// Table lock is stored inside pipeline here. - pipeline.addTableLock(table_lock); - pipeline.setLimits(limits); /** @@ -59,11 +57,15 @@ void SettingQuotaAndLimitsStep::transformPipeline(QueryPipeline & pipeline) if (quota) pipeline.setQuota(quota); + /// Order of resources below is important. if (context) pipeline.addInterpreterContext(std::move(context)); if (storage) pipeline.addStorageHolder(std::move(storage)); + + if (table_lock) + pipeline.addTableLock(std::move(table_lock)); } } diff --git a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h index 7ec4cfa91c6..66e44e18cd4 100644 --- a/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h +++ b/src/Processors/QueryPlan/SettingQuotaAndLimitsStep.h @@ -33,12 +33,12 @@ public: void transformPipeline(QueryPipeline & pipeline) override; private: + std::shared_ptr context; StoragePtr storage; TableLockHolder table_lock; StreamLocalLimits limits; SizeLimits leaf_limits; std::shared_ptr quota; - std::shared_ptr context; }; } From d968f276b5f6f0e6d5b85e968b5ead503649ba38 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Oct 2020 11:21:05 +0300 Subject: [PATCH 032/114] Fix buffer table. --- src/Storages/StorageBuffer.cpp | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 162463a4e33..f0fac1e4949 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -237,20 +237,24 @@ void StorageBuffer::read( query_plan, columns_intersection, destination_metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - auto adding_missed = std::make_unique( - query_plan.getCurrentDataStream(), - header_after_adding_defaults, - metadata_snapshot->getColumns().getDefaults(), context); + if (query_plan.isInitialized()) + { - adding_missed->setStepDescription("Add columns missing in destination table"); - query_plan.addStep(std::move(adding_missed)); + auto adding_missed = std::make_unique( + query_plan.getCurrentDataStream(), + header_after_adding_defaults, + metadata_snapshot->getColumns().getDefaults(), context); - auto converting = std::make_unique( - query_plan.getCurrentDataStream(), - header); + adding_missed->setStepDescription("Add columns missing in destination table"); + query_plan.addStep(std::move(adding_missed)); - converting->setStepDescription("Convert destination table columns to Buffer table structure"); - query_plan.addStep(std::move(converting)); + auto converting = std::make_unique( + query_plan.getCurrentDataStream(), + header); + + converting->setStepDescription("Convert destination table columns to Buffer table structure"); + query_plan.addStep(std::move(converting)); + } } } From efd1f10a6d8bff23132c1b20799ca62810c21b82 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Oct 2020 11:55:03 +0300 Subject: [PATCH 033/114] Fix build. --- src/Processors/QueryPlan/AddingMissedStep.cpp | 6 +++--- src/Processors/QueryPlan/AddingMissedStep.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Processors/QueryPlan/AddingMissedStep.cpp b/src/Processors/QueryPlan/AddingMissedStep.cpp index f2d06c033b1..d354405b5ac 100644 --- a/src/Processors/QueryPlan/AddingMissedStep.cpp +++ b/src/Processors/QueryPlan/AddingMissedStep.cpp @@ -25,10 +25,10 @@ static ITransformingStep::Traits getTraits() AddingMissedStep::AddingMissedStep( const DataStream & input_stream_, Block result_header_, - const ColumnDefaults & column_defaults_, + ColumnsDescription columns_, const Context & context_) : ITransformingStep(input_stream_, result_header_, getTraits()) - , column_defaults(column_defaults_) + , columns(std::move(columns_)) , context(context_) { updateDistinctColumns(output_stream->header, output_stream->distinct_columns); @@ -38,7 +38,7 @@ void AddingMissedStep::transformPipeline(QueryPipeline & pipeline) { pipeline.addSimpleTransform([&](const Block & header) { - return std::make_shared(header, output_stream->header, column_defaults, context); + return std::make_shared(header, output_stream->header, columns, context); }); } diff --git a/src/Processors/QueryPlan/AddingMissedStep.h b/src/Processors/QueryPlan/AddingMissedStep.h index 8e3b8148b6a..ce755b79fdf 100644 --- a/src/Processors/QueryPlan/AddingMissedStep.h +++ b/src/Processors/QueryPlan/AddingMissedStep.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include namespace DB { @@ -11,7 +11,7 @@ class AddingMissedStep : public ITransformingStep public: AddingMissedStep(const DataStream & input_stream_, Block result_header_, - const ColumnDefaults & column_defaults_, + ColumnsDescription columns_, const Context & context_); String getName() const override { return "AddingMissed"; } @@ -19,7 +19,7 @@ public: void transformPipeline(QueryPipeline & pipeline) override; private: - ColumnDefaults column_defaults; + ColumnsDescription columns; const Context & context; }; From 7caf6da363525a67c1eecfe421a1a20ec9e27229 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Oct 2020 11:56:21 +0300 Subject: [PATCH 034/114] Fix build. --- src/Storages/StorageBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index f0fac1e4949..182087c24ad 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -243,7 +243,7 @@ void StorageBuffer::read( auto adding_missed = std::make_unique( query_plan.getCurrentDataStream(), header_after_adding_defaults, - metadata_snapshot->getColumns().getDefaults(), context); + metadata_snapshot->getColumns(), context); adding_missed->setStepDescription("Add columns missing in destination table"); query_plan.addStep(std::move(adding_missed)); From c5cb05f5f33238d7561e953e9bca1dc494e5e1d6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 7 Oct 2020 14:26:29 +0300 Subject: [PATCH 035/114] Try fix tests. --- src/Processors/QueryPlan/QueryPlan.cpp | 11 +++++++++++ src/Processors/QueryPlan/QueryPlan.h | 5 +++++ src/Storages/MergeTree/StorageFromMergeTreeDataPart.h | 5 +---- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageMaterializedView.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/StorageView.cpp | 2 +- 9 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index 1ff844480a9..e681d4989d3 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -185,6 +185,17 @@ QueryPipelinePtr QueryPlan::buildQueryPipeline() return last_pipeline; } +Pipe QueryPlan::convertToPipe() +{ + if (!isInitialized()) + return {}; + + if (isCompleted()) + throw Exception("Cannot convert completed QueryPlan to Pipe", ErrorCodes::LOGICAL_ERROR); + + return QueryPipeline::getPipe(std::move(*buildQueryPipeline())); +} + void QueryPlan::addInterpreterContext(std::shared_ptr context) { interpreter_context.emplace_back(std::move(context)); diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 4f558e04c55..8aa0e868dc6 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -20,6 +20,8 @@ class WriteBuffer; class QueryPlan; using QueryPlanPtr = std::unique_ptr; +class Pipe; + /// A tree of query steps. /// The goal of QueryPlan is to build QueryPipeline. /// QueryPlan let delay pipeline creation which is helpful for pipeline-level optimisations. @@ -42,6 +44,9 @@ public: QueryPipelinePtr buildQueryPipeline(); + /// If initialized, build pipeline and convert to pipe. Otherwise, return empty pipe. + Pipe convertToPipe(); + struct ExplainPlanOptions { /// Add output header to step. diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 769e82155b9..7c30c7302b3 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -33,10 +33,7 @@ public: std::move(*MergeTreeDataSelectExecutor(part->storage) .readFromParts({part}, column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)); - if (!query_plan.isInitialized()) - return {}; - - return QueryPipeline::getPipe(std::move(*query_plan.buildQueryPipeline())); + return query_plan.convertToPipe(); } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 182087c24ad..f6e728ebbed 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -160,7 +160,7 @@ Pipe StorageBuffer::read( { QueryPlan plan; read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); + return plan.convertToPipe(); } void StorageBuffer::read( diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 6e7d7ea41bd..0b3070c88f6 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -505,7 +505,7 @@ Pipe StorageDistributed::read( { QueryPlan plan; read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); + return plan.convertToPipe(); } void StorageDistributed::read( diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 69669c0b680..9ea4a5d91d8 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -117,7 +117,7 @@ Pipe StorageMaterializedView::read( { QueryPlan plan; read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); + return plan.convertToPipe(); } void StorageMaterializedView::read( diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 35b147f22f7..6c0b082c6a9 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -195,7 +195,7 @@ Pipe StorageMergeTree::read( { QueryPlan plan; read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); + return plan.convertToPipe(); } std::optional StorageMergeTree::totalRows() const diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8e2ecf1f189..ef06d27101b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3551,7 +3551,7 @@ Pipe StorageReplicatedMergeTree::read( { QueryPlan plan; read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); + return plan.convertToPipe(); } diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index e71228f2a23..949e7930ccf 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -61,7 +61,7 @@ Pipe StorageView::read( { QueryPlan plan; read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - return QueryPipeline::getPipe(std::move(*plan.buildQueryPipeline())); + return plan.convertToPipe(); } void StorageView::read( From d94d88a6c02fab3384f63d0c796504c55555db5a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 7 Oct 2020 15:04:15 +0300 Subject: [PATCH 036/114] Fix style. --- src/Processors/QueryPlan/ReadFromStorageStep.cpp | 0 src/Processors/QueryPlan/ReadFromStorageStep.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/Processors/QueryPlan/ReadFromStorageStep.cpp delete mode 100644 src/Processors/QueryPlan/ReadFromStorageStep.h diff --git a/src/Processors/QueryPlan/ReadFromStorageStep.cpp b/src/Processors/QueryPlan/ReadFromStorageStep.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/Processors/QueryPlan/ReadFromStorageStep.h b/src/Processors/QueryPlan/ReadFromStorageStep.h deleted file mode 100644 index e69de29bb2d..00000000000 From 337098367bb80e38fc48f4cc55746c023f2502aa Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 13 Oct 2020 11:36:15 +0300 Subject: [PATCH 037/114] Try fix perf --- src/Interpreters/InterpreterSelectQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index fc496ce9dc3..9ba0b4668fb 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1424,6 +1424,8 @@ void InterpreterSelectQuery::executeFetchColumns( if (!max_block_size) throw Exception("Setting 'max_block_size' cannot be zero", ErrorCodes::PARAMETER_OUT_OF_BOUND); + query_plan.setMaxThreads(max_threads_execute_query); + /// Initialize the initial data streams to which the query transforms are superimposed. Table or subquery or prepared input? if (query_plan.isInitialized()) { @@ -1525,8 +1527,6 @@ void InterpreterSelectQuery::executeFetchColumns( else throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); - query_plan.setMaxThreads(max_threads_execute_query); - /// Aliases in table declaration. if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions) { From c91b45363055f3ed7f690c7c383aa01dc204d9cd Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Oct 2020 14:08:12 +0300 Subject: [PATCH 038/114] Try fix tests. --- src/Interpreters/InterpreterSelectQuery.cpp | 6 ++++-- src/Processors/QueryPlan/QueryPlan.h | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 2a4005e2acc..465dfb0c52c 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1439,8 +1439,6 @@ void InterpreterSelectQuery::executeFetchColumns( if (!max_block_size) throw Exception("Setting 'max_block_size' cannot be zero", ErrorCodes::PARAMETER_OUT_OF_BOUND); - query_plan.setMaxThreads(max_threads_execute_query); - /// Initialize the initial data streams to which the query transforms are superimposed. Table or subquery or prepared input? if (query_plan.isInitialized()) { @@ -1542,6 +1540,10 @@ void InterpreterSelectQuery::executeFetchColumns( else throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); + /// Specify the number of threads only if it wasn't specified in storage. + if (!query_plan.getMaxThreads()) + query_plan.setMaxThreads(max_threads_execute_query); + /// Aliases in table declaration. if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions) { diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 8aa0e868dc6..d53193a7aa5 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -69,6 +69,7 @@ public: /// Set upper limit for the recommend number of threads. Will be applied to the newly-created pipelines. /// TODO: make it in a better way. void setMaxThreads(size_t max_threads_) { max_threads = max_threads_; } + size_t getMaxThreads() const { return max_threads; } void addInterpreterContext(std::shared_ptr context); From a42fd18390bd0011516302f391721aec5d019947 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Oct 2020 15:01:22 +0300 Subject: [PATCH 039/114] Try fix tests. --- src/Storages/StorageMaterializedView.cpp | 29 +++++++++++++----------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index c07f9476a15..b587d4bc94c 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -140,21 +140,24 @@ void StorageMaterializedView::read( storage->read(query_plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - StreamLocalLimits limits; - SizeLimits leaf_limits; + if (query_plan.isInitialized()) + { + StreamLocalLimits limits; + SizeLimits leaf_limits; - /// Add table lock for destination table. - auto adding_limits_and_quota = std::make_unique( - query_plan.getCurrentDataStream(), - storage, - std::move(lock), - limits, - leaf_limits, - nullptr, - nullptr); + /// Add table lock for destination table. + auto adding_limits_and_quota = std::make_unique( + query_plan.getCurrentDataStream(), + storage, + std::move(lock), + limits, + leaf_limits, + nullptr, + nullptr); - adding_limits_and_quota->setStepDescription("Lock destination table for Buffer"); - query_plan.addStep(std::move(adding_limits_and_quota)); + adding_limits_and_quota->setStepDescription("Lock destination table for Buffer"); + query_plan.addStep(std::move(adding_limits_and_quota)); + } } BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) From 99ee127620d9df5ad41b769571459c2941777b25 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Nov 2020 10:50:38 +0300 Subject: [PATCH 040/114] Support `SETTINGS` clause for File engine Accept the usual user settings related to file formats. Most of the diff are the mechanistic code changes required to allow providing the required FormatSettings to the format factory. The File engine then extracts these settings from the `CREATE` query, and specifies them when creating the format parser. --- src/Formats/FormatFactory.cpp | 200 +++++++++--------- src/Formats/FormatFactory.h | 14 +- src/Formats/FormatSettings.h | 183 ++++++++-------- src/Processors/Formats/IRowInputFormat.cpp | 2 - src/Processors/Formats/IRowOutputFormat.h | 8 - .../Formats/Impl/ProtobufRowOutputFormat.cpp | 32 +-- .../Formats/Impl/ProtobufRowOutputFormat.h | 4 +- .../Formats/Impl/ValuesBlockInputFormat.cpp | 2 - src/Storages/Kafka/KafkaBlockOutputStream.cpp | 17 +- .../RabbitMQ/RabbitMQBlockOutputStream.cpp | 17 +- src/Storages/StorageFile.cpp | 89 +++++--- src/Storages/StorageFile.h | 8 +- src/TableFunctions/TableFunctionFile.cpp | 20 +- .../01544_file_engine_settings.reference | 2 + .../0_stateless/01544_file_engine_settings.sh | 23 ++ 15 files changed, 345 insertions(+), 276 deletions(-) create mode 100644 tests/queries/0_stateless/01544_file_engine_settings.reference create mode 100755 tests/queries/0_stateless/01544_file_engine_settings.sh diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 4dc5b816420..7011d2372b0 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -41,96 +41,75 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name) } -static FormatSettings getInputFormatSetting(const Settings & settings, const Context & context) +FormatSettings getFormatSettings(const Context & context) { + const auto & settings = context.getSettingsRef(); + FormatSettings format_settings; - format_settings.csv.delimiter = settings.format_csv_delimiter; - format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; + + format_settings.avro.allow_missing_fields = settings.input_format_avro_allow_missing_fields; + format_settings.avro.output_codec = settings.output_format_avro_codec; + format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval; + format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString(); format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; - format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null; + format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; + format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line; + format_settings.csv.delimiter = settings.format_csv_delimiter; format_settings.csv.empty_as_default = settings.input_format_defaults_for_omitted_fields; format_settings.csv.input_format_enum_as_number = settings.input_format_csv_enum_as_number; - format_settings.null_as_default = settings.input_format_null_as_default; - format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; - format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions; - format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals; - format_settings.with_names_use_header = settings.input_format_with_names_use_header; - format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields; - format_settings.import_nested_json = settings.input_format_import_nested_json; + format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null; + format_settings.custom.escaping_rule = settings.format_custom_escaping_rule; + format_settings.custom.field_delimiter = settings.format_custom_field_delimiter; + format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; + format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; + format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter; + format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; + format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; + format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; format_settings.date_time_input_format = settings.date_time_input_format; + format_settings.date_time_output_format = settings.date_time_output_format; + format_settings.enable_streaming = settings.output_format_enable_streaming; + format_settings.import_nested_json = settings.input_format_import_nested_json; format_settings.input_allow_errors_num = settings.input_format_allow_errors_num; format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio; - format_settings.template_settings.resultset_format = settings.format_template_resultset; - format_settings.template_settings.row_format = settings.format_template_row; - format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; - format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default; - format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number; + format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; + format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; + format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; + format_settings.null_as_default = settings.input_format_null_as_default; + format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; + format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; + format_settings.pretty.color = settings.output_format_pretty_color; + format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; + format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; + format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; + format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers; + format_settings.regexp.escaping_rule = settings.format_regexp_escaping_rule; + format_settings.regexp.regexp = settings.format_regexp; + format_settings.regexp.skip_unmatched = settings.format_regexp_skip_unmatched; format_settings.schema.format_schema = settings.format_schema; format_settings.schema.format_schema_path = context.getFormatSchemaPath(); format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER); - format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter; - format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; - format_settings.custom.escaping_rule = settings.format_custom_escaping_rule; - format_settings.custom.field_delimiter = settings.format_custom_field_delimiter; - format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; - format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; - format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; - format_settings.regexp.regexp = settings.format_regexp; - format_settings.regexp.escaping_rule = settings.format_regexp_escaping_rule; - format_settings.regexp.skip_unmatched = settings.format_regexp_skip_unmatched; + format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields; + format_settings.template_settings.resultset_format = settings.format_template_resultset; + format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; + format_settings.template_settings.row_format = settings.format_template_row; + format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line; + format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default; + format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number; + format_settings.tsv.null_representation = settings.output_format_tsv_null_representation; + format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals; + format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions; + format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; + format_settings.with_names_use_header = settings.input_format_with_names_use_header; + format_settings.write_statistics = settings.output_format_write_statistics; /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context - if (context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER)) + if (format_settings.schema.is_server) { const Poco::URI & avro_schema_registry_url = settings.format_avro_schema_registry_url; if (!avro_schema_registry_url.empty()) context.getRemoteHostFilter().checkURL(avro_schema_registry_url); } - format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString(); - format_settings.avro.allow_missing_fields = settings.input_format_avro_allow_missing_fields; - - return format_settings; -} - -static FormatSettings getOutputFormatSetting(const Settings & settings, const Context & context) -{ - FormatSettings format_settings; - format_settings.enable_streaming = settings.output_format_enable_streaming; - format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; - format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; - format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; - format_settings.csv.delimiter = settings.format_csv_delimiter; - format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; - format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; - format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line; - format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; - format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; - format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; - format_settings.pretty.color = settings.output_format_pretty_color; - format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? - FormatSettings::Pretty::Charset::ASCII : - FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers; - format_settings.template_settings.resultset_format = settings.format_template_resultset; - format_settings.template_settings.row_format = settings.format_template_row; - format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; - format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line; - format_settings.tsv.null_representation = settings.output_format_tsv_null_representation; - format_settings.write_statistics = settings.output_format_write_statistics; - format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; - format_settings.schema.format_schema = settings.format_schema; - format_settings.schema.format_schema_path = context.getFormatSchemaPath(); - format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER); - format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter; - format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; - format_settings.custom.escaping_rule = settings.format_custom_escaping_rule; - format_settings.custom.field_delimiter = settings.format_custom_field_delimiter; - format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; - format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; - format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; - format_settings.avro.output_codec = settings.output_format_avro_codec; - format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval; - format_settings.date_time_output_format = settings.date_time_output_format; return format_settings; } @@ -142,7 +121,7 @@ BlockInputStreamPtr FormatFactory::getInput( const Block & sample, const Context & context, UInt64 max_block_size, - ReadCallback callback) const + std::optional format_settings) const { if (name == "Native") return std::make_shared(buf, sample, 0); @@ -153,10 +132,12 @@ BlockInputStreamPtr FormatFactory::getInput( if (!input_getter) throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); - const Settings & settings = context.getSettingsRef(); - FormatSettings format_settings = getInputFormatSetting(settings, context); + if (!format_settings) + { + format_settings = getFormatSettings(context); + } - return input_getter(buf, sample, max_block_size, callback ? callback : ReadCallback(), format_settings); + return input_getter(buf, sample, max_block_size, {}, *format_settings); } const Settings & settings = context.getSettingsRef(); @@ -182,17 +163,21 @@ BlockInputStreamPtr FormatFactory::getInput( if (!input_getter) throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); - FormatSettings format_settings = getInputFormatSetting(settings, context); + if (!format_settings) + { + format_settings = getFormatSettings(context); + } RowInputFormatParams row_input_format_params; row_input_format_params.max_block_size = max_block_size; - row_input_format_params.allow_errors_num = format_settings.input_allow_errors_num; - row_input_format_params.allow_errors_ratio = format_settings.input_allow_errors_ratio; - row_input_format_params.callback = std::move(callback); + row_input_format_params.allow_errors_num = format_settings->input_allow_errors_num; + row_input_format_params.allow_errors_ratio = format_settings->input_allow_errors_ratio; row_input_format_params.max_execution_time = settings.max_execution_time; row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode; - auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, row_input_format_params, format_settings}; + auto input_creator_params = + ParallelParsingBlockInputStream::InputCreatorParams{sample, + row_input_format_params, *format_settings}; ParallelParsingBlockInputStream::Params params{buf, input_getter, input_creator_params, file_segmentation_engine, static_cast(settings.max_threads), @@ -200,13 +185,15 @@ BlockInputStreamPtr FormatFactory::getInput( return std::make_shared(params); } - auto format = getInputFormat(name, buf, sample, context, max_block_size, std::move(callback)); + auto format = getInputFormat(name, buf, sample, context, max_block_size, + format_settings); return std::make_shared(std::move(format)); } -BlockOutputStreamPtr FormatFactory::getOutput( - const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const bool ignore_no_row_delimiter) const +BlockOutputStreamPtr FormatFactory::getOutput(const String & name, + WriteBuffer & buf, const Block & sample, const Context & context, + WriteCallback callback, std::optional format_settings) const { if (!getCreators(name).output_processor_creator) { @@ -214,18 +201,23 @@ BlockOutputStreamPtr FormatFactory::getOutput( if (!output_getter) throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - const Settings & settings = context.getSettingsRef(); - FormatSettings format_settings = getOutputFormatSetting(settings, context); + if (!format_settings) + { + format_settings = getFormatSettings(context); + } /** Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ return std::make_shared( - output_getter(buf, sample, std::move(callback), format_settings), sample); + output_getter(buf, sample, std::move(callback), *format_settings), + sample); } - auto format = getOutputFormat(name, buf, sample, context, std::move(callback), ignore_no_row_delimiter); - return std::make_shared(std::make_shared(format), sample); + auto format = getOutputFormat(name, buf, sample, context, std::move(callback), + format_settings); + return std::make_shared( + std::make_shared(format), sample); } @@ -235,24 +227,27 @@ InputFormatPtr FormatFactory::getInputFormat( const Block & sample, const Context & context, UInt64 max_block_size, - ReadCallback callback) const + std::optional format_settings) const { const auto & input_getter = getCreators(name).input_processor_creator; if (!input_getter) throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); const Settings & settings = context.getSettingsRef(); - FormatSettings format_settings = getInputFormatSetting(settings, context); + + if (!format_settings) + { + format_settings = getFormatSettings(context); + } RowInputFormatParams params; params.max_block_size = max_block_size; - params.allow_errors_num = format_settings.input_allow_errors_num; - params.allow_errors_ratio = format_settings.input_allow_errors_ratio; - params.callback = std::move(callback); + params.allow_errors_num = format_settings->input_allow_errors_num; + params.allow_errors_ratio = format_settings->input_allow_errors_ratio; params.max_execution_time = settings.max_execution_time; params.timeout_overflow_mode = settings.timeout_overflow_mode; - auto format = input_getter(buf, sample, params, format_settings); + auto format = input_getter(buf, sample, params, *format_settings); /// It's a kludge. Because I cannot remove context from values format. if (auto * values = typeid_cast(format.get())) @@ -263,26 +258,29 @@ InputFormatPtr FormatFactory::getInputFormat( OutputFormatPtr FormatFactory::getOutputFormat( - const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const bool ignore_no_row_delimiter) const + const String & name, WriteBuffer & buf, const Block & sample, + const Context & context, WriteCallback callback, + std::optional format_settings) const { const auto & output_getter = getCreators(name).output_processor_creator; if (!output_getter) throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - const Settings & settings = context.getSettingsRef(); - FormatSettings format_settings = getOutputFormatSetting(settings, context); + if (!format_settings) + { + format_settings = getFormatSettings(context); + } RowOutputFormatParams params; - params.ignore_no_row_delimiter = ignore_no_row_delimiter; params.callback = std::move(callback); /** TODO: Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ - auto format = output_getter(buf, sample, params, format_settings); + auto format = output_getter(buf, sample, params, *format_settings); /// Enable auto-flush for streaming mode. Currently it is needed by INSERT WATCH query. - if (format_settings.enable_streaming) + if (format_settings->enable_streaming) format->setAutoFlush(); /// It's a kludge. Because I cannot remove context from MySQL format. diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index d49414e3944..619acd10e0f 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -16,6 +17,7 @@ namespace DB class Block; class Context; struct FormatSettings; +struct Settings; class ReadBuffer; class WriteBuffer; @@ -32,6 +34,7 @@ struct RowOutputFormatParams; using InputFormatPtr = std::shared_ptr; using OutputFormatPtr = std::shared_ptr; +FormatSettings getFormatSettings(const Context & context); /** Allows to create an IBlockInputStream or IBlockOutputStream by the name of the format. * Note: format and compression are independent things. @@ -105,10 +108,11 @@ public: const Block & sample, const Context & context, UInt64 max_block_size, - ReadCallback callback = {}) const; + std::optional format_settings = std::nullopt) const; BlockOutputStreamPtr getOutput(const String & name, WriteBuffer & buf, - const Block & sample, const Context & context, WriteCallback callback = {}, const bool ignore_no_row_delimiter = false) const; + const Block & sample, const Context & context, WriteCallback callback = {}, + std::optional format_settings = std::nullopt) const; InputFormatPtr getInputFormat( const String & name, @@ -116,10 +120,12 @@ public: const Block & sample, const Context & context, UInt64 max_block_size, - ReadCallback callback = {}) const; + std::optional format_settings = std::nullopt) const; OutputFormatPtr getOutputFormat( - const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, const bool ignore_no_row_delimiter = false) const; + const String & name, WriteBuffer & buf, const Block & sample, + const Context & context, WriteCallback callback = {}, + std::optional format_settings = std::nullopt) const; /// Register format by its name. void registerInputFormat(const String & name, InputCreator input_creator); diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 8d7c3cdb49f..d2a596bca4d 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -17,76 +17,6 @@ struct FormatSettings /// Option means that each chunk of data need to be formatted independently. Also each chunk will be flushed at the end of processing. bool enable_streaming = false; - struct JSON - { - bool quote_64bit_integers = true; - bool quote_denormals = true; - bool escape_forward_slashes = true; - }; - - JSON json; - - struct CSV - { - char delimiter = ','; - bool allow_single_quotes = true; - bool allow_double_quotes = true; - bool unquoted_null_literal_as_null = false; - bool empty_as_default = false; - bool crlf_end_of_line = false; - bool input_format_enum_as_number = false; - }; - - CSV csv; - - struct Pretty - { - UInt64 max_rows = 10000; - UInt64 max_column_pad_width = 250; - UInt64 max_value_width = 10000; - bool color = true; - - bool output_format_pretty_row_numbers = false; - - enum class Charset - { - UTF8, - ASCII, - }; - - Charset charset = Charset::UTF8; - }; - - Pretty pretty; - - struct Values - { - bool interpret_expressions = true; - bool deduce_templates_of_expressions = true; - bool accurate_types_of_literals = true; - }; - - Values values; - - struct Template - { - String resultset_format; - String row_format; - String row_between_delimiter; - }; - - Template template_settings; - - struct TSV - { - bool empty_as_default = false; - bool crlf_end_of_line = false; - String null_representation = "\\N"; - bool input_format_enum_as_number = false; - }; - - TSV tsv; - bool skip_unknown_fields = false; bool with_names_use_header = false; bool write_statistics = true; @@ -113,24 +43,29 @@ struct FormatSettings UInt64 input_allow_errors_num = 0; Float32 input_allow_errors_ratio = 0; - struct Arrow + struct { UInt64 row_group_size = 1000000; } arrow; - struct Parquet + struct { - UInt64 row_group_size = 1000000; - } parquet; + String schema_registry_url; + String output_codec; + UInt64 output_sync_interval = 16 * 1024; + bool allow_missing_fields = false; + } avro; - struct Schema + struct CSV { - std::string format_schema; - std::string format_schema_path; - bool is_server = false; - }; - - Schema schema; + char delimiter = ','; + bool allow_single_quotes = true; + bool allow_double_quotes = true; + bool unquoted_null_literal_as_null = false; + bool empty_as_default = false; + bool crlf_end_of_line = false; + bool input_format_enum_as_number = false; + } csv; struct Custom { @@ -141,29 +76,89 @@ struct FormatSettings std::string row_between_delimiter; std::string field_delimiter; std::string escaping_rule; - }; + } custom; - Custom custom; - - struct Avro + struct { - String schema_registry_url; - String output_codec; - UInt64 output_sync_interval = 16 * 1024; - bool allow_missing_fields = false; - }; + bool quote_64bit_integers = true; + bool quote_denormals = true; + bool escape_forward_slashes = true; + bool serialize_as_strings = false; + } json; - Avro avro; + struct + { + UInt64 row_group_size = 1000000; + } parquet; - struct Regexp + struct Pretty + { + UInt64 max_rows = 10000; + UInt64 max_column_pad_width = 250; + UInt64 max_value_width = 10000; + bool color = true; + + bool output_format_pretty_row_numbers = false; + + enum class Charset + { + UTF8, + ASCII, + }; + + Charset charset = Charset::UTF8; + } pretty; + + struct + { + bool write_row_delimiters = true; + /** + * Some buffers (kafka / rabbit) split the rows internally using callback + * so we can push there formats without framing / delimiters (like + * ProtobufSingle). In other cases you can't write more than single row + * in unframed format. + * Not sure we need this parameter at all, it only serves as an additional + * safety check in ProtobufSingle format, but exporting constant-size + * records w/o delimiters might be generally useful, not only for Kafka. + */ + bool allow_many_rows_no_delimiters = false; + } protobuf; + + struct { std::string regexp; std::string escaping_rule; bool skip_unmatched = false; - }; + } regexp; - Regexp regexp; + struct + { + std::string format_schema; + std::string format_schema_path; + bool is_server = false; + } schema; + struct + { + String resultset_format; + String row_format; + String row_between_delimiter; + } template_settings; + + struct + { + bool empty_as_default = false; + bool crlf_end_of_line = false; + String null_representation = "\\N"; + bool input_format_enum_as_number = false; + } tsv; + + struct + { + bool interpret_expressions = true; + bool deduce_templates_of_expressions = true; + bool accurate_types_of_literals = true; + } values; }; } diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index 12d4db1f4a8..48cfdb12d8b 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -63,8 +63,6 @@ Chunk IRowInputFormat::generate() info.read_columns.clear(); if (!readRow(columns, info)) break; - if (params.callback) - params.callback(); for (size_t column_idx = 0; column_idx < info.read_columns.size(); ++column_idx) { diff --git a/src/Processors/Formats/IRowOutputFormat.h b/src/Processors/Formats/IRowOutputFormat.h index 4312691ea5e..4fb94f7b7f7 100644 --- a/src/Processors/Formats/IRowOutputFormat.h +++ b/src/Processors/Formats/IRowOutputFormat.h @@ -15,14 +15,6 @@ struct RowOutputFormatParams // Callback used to indicate that another row is written. WriteCallback callback; - - /** - * some buffers (kafka / rabbit) split the rows internally using callback - * so we can push there formats without framing / delimiters - * (like ProtobufSingle). In other cases you can't write more than single row - * in unframed format. - */ - bool ignore_no_row_delimiter = false; }; class WriteBuffer; diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp index 930a83c52da..3c885e80e31 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp @@ -23,18 +23,22 @@ ProtobufRowOutputFormat::ProtobufRowOutputFormat( const Block & header, const RowOutputFormatParams & params_, const FormatSchemaInfo & format_schema, - const bool use_length_delimiters_) + const FormatSettings & settings) : IRowOutputFormat(header, out_, params_) , data_types(header.getDataTypes()) - , writer(out, ProtobufSchemas::instance().getMessageTypeForFormatSchema(format_schema), header.getNames(), use_length_delimiters_) - , throw_on_multiple_rows_undelimited(!use_length_delimiters_ && !params_.ignore_no_row_delimiter) + , writer(out, + ProtobufSchemas::instance().getMessageTypeForFormatSchema(format_schema), + header.getNames(), settings.protobuf.write_row_delimiters) + , allow_only_one_row( + !settings.protobuf.write_row_delimiters + && !settings.protobuf.allow_many_rows_no_delimiters) { value_indices.resize(header.columns()); } void ProtobufRowOutputFormat::write(const Columns & columns, size_t row_num) { - if (throw_on_multiple_rows_undelimited && !first_row) + if (allow_only_one_row && !first_row) { throw Exception("The ProtobufSingle format can't be used to write multiple rows because this format doesn't have any row delimiter.", ErrorCodes::NO_ROW_DELIMITER); } @@ -51,19 +55,23 @@ void ProtobufRowOutputFormat::write(const Columns & columns, size_t row_num) void registerOutputFormatProcessorProtobuf(FormatFactory & factory) { - for (bool use_length_delimiters : {false, true}) + for (bool write_row_delimiters : {false, true}) { factory.registerOutputFormatProcessor( - use_length_delimiters ? "Protobuf" : "ProtobufSingle", - [use_length_delimiters](WriteBuffer & buf, + write_row_delimiters ? "Protobuf" : "ProtobufSingle", + [write_row_delimiters](WriteBuffer & buf, const Block & header, const RowOutputFormatParams & params, - const FormatSettings & settings) + const FormatSettings & _settings) { - return std::make_shared(buf, header, params, - FormatSchemaInfo(settings.schema.format_schema, "Protobuf", true, - settings.schema.is_server, settings.schema.format_schema_path), - use_length_delimiters); + FormatSettings settings = _settings; + settings.protobuf.write_row_delimiters = write_row_delimiters; + return std::make_shared( + buf, header, params, + FormatSchemaInfo(settings.schema.format_schema, "Protobuf", + true, settings.schema.is_server, + settings.schema.format_schema_path), + settings); }); } } diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h index 740efcfa24c..847f7607ff5 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h @@ -41,7 +41,7 @@ public: const Block & header, const RowOutputFormatParams & params_, const FormatSchemaInfo & format_schema, - const bool use_length_delimiters_); + const FormatSettings & settings); String getName() const override { return "ProtobufRowOutputFormat"; } @@ -53,7 +53,7 @@ private: DataTypes data_types; ProtobufWriter writer; std::vector value_indices; - const bool throw_on_multiple_rows_undelimited; + const bool allow_only_one_row; }; } diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index de5a1b71580..c3b753e7261 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -54,8 +54,6 @@ Chunk ValuesBlockInputFormat::generate() if (buf.eof() || *buf.position() == ';') break; readRow(columns, rows_in_block); - if (params.callback) - params.callback(); } catch (Exception & e) { diff --git a/src/Storages/Kafka/KafkaBlockOutputStream.cpp b/src/Storages/Kafka/KafkaBlockOutputStream.cpp index 9d7fe465d44..e7bf562339f 100644 --- a/src/Storages/Kafka/KafkaBlockOutputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockOutputStream.cpp @@ -32,13 +32,16 @@ void KafkaBlockOutputStream::writePrefix() if (!buffer) throw Exception("Failed to create Kafka producer!", ErrorCodes::CANNOT_CREATE_IO_BUFFER); - child = FormatFactory::instance().getOutput( - storage.getFormatName(), *buffer, getHeader(), *context, [this](const Columns & columns, size_t row) - { - buffer->countRow(columns, row); - }, - /* ignore_no_row_delimiter = */ true - ); + auto format_settings = getFormatSettings(*context); + format_settings.protobuf.allow_many_rows_no_delimiters = true; + + child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer, + getHeader(), *context, + [this](const Columns & columns, size_t row) + { + buffer->countRow(columns, row); + }, + format_settings); } void KafkaBlockOutputStream::write(const Block & block) diff --git a/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp b/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp index 76129dee30d..b3bd57bdd0b 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp +++ b/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp @@ -42,13 +42,16 @@ void RabbitMQBlockOutputStream::writePrefix() buffer->activateWriting(); - child = FormatFactory::instance().getOutput( - storage.getFormatName(), *buffer, getHeader(), context, [this](const Columns & /* columns */, size_t /* rows */) - { - buffer->countRow(); - }, - /* ignore_no_row_delimiter = */ true - ); + auto format_settings = getFormatSettings(context); + format_settings.protobuf.allow_many_rows_no_delimiters = true; + + child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer, + getHeader(), context, + [this](const Columns & /* columns */, size_t /* rows */) + { + buffer->countRow(); + }, + format_settings); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 8c7cd7b63d7..aa2696493f7 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -202,6 +203,7 @@ StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArgu StorageFile::StorageFile(CommonArguments args) : IStorage(args.table_id) , format_name(args.format_name) + , format_settings(args.format_settings) , compression_method(args.compression_method) , base_path(args.context.getPath()) { @@ -324,9 +326,11 @@ public: method = chooseCompressionMethod(current_path, storage->compression_method); } - read_buf = wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method); - reader = FormatFactory::instance().getInput( - storage->format_name, *read_buf, metadata_snapshot->getSampleBlock(), context, max_block_size); + read_buf = wrapReadBufferWithCompressionMethod( + std::move(nested_buffer), method); + reader = FormatFactory::instance().getInput(storage->format_name, + *read_buf, metadata_snapshot->getSampleBlock(), context, + max_block_size, storage->format_settings); if (columns_description.hasDefaults()) reader = std::make_shared(reader, columns_description, context); @@ -430,8 +434,11 @@ Pipe StorageFile::read( pipes.reserve(num_streams); for (size_t i = 0; i < num_streams; ++i) + { pipes.emplace_back(std::make_shared( - this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns())); + this_ptr, metadata_snapshot, context, max_block_size, files_info, + metadata_snapshot->getColumns())); + } return Pipe::unitePipes(std::move(pipes)); } @@ -444,7 +451,8 @@ public: StorageFile & storage_, const StorageMetadataPtr & metadata_snapshot_, const CompressionMethod compression_method, - const Context & context) + const Context & context, + const FormatSettings & format_settings) : storage(storage_) , metadata_snapshot(metadata_snapshot_) , lock(storage.rwlock) @@ -472,7 +480,9 @@ public: write_buf = wrapWriteBufferWithCompressionMethod(std::move(naked_buffer), compression_method, 3); - writer = FormatFactory::instance().getOutput(storage.format_name, *write_buf, metadata_snapshot->getSampleBlock(), context); + writer = FormatFactory::instance().getOutput(storage.format_name, + *write_buf, metadata_snapshot->getSampleBlock(), context, + {}, format_settings); } Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } @@ -521,7 +531,8 @@ BlockOutputStreamPtr StorageFile::write( path = paths[0]; return std::make_shared(*this, metadata_snapshot, - chooseCompressionMethod(path, compression_method), context); + chooseCompressionMethod(path, compression_method), context, + format_settings); } Strings StorageFile::getDataPaths() const @@ -581,32 +592,54 @@ void StorageFile::truncate( void registerStorageFile(StorageFactory & factory) { + StorageFactory::StorageFeatures storage_features{ + .supports_settings = true, + .source_access_type = AccessType::FILE + }; + factory.registerStorage( "File", - [](const StorageFactory::Arguments & args) + [](const StorageFactory::Arguments & factory_args) { - ASTs & engine_args = args.engine_args; + StorageFile::CommonArguments storage_args{ + .table_id = factory_args.table_id, + .columns = factory_args.columns, + .constraints = factory_args.constraints, + .context = factory_args.context + }; - if (!(engine_args.size() >= 1 && engine_args.size() <= 3)) // NOLINT + ASTs & engine_args_ast = factory_args.engine_args; + + if (!(engine_args_ast.size() >= 1 && engine_args_ast.size() <= 3)) // NOLINT throw Exception( "Storage File requires from 1 to 3 arguments: name of used format, source and compression_method.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context); - String format_name = engine_args[0]->as().value.safeGet(); + engine_args_ast[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[0], factory_args.local_context); + storage_args.format_name = engine_args_ast[0]->as().value.safeGet(); - String compression_method; - StorageFile::CommonArguments common_args{ - args.table_id, format_name, compression_method, args.columns, args.constraints, args.context}; + if (factory_args.storage_def->settings) + { + Context local_context_copy = factory_args.local_context; + local_context_copy.applySettingsChanges( + factory_args.storage_def->settings->changes); + storage_args.format_settings = getFormatSettings( + local_context_copy); + } + else + { + storage_args.format_settings = getFormatSettings( + factory_args.local_context); + } - if (engine_args.size() == 1) /// Table in database - return StorageFile::create(args.relative_data_path, common_args); + if (engine_args_ast.size() == 1) /// Table in database + return StorageFile::create(factory_args.relative_data_path, storage_args); /// Will use FD if engine_args[1] is int literal or identifier with std* name int source_fd = -1; String source_path; - if (auto opt_name = tryGetIdentifierName(engine_args[1])) + if (auto opt_name = tryGetIdentifierName(engine_args_ast[1])) { if (*opt_name == "stdin") source_fd = STDIN_FILENO; @@ -618,7 +651,7 @@ void registerStorageFile(StorageFactory & factory) throw Exception( "Unknown identifier '" + *opt_name + "' in second arg of File storage constructor", ErrorCodes::UNKNOWN_IDENTIFIER); } - else if (const auto * literal = engine_args[1]->as()) + else if (const auto * literal = engine_args_ast[1]->as()) { auto type = literal->value.getType(); if (type == Field::Types::Int64) @@ -631,23 +664,23 @@ void registerStorageFile(StorageFactory & factory) throw Exception("Second argument must be path or file descriptor", ErrorCodes::BAD_ARGUMENTS); } - if (engine_args.size() == 3) + if (engine_args_ast.size() == 3) { - engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.local_context); - compression_method = engine_args[2]->as().value.safeGet(); + engine_args_ast[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[2], factory_args.local_context); + storage_args.compression_method = engine_args_ast[2]->as().value.safeGet(); } else - compression_method = "auto"; + storage_args.compression_method = "auto"; if (0 <= source_fd) /// File descriptor - return StorageFile::create(source_fd, common_args); + return StorageFile::create(source_fd, storage_args); else /// User's file - return StorageFile::create(source_path, args.context.getUserFilesPath(), common_args); + return StorageFile::create(source_path, factory_args.context.getUserFilesPath(), storage_args); }, - { - .source_access_type = AccessType::FILE, - }); + storage_features); } + + NamesAndTypesList StorageFile::getVirtuals() const { return NamesAndTypesList{ diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index f331538b4c7..695cd0d3912 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -50,9 +50,10 @@ public: struct CommonArguments { - const StorageID & table_id; - const std::string & format_name; - const std::string & compression_method; + StorageID table_id; + std::string format_name; + FormatSettings format_settings; + std::string compression_method; const ColumnsDescription & columns; const ConstraintsDescription & constraints; const Context & context; @@ -79,6 +80,7 @@ private: explicit StorageFile(CommonArguments args); std::string format_name; + FormatSettings format_settings; int table_fd = -1; String compression_method; diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 39de6dce92c..f7d76309a7f 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -1,10 +1,12 @@ -#include -#include -#include -#include #include -#include + #include "registerTableFunctions.h" +#include +#include +#include +#include +#include +#include namespace DB { @@ -12,7 +14,13 @@ StoragePtr TableFunctionFile::getStorage( const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const std::string & compression_method_) const { - StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format_, compression_method_, columns, ConstraintsDescription{}, global_context}; + StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), + format_, + getFormatSettings(global_context), + compression_method_, + columns, + ConstraintsDescription{}, + global_context}; return StorageFile::create(source, global_context.getUserFilesPath(), args); } diff --git a/tests/queries/0_stateless/01544_file_engine_settings.reference b/tests/queries/0_stateless/01544_file_engine_settings.reference new file mode 100644 index 00000000000..d2afb8fc688 --- /dev/null +++ b/tests/queries/0_stateless/01544_file_engine_settings.reference @@ -0,0 +1,2 @@ +1|1 +1 1 diff --git a/tests/queries/0_stateless/01544_file_engine_settings.sh b/tests/queries/0_stateless/01544_file_engine_settings.sh new file mode 100755 index 00000000000..b13ec0f3db3 --- /dev/null +++ b/tests/queries/0_stateless/01544_file_engine_settings.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eu + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +the_file="$CLICKHOUSE_TMP/01544-t.csv" +rm -f -- "$the_file" + +# We are going to check that format settings work for File engine, +# by creating a table with a non-default delimiter, and reading from it. +${CLICKHOUSE_LOCAL} --query " + create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|'; + insert into t select 1 a, 1 b; +" + +# See what's in the file +cat "$the_file" + +${CLICKHOUSE_LOCAL} --query " + create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|'; + select * from t; +" From db1e9750b935221b9ece4743a8e381a1215872c3 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Nov 2020 17:35:06 +0300 Subject: [PATCH 041/114] fix build --- src/Formats/tests/tab_separated_streams.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Formats/tests/tab_separated_streams.cpp b/src/Formats/tests/tab_separated_streams.cpp index bb38d51cf98..bd733e4b9aa 100644 --- a/src/Formats/tests/tab_separated_streams.cpp +++ b/src/Formats/tests/tab_separated_streams.cpp @@ -38,8 +38,8 @@ try FormatSettings format_settings; - RowInputFormatParams in_params{DEFAULT_INSERT_BLOCK_SIZE, 0, 0, []{}}; - RowOutputFormatParams out_params{[](const Columns & /* columns */, size_t /* row */){},false}; + RowInputFormatParams in_params{DEFAULT_INSERT_BLOCK_SIZE, 0, 0}; + RowOutputFormatParams out_params{[](const Columns & /* columns */, size_t /* row */){}}; InputFormatPtr input_format = std::make_shared(sample, in_buf, in_params, false, false, format_settings); BlockInputStreamPtr block_input = std::make_shared(std::move(input_format)); From 0f22a9dfcdf183fd81f762e0b0bf844b32d94b29 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Nov 2020 19:13:42 +0300 Subject: [PATCH 042/114] fix build --- src/Processors/Formats/IRowInputFormat.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index 1931fba2a0d..14109f9c2be 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -27,9 +27,6 @@ struct RowInputFormatParams UInt64 allow_errors_num; Float64 allow_errors_ratio; - using ReadCallback = std::function; - ReadCallback callback; - Poco::Timespan max_execution_time = 0; OverflowMode timeout_overflow_mode = OverflowMode::THROW; }; From 364607d87d5e2dd417466f26315e6059ae118014 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 2 Nov 2020 19:18:18 +0300 Subject: [PATCH 043/114] Merge with master --- src/Storages/StorageProxy.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 7b010476b22..bb5223d0e3d 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -102,12 +102,11 @@ public: } Pipe alterPartition( - const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const PartitionCommands & commands, const Context & context) override { - return getNested()->alterPartition(query, metadata_snapshot, commands, context); + return getNested()->alterPartition(metadata_snapshot, commands, context); } void checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & metadata_snapshot, const Settings & settings) const override From dac670599521b5d86510b7109430d8ed50eb9f6b Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 2 Nov 2020 20:30:53 +0300 Subject: [PATCH 044/114] DROP PART and clear blocks in a single zookeeper op --- src/Storages/MergeTree/MergeTreeData.cpp | 8 +++ src/Storages/MergeTree/MergeTreeData.h | 2 + src/Storages/StorageMergeTree.cpp | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 54 ++++++++++----------- src/Storages/StorageReplicatedMergeTree.h | 1 + 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index cd776a661ed..fb7a7c02078 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2642,6 +2642,14 @@ void MergeTreeData::checkPartitionCanBeDropped(const ASTPtr & partition) global_context.checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, partition_size); } +void MergeTreeData::checkPartCanBeDropped(const ASTPtr & part_ast) +{ + String part_name = part_ast->as().value.safeGet(); + auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Committed}); + if (!part) + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in commited state", part_name); +} + void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & name, bool moving_part, const Context & context) { String partition_id; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 889e4fb16b4..b5506d87a8b 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -560,6 +560,8 @@ public: void checkPartitionCanBeDropped(const ASTPtr & partition) override; + void checkPartCanBeDropped(const ASTPtr & part); + size_t getColumnCompressedSize(const std::string & name) const { auto lock = lockParts(); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 0c5fb5d9e8c..844884714ab 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1117,9 +1117,7 @@ Pipe StorageMergeTree::alterPartition( { case PartitionCommand::DROP_PARTITION: if (command.part) - { - /// TODO(nv) what would be a good check here? - } + checkPartCanBeDropped(command.partition); else checkPartitionCanBeDropped(command.partition); dropPartition(command.partition, command.detach, command.part, query_context); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 1cbb2fe800f..6dae5a2d7d5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4191,9 +4191,7 @@ Pipe StorageReplicatedMergeTree::alterPartition( { case PartitionCommand::DROP_PARTITION: if (command.part) - { - /// TODO(nv) what to check here? it is possible to drop a big partition by dropping small parts... - } + checkPartCanBeDropped(command.partition); else checkPartitionCanBeDropped(command.partition); dropPartition(command.partition, command.detach, command.part, query_context); @@ -5479,8 +5477,8 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( } -void StorageReplicatedMergeTree::clearBlocksInPartition( - zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num) +void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( + Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num) { Strings blocks; if (Coordination::Error::ZOK != zookeeper.tryGetChildren(zookeeper_path + "/blocks", blocks)) @@ -5497,7 +5495,6 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( } } - zkutil::AsyncResponses to_delete_futures; for (auto & pair : get_futures) { const String & path = pair.first; @@ -5510,23 +5507,25 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( Int64 block_num = 0; bool parsed = tryReadIntText(block_num, buf) && buf.eof(); if (!parsed || (min_block_num <= block_num && block_num <= max_block_num)) - to_delete_futures.emplace_back(path, zookeeper.asyncTryRemove(path)); + ops.emplace_back(zkutil::makeRemoveRequest(path, -1)); } +} - for (auto & pair : to_delete_futures) +void StorageReplicatedMergeTree::clearBlocksInPartition( + zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num) +{ + Coordination::Requests delete_requests; + getClearBlocksInPartitionOps(delete_requests, zookeeper, partition_id, min_block_num, max_block_num); + Coordination::Responses delete_responses; + auto code = zookeeper.tryMulti(delete_requests, delete_responses); + if (code != Coordination::Error::ZOK) { - const String & path = pair.first; - Coordination::Error rc = pair.second.get().error; - if (rc == Coordination::Error::ZNOTEMPTY) - { - /// Can happen if there are leftover block nodes with children created by previous server versions. - zookeeper.removeRecursive(path); - } - else if (rc != Coordination::Error::ZOK) - LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, Coordination::errorMessage(rc)); + for (size_t i = 0; i < delete_requests.size(); ++i) + if (delete_responses[i]->error != Coordination::Error::ZOK) + LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", delete_requests[i]->getPath(), Coordination::errorMessage(delete_responses[i]->error)); } - LOG_TRACE(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id); + LOG_TRACE(log, "Deleted {} deduplication block IDs in partition ID {}", delete_requests.size(), partition_id); } void StorageReplicatedMergeTree::replacePartitionFrom( @@ -6053,22 +6052,22 @@ bool StorageReplicatedMergeTree::dropPart( { ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper); - /// TODO(nv) It is possible that part does not exist on replica which executes this statement. - /// Also, it possible for the part to not exist on any replicas, replica which created log entries for the part disappeared. - auto part = data_parts_by_info.find(part_info); - if (part == data_parts_by_info.end()) - throw Exception("Part " + part_name + " not found locally, won't try to drop it.", ErrorCodes::NOT_IMPLEMENTED); + auto part = getPartIfExists(part_info, {MergeTreeDataPartState::Committed}); - /// TODO(nv) get ops and commit together w/ log entry? - clearBlocksInPartition(*zookeeper, part_info.partition_id, part_info.min_block, part_info.max_block); + if (!part) + throw Exception("Part " + part_name + " not found locally, won't try to drop it.", ErrorCodes::NOT_IMPLEMENTED); /// There isn't a lot we can do otherwise. Can't cancel merges because it is possible that a replica already /// finished the merge. - if (partIsAssignedToBackgroundOperation(*part)) + if (partIsAssignedToBackgroundOperation(part)) throw Exception("Part " + part_name + " is currently participating in a background operation (mutation/merge)" + ", try again later.", ErrorCodes::PART_IS_TEMPORARILY_LOCKED); + Coordination::Requests ops; + getClearBlocksInPartitionOps(ops, *zookeeper, part_info.partition_id, part_info.min_block, part_info.max_block); + size_t clean_block_ops_size = ops.size(); + /// If `part_name` is result of a recent merge and source parts are still available then /// DROP_RANGE with detach will move this part together with source parts to `detached/` dir. entry.type = LogEntry::DROP_RANGE; @@ -6077,7 +6076,6 @@ bool StorageReplicatedMergeTree::dropPart( entry.detach = detach; entry.create_time = time(nullptr); - Coordination::Requests ops; ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/log", merge_pred.getVersion())); /// Make sure no new events were added to the log. ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); /// Just update version. @@ -6092,7 +6090,7 @@ bool StorageReplicatedMergeTree::dropPart( else zkutil::KeeperMultiException::check(rc, ops, responses); - String log_znode_path = dynamic_cast(*responses[1]).path_created; + String log_znode_path = dynamic_cast(*responses[clean_block_ops_size + 1]).path_created; entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); return true; diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index bf02b6ae05b..2fc7848d87b 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -535,6 +535,7 @@ private: std::mutex existing_nodes_cache_mutex; bool existsNodeCached(const std::string & path); + void getClearBlocksInPartitionOps(Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num); /// Remove block IDs from `blocks/` in ZooKeeper for the given partition ID in the given block number range. void clearBlocksInPartition( zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num); From da8ac948bfb1b329bcc3d479981177ad0b2344f2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Nov 2020 12:24:10 +0300 Subject: [PATCH 045/114] Disallow to drop parts with parallel quorum --- src/Storages/StorageMergeTree.cpp | 7 +- src/Storages/StorageReplicatedMergeTree.cpp | 94 ++++++------------- src/Storages/StorageReplicatedMergeTree.h | 6 +- .../0_stateless/01451_detach_drop_part.sql | 2 + ...eplicated_detach_drop_and_quorum.reference | 5 + ...1451_replicated_detach_drop_and_quorum.sql | 49 ++++++++++ .../01451_replicated_detach_drop_part.sql | 2 + 7 files changed, 98 insertions(+), 67 deletions(-) create mode 100644 tests/queries/0_stateless/01451_replicated_detach_drop_and_quorum.reference create mode 100644 tests/queries/0_stateless/01451_replicated_detach_drop_and_quorum.sql diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 844884714ab..8f32d784055 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -42,6 +42,7 @@ namespace ErrorCodes extern const int CANNOT_ASSIGN_OPTIMIZE; extern const int TIMEOUT_EXCEEDED; extern const int UNKNOWN_POLICY; + extern const int NO_SUCH_DATA_PART; } namespace ActionLocks @@ -1229,7 +1230,6 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool MergeTreeData::DataPartsVector parts_to_remove; - /// TODO: should we include PreComitted parts like in Replicated case? if (drop_part) { String part_name = partition->as().value.safeGet(); @@ -1237,7 +1237,10 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool if (part) parts_to_remove.push_back(part); - } else + else + throw Exception("Part " + part_name + " not found, won't try to drop it.", ErrorCodes::NO_SUCH_DATA_PART); + } + else { String partition_id = getPartitionIDFromQuery(partition, context); parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a8e2df4e8f9..2f3d3ae1010 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -115,6 +115,7 @@ namespace ErrorCodes extern const int DIRECTORY_ALREADY_EXISTS; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int UNKNOWN_POLICY; + extern const int NO_SUCH_DATA_PART; } namespace ActionLocks @@ -3275,61 +3276,34 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) } } -void StorageReplicatedMergeTree::updateLastPartNodeIfMatches(const String & partition_id, const String & old_part_name, const String & new_part_name) + +bool StorageReplicatedMergeTree::partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const +{ + auto zookeeper = getZooKeeper(); + return zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_info.getPartName()); +} + +bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & part_info) const { auto zookeeper = getZooKeeper(); - const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + const String parts_with_quorum_path = zookeeper_path + "/quorum/last_part"; - while (true) - { - Coordination::Stat added_parts_stat; - String old_added_parts = zookeeper->get(quorum_last_part_path, &added_parts_stat); + String parts_with_quorum_str = zookeeper->get(parts_with_quorum_path); - ReplicatedMergeTreeQuorumAddedParts parts_with_quorum(format_version); + if (parts_with_quorum_str.empty()) + return false; - if (!old_added_parts.empty()) - parts_with_quorum.fromString(old_added_parts); + ReplicatedMergeTreeQuorumAddedParts parts_with_quorum(format_version); + parts_with_quorum.fromString(parts_with_quorum_str); - if (!parts_with_quorum.added_parts.count(partition_id)) - { - /// There is no information about partition at all. - break; - } + auto partition_it = parts_with_quorum.added_parts.find(part_info.partition_id); + if (partition_it == parts_with_quorum.added_parts.end()) + return false; - /// Part for which last quorum was reached in partition_id. - auto quorum_part_info = MergeTreePartInfo::fromPartName(parts_with_quorum.added_parts.at(partition_id), format_version); - auto old_part_info = MergeTreePartInfo::fromPartName(old_part_name, format_version); - - /// Update last part for which quorum was reached. - if (old_part_info.contains(quorum_part_info)) - parts_with_quorum.added_parts.emplace(partition_id, new_part_name); - - /// Serialize and try update. - String new_added_parts = parts_with_quorum.toString(); - - auto code = zookeeper->trySet(quorum_last_part_path, new_added_parts, added_parts_stat.version); - - if (code == Coordination::Error::ZOK) - { - break; - } - else if (code == Coordination::Error::ZNONODE) - { - /// Node is deleted. It is impossible, but it is Ok. - break; - } - else if (code == Coordination::Error::ZBADVERSION) - { - /// Node was updated meanwhile. We must re-read it and repeat all the actions. - continue; - } - else - throw Coordination::Exception(code, quorum_last_part_path); - } + return partition_it->second == part_info.getPartName(); } - bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot, const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_) { @@ -4361,23 +4335,7 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool de } } - bool drop_entire_partition = !drop_part; - - /// Cleaning possibly stored information about parts from /quorum/last_part node in ZooKeeper. - if (drop_part) - { - auto part_info = MergeTreePartInfo::fromPartName(partition->as().value.safeGet(), format_version); - auto data_parts_vec = getDataPartsVectorInPartition(DataPartState::Committed, part_info.partition_id); - std::sort(data_parts_vec.begin(), data_parts_vec.end(), LessDataPart()); - - auto prev_part = std::upper_bound(data_parts_vec.begin(), data_parts_vec.end(), part_info, LessDataPart()); - if (prev_part != data_parts_vec.end()) - updateLastPartNodeIfMatches(part_info.partition_id, part_info.getPartName(), (*prev_part)->info.getPartName()); - else if (data_parts_vec.empty()) - drop_entire_partition = true; - } - - if (drop_entire_partition) + if (!drop_part) { String partition_id = getPartitionIDFromQuery(partition, query_context); cleanLastPartNode(partition_id); @@ -6058,14 +6016,22 @@ bool StorageReplicatedMergeTree::dropPart( auto part = getPartIfExists(part_info, {MergeTreeDataPartState::Committed}); if (!part) - throw Exception("Part " + part_name + " not found locally, won't try to drop it.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Part " + part_name + " not found locally, won't try to drop it.", ErrorCodes::NO_SUCH_DATA_PART); /// There isn't a lot we can do otherwise. Can't cancel merges because it is possible that a replica already /// finished the merge. if (partIsAssignedToBackgroundOperation(part)) throw Exception("Part " + part_name + " is currently participating in a background operation (mutation/merge)" - + ", try again later.", ErrorCodes::PART_IS_TEMPORARILY_LOCKED); + + ", try again later", ErrorCodes::PART_IS_TEMPORARILY_LOCKED); + + if (partIsLastQuorumPart(part->info)) + throw Exception("Part " + part_name + " is last inserted part with quorum in partition. Cannot drop", + ErrorCodes::NOT_IMPLEMENTED); + + if (partIsInsertingWithParallelQuorum(part->info)) + throw Exception("Part " + part_name + " is inserting with parallel quorum. Cannot drop", + ErrorCodes::NOT_IMPLEMENTED); Coordination::Requests ops; getClearBlocksInPartitionOps(ops, *zookeeper, part_info.partition_id, part_info.min_block, part_info.max_block); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 2fc7848d87b..b72c3344f3e 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -502,7 +502,11 @@ private: /// Deletes info from quorum/last_part node for particular partition_id. void cleanLastPartNode(const String & partition_id); - void updateLastPartNodeIfMatches(const String & partition_id, const String & old_part_name, const String & new_part_name); + /// Part name is stored in quorum/last_part for corresponding partition_id. + bool partIsLastQuorumPart(const MergeTreePartInfo & part_info) const; + + /// Part currently inserting with quorum (node quorum/parallel/part_name exists) + bool partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const; /// Creates new block number if block with such block_id does not exist std::optional allocateBlockNumber( diff --git a/tests/queries/0_stateless/01451_detach_drop_part.sql b/tests/queries/0_stateless/01451_detach_drop_part.sql index 47e61f2d924..7a2815f9a3e 100644 --- a/tests/queries/0_stateless/01451_detach_drop_part.sql +++ b/tests/queries/0_stateless/01451_detach_drop_part.sql @@ -9,6 +9,8 @@ INSERT INTO mt VALUES (2); SELECT v FROM mt ORDER BY v; +ALTER TABLE mt DETACH PART 'all_100_100_0'; -- { serverError 232 } + ALTER TABLE mt DETACH PART 'all_2_2_0'; SELECT v FROM mt ORDER BY v; diff --git a/tests/queries/0_stateless/01451_replicated_detach_drop_and_quorum.reference b/tests/queries/0_stateless/01451_replicated_detach_drop_and_quorum.reference new file mode 100644 index 00000000000..3768ac7b852 --- /dev/null +++ b/tests/queries/0_stateless/01451_replicated_detach_drop_and_quorum.reference @@ -0,0 +1,5 @@ +all_0_0_0 +all_2_2_0 +1 +all_2_2_0 +1 diff --git a/tests/queries/0_stateless/01451_replicated_detach_drop_and_quorum.sql b/tests/queries/0_stateless/01451_replicated_detach_drop_and_quorum.sql new file mode 100644 index 00000000000..fca14b81e27 --- /dev/null +++ b/tests/queries/0_stateless/01451_replicated_detach_drop_and_quorum.sql @@ -0,0 +1,49 @@ +SET replication_alter_partitions_sync = 2; + + +DROP TABLE IF EXISTS replica1; +DROP TABLE IF EXISTS replica2; + +CREATE TABLE replica1 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/quorum', 'r1') order by tuple() settings max_replicated_merges_in_queue = 0; +CREATE TABLE replica2 (v UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/01451/quorum', 'r2') order by tuple() settings max_replicated_merges_in_queue = 0; + +INSERT INTO replica1 VALUES (0); + +SYSTEM SYNC REPLICA replica2; + +SELECT name FROM system.parts WHERE table = 'replica2' and database = currentDatabase() and active = 1; + +ALTER TABLE replica2 DETACH PART 'all_0_0_0'; + +SELECT * FROM replica1; + +SELECT * FROM replica2; + +-- drop of empty partition works +ALTER TABLE replica2 DROP PARTITION ID 'all'; + +SET insert_quorum=2; + +INSERT INTO replica2 VALUES (1); + +SYSTEM SYNC REPLICA replica2; + +ALTER TABLE replica1 DETACH PART 'all_2_2_0'; --{serverError 48} + +SELECT name FROM system.parts WHERE table = 'replica1' and database = currentDatabase() and active = 1 ORDER BY name; + +SELECT COUNT() FROM replica1; + +SET insert_quorum_parallel=1; + +INSERT INTO replica2 VALUES (2); + +-- should work, parallel quorum nodes exists only during insert +ALTER TABLE replica1 DROP PART 'all_3_3_0'; + +SELECT name FROM system.parts WHERE table = 'replica1' and database = currentDatabase() and active = 1 ORDER BY name; + +SELECT COUNT() FROM replica1; + +DROP TABLE IF EXISTS replica1; +DROP TABLE IF EXISTS replica2; diff --git a/tests/queries/0_stateless/01451_replicated_detach_drop_part.sql b/tests/queries/0_stateless/01451_replicated_detach_drop_part.sql index 1209f11b68e..3cd9fc7bc7e 100644 --- a/tests/queries/0_stateless/01451_replicated_detach_drop_part.sql +++ b/tests/queries/0_stateless/01451_replicated_detach_drop_part.sql @@ -10,6 +10,8 @@ INSERT INTO replica1 VALUES (0); INSERT INTO replica1 VALUES (1); INSERT INTO replica1 VALUES (2); +ALTER TABLE replica1 DETACH PART 'all_100_100_0'; -- { serverError 232 } + SELECT v FROM replica1 ORDER BY v; SYSTEM SYNC REPLICA replica2; From faca9d9b91982eaecb1f46ee31d89bfc75d8ae39 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Nov 2020 18:41:46 +0300 Subject: [PATCH 046/114] Check max bytes on disk --- src/Storages/MergeTree/MergeTreeData.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index fb7a7c02078..a06fbc338a4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2648,6 +2648,9 @@ void MergeTreeData::checkPartCanBeDropped(const ASTPtr & part_ast) auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Committed}); if (!part) throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in commited state", part_name); + + auto table_id = getStorageID(); + global_context.checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, part->getBytesOnDisk()); } void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & name, bool moving_part, const Context & context) From fb5e03ebaf163846d37e57df3a94d74ba9b82416 Mon Sep 17 00:00:00 2001 From: taichong Date: Thu, 5 Nov 2020 17:01:14 +0800 Subject: [PATCH 047/114] sync MySQL DDL atomicly Now ClickHouse will write metadata and then execute DDL from MySQL. But If ClickHouse is crashed and it just begin to execute DDL, we restart the server and then the sync will may be stopped. e.g.: MySQL : create table foo.a(id int key); uuid1:1 ClickHouse : write uuid1:1 to .metadata -> server crashed; Because of ClickHouse's metadata records: uuid1:1, if MySQL run a query:`insert into foo.a values(1);` ClickHouse will throw a exception: the table foo.a is not exist. --- .../MySQL/MaterializeMySQLSyncThread.cpp | 49 ++++++++++++------- .../MySQL/MaterializeMySQLSyncThread.h | 1 + 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp index e26f5c2fd52..f4e10757dfb 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp @@ -619,24 +619,7 @@ void MaterializeMySQLSyncThread::onEvent(Buffers & buffers, const BinlogEventPtr else if (receive_event->type() == MYSQL_QUERY_EVENT) { QueryEvent & query_event = static_cast(*receive_event); - flushBuffersData(buffers, metadata); - - try - { - Context query_context = createQueryContext(global_context); - String comment = "Materialize MySQL step 2: execute MySQL DDL for sync data"; - String event_database = query_event.schema == mysql_database_name ? database_name : ""; - tryToExecuteQuery(query_prefix + query_event.query, query_context, event_database, comment); - } - catch (Exception & exception) - { - tryLogCurrentException(log); - - /// If some DDL query was not successfully parsed and executed - /// Then replication may fail on next binlog events anyway - if (exception.code() != ErrorCodes::SYNTAX_ERROR) - throw; - } + metadata.transaction(client.getPosition(),[&](){ executeDDLAtomic(buffers, query_event); }); } else if (receive_event->header.type != HEARTBEAT_EVENT) { @@ -651,6 +634,36 @@ void MaterializeMySQLSyncThread::onEvent(Buffers & buffers, const BinlogEventPtr } } +void MaterializeMySQLSyncThread::executeDDLAtomic(Buffers & buffers, const QueryEvent & query_event) +{ + buffers.commit(global_context); + + try + { + Context query_context = createQueryContext(global_context); + String comment = "Materialize MySQL step 2: execute MySQL DDL for sync data"; + String event_database = query_event.schema == mysql_database_name ? database_name : ""; + tryToExecuteQuery(query_prefix + query_event.query, query_context, event_database, comment); + + const auto & position_message = [&]() + { + std::stringstream ss; + client.getPosition().dump(ss); + return ss.str(); + }; + LOG_INFO(log, "MySQL executed position: \n {}", position_message()); + } + catch (Exception & exception) + { + tryLogCurrentException(log); + + /// If some DDL query was not successfully parsed and executed + /// Then replication may fail on next binlog events anyway + if (exception.code() != ErrorCodes::SYNTAX_ERROR) + throw; + } +} + bool MaterializeMySQLSyncThread::isMySQLSyncThread() { return getThreadName() == MYSQL_BACKGROUND_THREAD_NAME; diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.h b/src/Databases/MySQL/MaterializeMySQLSyncThread.h index 0327690f1b0..44f6bb77d79 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.h +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.h @@ -100,6 +100,7 @@ private: std::atomic sync_quit{false}; std::unique_ptr background_thread_pool; + void executeDDLAtomic(Buffers & buffers, const QueryEvent & query_event); }; } From 14f31f5117f3b36fd7aec104d87184f1c7bf339b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 5 Nov 2020 14:28:20 +0300 Subject: [PATCH 048/114] add support of settings to URL, fix url and file table functions --- src/Storages/StorageFile.cpp | 13 +++--- src/Storages/StorageFile.h | 8 +++- src/Storages/StorageURL.cpp | 43 ++++++++++++++++--- src/Storages/StorageURL.h | 26 +++++++---- src/Storages/StorageXDBC.cpp | 15 ++++--- src/TableFunctions/TableFunctionFile.cpp | 11 +++-- src/TableFunctions/TableFunctionURL.cpp | 17 +++++--- .../01545_url_file_format_settings.reference | 4 ++ .../01545_url_file_format_settings.sql | 19 ++++++++ 9 files changed, 117 insertions(+), 39 deletions(-) create mode 100644 tests/queries/0_stateless/01545_url_file_format_settings.reference create mode 100644 tests/queries/0_stateless/01545_url_file_format_settings.sql diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index aa2696493f7..b130b2b0935 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -452,7 +452,7 @@ public: const StorageMetadataPtr & metadata_snapshot_, const CompressionMethod compression_method, const Context & context, - const FormatSettings & format_settings) + std::optional format_settings) : storage(storage_) , metadata_snapshot(metadata_snapshot_) , lock(storage.rwlock) @@ -618,18 +618,21 @@ void registerStorageFile(StorageFactory & factory) engine_args_ast[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[0], factory_args.local_context); storage_args.format_name = engine_args_ast[0]->as().value.safeGet(); + // Use format settings from global server context + settings from + // the SETTINGS clause of the create query. Settings from current + // session and user are ignored. if (factory_args.storage_def->settings) { - Context local_context_copy = factory_args.local_context; - local_context_copy.applySettingsChanges( + Context global_context_copy = factory_args.context; + global_context_copy.applySettingsChanges( factory_args.storage_def->settings->changes); storage_args.format_settings = getFormatSettings( - local_context_copy); + global_context_copy); } else { storage_args.format_settings = getFormatSettings( - factory_args.local_context); + factory_args.context); } if (engine_args_ast.size() == 1) /// Table in database diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 695cd0d3912..3d678ee0ded 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -52,7 +52,7 @@ public: { StorageID table_id; std::string format_name; - FormatSettings format_settings; + std::optional format_settings; std::string compression_method; const ColumnsDescription & columns; const ConstraintsDescription & constraints; @@ -80,7 +80,11 @@ private: explicit StorageFile(CommonArguments args); std::string format_name; - FormatSettings format_settings; + // We use format settings from global context + CREATE query for File table + // function -- in this case, format_settings is set. + // For `file` table function, we use format settings from current user context, + // in this case, format_settings is not set. + std::optional format_settings; int table_fd = -1; String compression_method; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 55c16496ba5..4483b012d69 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -32,6 +33,7 @@ IStorageURLBase::IStorageURLBase( const Context & context_, const StorageID & table_id_, const String & format_name_, + std::optional format_settings_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & compression_method_) @@ -40,6 +42,7 @@ IStorageURLBase::IStorageURLBase( , context_global(context_) , compression_method(compression_method_) , format_name(format_name_) + , format_settings(format_settings_) { context_global.getRemoteHostFilter().checkURL(uri); @@ -58,6 +61,7 @@ namespace const std::string & method, std::function callback, const String & format, + std::optional format_settings, String name_, const Block & sample_block, const Context & context, @@ -96,8 +100,10 @@ namespace context.getRemoteHostFilter()), compression_method); - reader = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); - reader = std::make_shared(reader, columns, context); + reader = FormatFactory::instance().getInput(format, *read_buf, + sample_block, context, max_block_size, format_settings); + reader = std::make_shared(reader, + columns, context); } String getName() const override @@ -134,6 +140,7 @@ namespace StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, const String & format, + std::optional format_settings, const Block & sample_block_, const Context & context, const ConnectionTimeouts & timeouts, @@ -143,7 +150,8 @@ StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), compression_method, 3); - writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context); + writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, + context, {} /* write callback */, format_settings); } @@ -214,6 +222,7 @@ Pipe IStorageURLBase::read( column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size), format_name, + format_settings, getName(), getHeaderBlock(column_names, metadata_snapshot), context, @@ -225,8 +234,8 @@ Pipe IStorageURLBase::read( BlockOutputStreamPtr IStorageURLBase::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) { - return std::make_shared( - uri, format_name, metadata_snapshot->getSampleBlock(), context_global, + return std::make_shared(uri, format_name, + format_settings, metadata_snapshot->getSampleBlock(), context_global, ConnectionTimeouts::getHTTPTimeouts(context_global), chooseCompressionMethod(uri.toString(), compression_method)); } @@ -255,16 +264,38 @@ void registerStorageURL(StorageFactory & factory) { engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.local_context); compression_method = engine_args[2]->as().value.safeGet(); - } else compression_method = "auto"; + } + else + { + compression_method = "auto"; + } + + // Use format settings from global server context + settings from + // the SETTINGS clause of the create query. Settings from current + // session and user are ignored. + FormatSettings format_settings; + if (args.storage_def->settings) + { + Context global_context_copy = args.context; + global_context_copy.applySettingsChanges( + args.storage_def->settings->changes); + format_settings = getFormatSettings(global_context_copy); + } + else + { + format_settings = getFormatSettings(args.context); + } return StorageURL::create( uri, args.table_id, format_name, + format_settings, args.columns, args.constraints, args.context, compression_method); }, { + .supports_settings = true, .source_access_type = AccessType::URL, }); } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 7983ad71520..4d6fdd1435c 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -36,6 +36,7 @@ protected: const Context & context_, const StorageID & id_, const String & format_name_, + std::optional format_settings_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & compression_method_); @@ -44,6 +45,11 @@ protected: const Context & context_global; String compression_method; String format_name; + // For URL engine, we use format settings from server context + `SETTINGS` + // clause of the `CREATE` query. In this case, format_settings is set. + // For `url` table function, we use settings from current query context. + // In this case, format_settings is not set. + std::optional format_settings; private: virtual std::string getReadMethod() const; @@ -73,6 +79,7 @@ public: StorageURLBlockOutputStream( const Poco::URI & uri, const String & format, + std::optional format_settings, const Block & sample_block_, const Context & context, const ConnectionTimeouts & timeouts, @@ -97,15 +104,16 @@ class StorageURL final : public ext::shared_ptr_helper, public IStor { friend struct ext::shared_ptr_helper; public: - StorageURL( - const Poco::URI & uri_, - const StorageID & table_id_, - const String & format_name_, - const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, - Context & context_, - const String & compression_method_) - : IStorageURLBase(uri_, context_, table_id_, format_name_, columns_, constraints_, compression_method_) + StorageURL(const Poco::URI & uri_, + const StorageID & table_id_, + const String & format_name_, + std::optional format_settings_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, + Context & context_, + const String & compression_method_) + : IStorageURLBase(uri_, context_, table_id_, format_name_, + format_settings_, columns_, constraints_, compression_method_) { } diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index 3350a4352db..590339c73cb 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -1,17 +1,18 @@ #include "StorageXDBC.h" + +#include +#include +#include #include #include #include +#include +#include +#include #include #include #include #include -#include -#include -#include -#include - -#include namespace DB { @@ -33,6 +34,7 @@ StorageXDBC::StorageXDBC( context_, table_id_, IXDBCBridgeHelper::DEFAULT_FORMAT, + getFormatSettings(context_), columns_, ConstraintsDescription{}, "" /* CompressionMethod */) @@ -121,6 +123,7 @@ BlockOutputStreamPtr StorageXDBC::write(const ASTPtr & /*query*/, const StorageM return std::make_shared( request_uri, format_name, + getFormatSettings(context), metadata_snapshot->getSampleBlock(), context, ConnectionTimeouts::getHTTPTimeouts(context), diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index f7d76309a7f..13ac6dc2145 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -10,13 +10,16 @@ namespace DB { -StoragePtr TableFunctionFile::getStorage( - const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const std::string & compression_method_) const +StoragePtr TableFunctionFile::getStorage(const String & source, + const String & format_, const ColumnsDescription & columns, + Context & global_context, const std::string & table_name, + const std::string & compression_method_) const { + // For `file` table function, we are going to use format settings from the + // query context. StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format_, - getFormatSettings(global_context), + std::nullopt /*format settings*/, compression_method_, columns, ConstraintsDescription{}, diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 6139e6ffecb..1c0109e892b 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -1,10 +1,12 @@ -#include -#include -#include -#include #include -#include + #include "registerTableFunctions.h" +#include +#include +#include +#include +#include +#include namespace DB @@ -14,8 +16,9 @@ StoragePtr TableFunctionURL::getStorage( const std::string & table_name, const String & compression_method_) const { Poco::URI uri(source); - return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), format_, columns, ConstraintsDescription{}, - global_context, compression_method_); + return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), + format_, std::nullopt /*format settings*/, columns, + ConstraintsDescription{}, global_context, compression_method_); } void registerTableFunctionURL(TableFunctionFactory & factory) diff --git a/tests/queries/0_stateless/01545_url_file_format_settings.reference b/tests/queries/0_stateless/01545_url_file_format_settings.reference new file mode 100644 index 00000000000..8a6795a16b8 --- /dev/null +++ b/tests/queries/0_stateless/01545_url_file_format_settings.reference @@ -0,0 +1,4 @@ +1 2 +1 2 +1 2 +1 2 diff --git a/tests/queries/0_stateless/01545_url_file_format_settings.sql b/tests/queries/0_stateless/01545_url_file_format_settings.sql new file mode 100644 index 00000000000..2ede999b1d0 --- /dev/null +++ b/tests/queries/0_stateless/01545_url_file_format_settings.sql @@ -0,0 +1,19 @@ +create table file_delim(a int, b int) engine File(CSV, '01545_url_file_format_settings.csv') settings format_csv_delimiter = '|'; + +truncate table file_delim; + +insert into file_delim select 1, 2; + +-- select 1, 2 format CSV settings format_csv_delimiter='/'; +create table url_delim(a int, b int) engine URL('http://127.0.0.1:8123/?query=select%201%2C%202%20format%20CSV%20settings%20format_csv_delimiter%3D%27/%27%3B%0A', CSV) settings format_csv_delimiter = '/'; + +select * from file_delim; + +select * from url_delim; + +select * from file('01545_url_file_format_settings.csv', CSV, 'a int, b int') settings format_csv_delimiter = '|'; + +select * from url('http://127.0.0.1:8123/?query=select%201%2C%202%20format%20CSV%20settings%20format_csv_delimiter%3D%27/%27%3B%0A', CSV, 'a int, b int') settings format_csv_delimiter = '/'; + + + From 9ce6c4dcb8b3c5f63f0bdf6a5caf568a55cdeef8 Mon Sep 17 00:00:00 2001 From: taichong Date: Fri, 6 Nov 2020 08:20:28 +0800 Subject: [PATCH 049/114] according to review modify code --- .../MySQL/MaterializeMySQLSyncThread.cpp | 17 +++++------------ .../MySQL/MaterializeMySQLSyncThread.h | 2 +- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp index f4e10757dfb..e1093f13119 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp @@ -619,7 +619,10 @@ void MaterializeMySQLSyncThread::onEvent(Buffers & buffers, const BinlogEventPtr else if (receive_event->type() == MYSQL_QUERY_EVENT) { QueryEvent & query_event = static_cast(*receive_event); - metadata.transaction(client.getPosition(),[&](){ executeDDLAtomic(buffers, query_event); }); + Position position_before_ddl; + position_before_ddl.update(metadata.binlog_position, metadata.binlog_file, metadata.executed_gtid_set); + metadata.transaction(position_before_ddl, [&]() { buffers.commit(global_context); }); + metadata.transaction(client.getPosition(),[&](){ executeDDLAtomic(query_event); }); } else if (receive_event->header.type != HEARTBEAT_EVENT) { @@ -634,24 +637,14 @@ void MaterializeMySQLSyncThread::onEvent(Buffers & buffers, const BinlogEventPtr } } -void MaterializeMySQLSyncThread::executeDDLAtomic(Buffers & buffers, const QueryEvent & query_event) +void MaterializeMySQLSyncThread::executeDDLAtomic(const QueryEvent & query_event) { - buffers.commit(global_context); - try { Context query_context = createQueryContext(global_context); String comment = "Materialize MySQL step 2: execute MySQL DDL for sync data"; String event_database = query_event.schema == mysql_database_name ? database_name : ""; tryToExecuteQuery(query_prefix + query_event.query, query_context, event_database, comment); - - const auto & position_message = [&]() - { - std::stringstream ss; - client.getPosition().dump(ss); - return ss.str(); - }; - LOG_INFO(log, "MySQL executed position: \n {}", position_message()); } catch (Exception & exception) { diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.h b/src/Databases/MySQL/MaterializeMySQLSyncThread.h index 44f6bb77d79..9a0df4823e5 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.h +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.h @@ -100,7 +100,7 @@ private: std::atomic sync_quit{false}; std::unique_ptr background_thread_pool; - void executeDDLAtomic(Buffers & buffers, const QueryEvent & query_event); + void executeDDLAtomic(const QueryEvent & query_event); }; } From 3c60f6cec20efb818eb5fc67990492458dca9a32 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Sat, 7 Nov 2020 11:53:39 +0300 Subject: [PATCH 050/114] make a separate settings collection + some cleanup --- src/Core/Settings.cpp | 2 + src/Core/Settings.h | 9 +++ src/Formats/FormatFactory.cpp | 79 ++++++++++--------- src/Formats/FormatFactory.h | 13 ++- src/Formats/FormatSettings.h | 26 +++--- src/Storages/StorageFile.cpp | 22 +++++- src/Storages/StorageURL.cpp | 28 +++++-- src/Storages/StorageURL.h | 6 +- .../01544_file_engine_settings.reference | 1 + .../0_stateless/01544_file_engine_settings.sh | 6 +- 10 files changed, 124 insertions(+), 68 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index ccb583ea9e0..4e59293c88d 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -130,4 +130,6 @@ void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfigura } } +IMPLEMENT_SETTINGS_TRAITS(FormatFactorySettingsTraits, FORMAT_FACTORY_SETTINGS) + } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4d4712dcba7..b920508544b 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -518,4 +518,13 @@ struct Settings : public BaseSettings static void checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path); }; +/* + * User-specified file format settings for File and ULR engines. + */ +DECLARE_SETTINGS_TRAITS(FormatFactorySettingsTraits, FORMAT_FACTORY_SETTINGS) + +struct FormatFactorySettings : public BaseSettings +{ +}; + } diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 7011d2372b0..631d7479262 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -40,11 +40,17 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name) throw Exception("Unknown format " + name, ErrorCodes::UNKNOWN_FORMAT); } - FormatSettings getFormatSettings(const Context & context) { const auto & settings = context.getSettingsRef(); + return getFormatSettings(context, settings); +} + +template +FormatSettings getFormatSettings(const Context & context, + const Settings & settings) +{ FormatSettings format_settings; format_settings.avro.allow_missing_fields = settings.input_format_avro_allow_missing_fields; @@ -114,6 +120,14 @@ FormatSettings getFormatSettings(const Context & context) return format_settings; } +template +FormatSettings getFormatSettings(const Context & context, + const FormatFactorySettings & settings); + +template +FormatSettings getFormatSettings(const Context & context, + const Settings & settings); + BlockInputStreamPtr FormatFactory::getInput( const String & name, @@ -121,23 +135,22 @@ BlockInputStreamPtr FormatFactory::getInput( const Block & sample, const Context & context, UInt64 max_block_size, - std::optional format_settings) const + const std::optional & _format_settings) const { if (name == "Native") return std::make_shared(buf, sample, 0); + auto format_settings = _format_settings + ? *_format_settings : getFormatSettings(context); + if (!getCreators(name).input_processor_creator) { const auto & input_getter = getCreators(name).input_creator; if (!input_getter) throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); - if (!format_settings) - { - format_settings = getFormatSettings(context); - } - return input_getter(buf, sample, max_block_size, {}, *format_settings); + return input_getter(buf, sample, max_block_size, {}, format_settings); } const Settings & settings = context.getSettingsRef(); @@ -163,21 +176,16 @@ BlockInputStreamPtr FormatFactory::getInput( if (!input_getter) throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); - if (!format_settings) - { - format_settings = getFormatSettings(context); - } - RowInputFormatParams row_input_format_params; row_input_format_params.max_block_size = max_block_size; - row_input_format_params.allow_errors_num = format_settings->input_allow_errors_num; - row_input_format_params.allow_errors_ratio = format_settings->input_allow_errors_ratio; + row_input_format_params.allow_errors_num = format_settings.input_allow_errors_num; + row_input_format_params.allow_errors_ratio = format_settings.input_allow_errors_ratio; row_input_format_params.max_execution_time = settings.max_execution_time; row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode; auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, - row_input_format_params, *format_settings}; + row_input_format_params, format_settings}; ParallelParsingBlockInputStream::Params params{buf, input_getter, input_creator_params, file_segmentation_engine, static_cast(settings.max_threads), @@ -193,24 +201,22 @@ BlockInputStreamPtr FormatFactory::getInput( BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, - WriteCallback callback, std::optional format_settings) const + WriteCallback callback, const std::optional & _format_settings) const { + auto format_settings = _format_settings + ? *_format_settings : getFormatSettings(context); + if (!getCreators(name).output_processor_creator) { const auto & output_getter = getCreators(name).output_creator; if (!output_getter) throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - if (!format_settings) - { - format_settings = getFormatSettings(context); - } - /** Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ return std::make_shared( - output_getter(buf, sample, std::move(callback), *format_settings), + output_getter(buf, sample, std::move(callback), format_settings), sample); } @@ -227,7 +233,7 @@ InputFormatPtr FormatFactory::getInputFormat( const Block & sample, const Context & context, UInt64 max_block_size, - std::optional format_settings) const + const std::optional & _format_settings) const { const auto & input_getter = getCreators(name).input_processor_creator; if (!input_getter) @@ -235,19 +241,18 @@ InputFormatPtr FormatFactory::getInputFormat( const Settings & settings = context.getSettingsRef(); - if (!format_settings) - { - format_settings = getFormatSettings(context); - } + auto format_settings = _format_settings + ? *_format_settings : getFormatSettings(context); RowInputFormatParams params; params.max_block_size = max_block_size; - params.allow_errors_num = format_settings->input_allow_errors_num; - params.allow_errors_ratio = format_settings->input_allow_errors_ratio; + params.allow_errors_num = format_settings.input_allow_errors_num; + params.allow_errors_ratio = format_settings.input_allow_errors_ratio; params.max_execution_time = settings.max_execution_time; params.timeout_overflow_mode = settings.timeout_overflow_mode; - auto format = input_getter(buf, sample, params, *format_settings); + auto format = input_getter(buf, sample, params, format_settings); + /// It's a kludge. Because I cannot remove context from values format. if (auto * values = typeid_cast(format.get())) @@ -260,27 +265,25 @@ InputFormatPtr FormatFactory::getInputFormat( OutputFormatPtr FormatFactory::getOutputFormat( const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, - std::optional format_settings) const + const std::optional & _format_settings) const { const auto & output_getter = getCreators(name).output_processor_creator; if (!output_getter) throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - if (!format_settings) - { - format_settings = getFormatSettings(context); - } - RowOutputFormatParams params; params.callback = std::move(callback); + auto format_settings = _format_settings + ? *_format_settings : getFormatSettings(context); + /** TODO: Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ - auto format = output_getter(buf, sample, params, *format_settings); + auto format = output_getter(buf, sample, params, format_settings); /// Enable auto-flush for streaming mode. Currently it is needed by INSERT WATCH query. - if (format_settings->enable_streaming) + if (format_settings.enable_streaming) format->setAutoFlush(); /// It's a kludge. Because I cannot remove context from MySQL format. diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 619acd10e0f..424ff8140a7 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -18,6 +18,7 @@ class Block; class Context; struct FormatSettings; struct Settings; +struct FormatFactorySettings; class ReadBuffer; class WriteBuffer; @@ -36,6 +37,10 @@ using OutputFormatPtr = std::shared_ptr; FormatSettings getFormatSettings(const Context & context); +template +FormatSettings getFormatSettings(const Context & context, + const T & settings); + /** Allows to create an IBlockInputStream or IBlockOutputStream by the name of the format. * Note: format and compression are independent things. */ @@ -108,11 +113,11 @@ public: const Block & sample, const Context & context, UInt64 max_block_size, - std::optional format_settings = std::nullopt) const; + const std::optional & format_settings = std::nullopt) const; BlockOutputStreamPtr getOutput(const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, - std::optional format_settings = std::nullopt) const; + const std::optional & format_settings = std::nullopt) const; InputFormatPtr getInputFormat( const String & name, @@ -120,12 +125,12 @@ public: const Block & sample, const Context & context, UInt64 max_block_size, - std::optional format_settings = std::nullopt) const; + const std::optional & format_settings = std::nullopt) const; OutputFormatPtr getOutputFormat( const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, - std::optional format_settings = std::nullopt) const; + const std::optional & format_settings = std::nullopt) const; /// Register format by its name. void registerInputFormat(const String & name, InputCreator input_creator); diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index d2a596bca4d..b3c01ddcf14 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -6,10 +6,16 @@ namespace DB { -/** Various tweaks for input/output formats. - * Text serialization/deserialization of data types also depend on some of these settings. - * NOTE Parameters for unrelated formats and unrelated data types - * are collected in this struct - it prevents modularity, but they are difficult to separate. +/** + * Various tweaks for input/output formats. Text serialization/deserialization + * of data types also depend on some of these settings. It is different from + * FormatFactorySettings in that it has all necessary user-provided settings + * combined with information from context etc, that we can use directly during + * serialization. In contrast, FormatFactorySettings' job is to reflect the + * changes made to user-visible format settings, such as when tweaking the + * the format for File engine. + * NOTE Parameters for unrelated formats and unrelated data types are collected + * in this struct - it prevents modularity, but they are difficult to separate. */ struct FormatSettings { @@ -113,13 +119,11 @@ struct FormatSettings { bool write_row_delimiters = true; /** - * Some buffers (kafka / rabbit) split the rows internally using callback - * so we can push there formats without framing / delimiters (like - * ProtobufSingle). In other cases you can't write more than single row - * in unframed format. - * Not sure we need this parameter at all, it only serves as an additional - * safety check in ProtobufSingle format, but exporting constant-size - * records w/o delimiters might be generally useful, not only for Kafka. + * Some buffers (kafka / rabbit) split the rows internally using callback, + * and always send one row per message, so we can push there formats + * without framing / delimiters (like ProtobufSingle). In other cases, + * we have to enforce exporting at most one row in the format output, + * because Protobuf without delimiters is not generally useful. */ bool allow_many_rows_no_delimiters = false; } protobuf; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index e0837822c2a..7556eacf371 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -452,7 +452,7 @@ public: const StorageMetadataPtr & metadata_snapshot_, const CompressionMethod compression_method, const Context & context, - std::optional format_settings) + const std::optional & format_settings) : storage(storage_) , metadata_snapshot(metadata_snapshot_) , lock(storage.rwlock) @@ -628,11 +628,25 @@ void registerStorageFile(StorageFactory & factory) // session and user are ignored. if (factory_args.storage_def->settings) { - Context global_context_copy = factory_args.context; - global_context_copy.applySettingsChanges( + FormatFactorySettings user_format_settings; + + // Apply changed settings from global context, but ignore the + // unknown ones, because we only have the format settings here. + const auto & changes = factory_args.context.getSettingsRef().changes(); + for (const auto & change : changes) + { + if (user_format_settings.has(change.name)) + { + user_format_settings.set(change.name, change.value); + } + } + + // Apply changes from SETTINGS clause, with validation. + user_format_settings.applyChanges( factory_args.storage_def->settings->changes); + storage_args.format_settings = getFormatSettings( - global_context_copy); + factory_args.context, user_format_settings); } else { diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 4483b012d69..e2f72207f96 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -33,7 +33,7 @@ IStorageURLBase::IStorageURLBase( const Context & context_, const StorageID & table_id_, const String & format_name_, - std::optional format_settings_, + const std::optional & format_settings_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & compression_method_) @@ -61,7 +61,7 @@ namespace const std::string & method, std::function callback, const String & format, - std::optional format_settings, + const std::optional & format_settings, String name_, const Block & sample_block, const Context & context, @@ -140,7 +140,7 @@ namespace StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, const String & format, - std::optional format_settings, + const std::optional & format_settings, const Block & sample_block_, const Context & context, const ConnectionTimeouts & timeouts, @@ -276,10 +276,24 @@ void registerStorageURL(StorageFactory & factory) FormatSettings format_settings; if (args.storage_def->settings) { - Context global_context_copy = args.context; - global_context_copy.applySettingsChanges( - args.storage_def->settings->changes); - format_settings = getFormatSettings(global_context_copy); + FormatFactorySettings user_format_settings; + + // Apply changed settings from global context, but ignore the + // unknown ones, because we only have the format settings here. + const auto & changes = args.context.getSettingsRef().changes(); + for (const auto & change : changes) + { + if (user_format_settings.has(change.name)) + { + user_format_settings.set(change.name, change.value); + } + } + + // Apply changes from SETTINGS clause, with validation. + user_format_settings.applyChanges(args.storage_def->settings->changes); + + format_settings = getFormatSettings(args.context, + user_format_settings); } else { diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 4d6fdd1435c..ee38896812f 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -36,7 +36,7 @@ protected: const Context & context_, const StorageID & id_, const String & format_name_, - std::optional format_settings_, + const std::optional & format_settings_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & compression_method_); @@ -79,7 +79,7 @@ public: StorageURLBlockOutputStream( const Poco::URI & uri, const String & format, - std::optional format_settings, + const std::optional & format_settings, const Block & sample_block_, const Context & context, const ConnectionTimeouts & timeouts, @@ -107,7 +107,7 @@ public: StorageURL(const Poco::URI & uri_, const StorageID & table_id_, const String & format_name_, - std::optional format_settings_, + const std::optional & format_settings_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, Context & context_, diff --git a/tests/queries/0_stateless/01544_file_engine_settings.reference b/tests/queries/0_stateless/01544_file_engine_settings.reference index d2afb8fc688..8413a79e8cc 100644 --- a/tests/queries/0_stateless/01544_file_engine_settings.reference +++ b/tests/queries/0_stateless/01544_file_engine_settings.reference @@ -1,2 +1,3 @@ 1|1 1 1 +1 2 diff --git a/tests/queries/0_stateless/01544_file_engine_settings.sh b/tests/queries/0_stateless/01544_file_engine_settings.sh index b13ec0f3db3..ff6b0d3d373 100755 --- a/tests/queries/0_stateless/01544_file_engine_settings.sh +++ b/tests/queries/0_stateless/01544_file_engine_settings.sh @@ -20,4 +20,8 @@ cat "$the_file" ${CLICKHOUSE_LOCAL} --query " create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|'; select * from t; -" +" + +# Also check that the File engine emplicitly created by clickhouse-local +# uses the modified settings. +${CLICKHOUSE_LOCAL} --structure="a int, b int" --input-format=CSV --format_csv_delimiter="|" --query="select * from table" <<<"1|2" From dfa35a5e6d8f4bd9a0456c9fa18ea5662e478141 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 8 Nov 2020 16:11:18 +0300 Subject: [PATCH 051/114] Update MergeTreeIndexConditionBloomFilter.cpp --- src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 69d63d0e80d..e84525df6e8 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -380,6 +380,9 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTEquals( if (!array_type) throw Exception("First argument for function " + function_name + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + /// We can treat `indexOf` function similar to `has`. + /// But it is little more cumbersome, compare: `has(arr, elem)` and `indexOf(arr, elem) != 0`. + /// The `parent` in this context is expected to be function `!=` (`notEquals`). if (function_name == "has" || indexOfCanUseBloomFilter(parent)) { out.function = RPNElement::FUNCTION_HAS; From 85f7eadc4f3798df0198aa805fb30f2a6214abfb Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 8 Nov 2020 16:14:32 +0300 Subject: [PATCH 052/114] Update MergeTreeIndexConditionBloomFilter.cpp --- .../MergeTreeIndexConditionBloomFilter.cpp | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index e84525df6e8..2cf10a9e677 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -324,6 +324,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTIn( return false; } + static bool indexOfCanUseBloomFilter(const ASTPtr & parent) { if (!parent) @@ -331,7 +332,7 @@ static bool indexOfCanUseBloomFilter(const ASTPtr & parent) if (const auto * function = parent->as()) { - if (function->name == "arrayElement") + if (function->name == "and") { return true; } @@ -344,21 +345,21 @@ static bool indexOfCanUseBloomFilter(const ASTPtr & parent) if (const ASTLiteral * left = function->arguments->children[0]->as()) { - if (function->name == "equals" && left->value.get() != 0) - return true; - else if (function->name == "less" && left->value.get() >= 0) - return true; - else if (function->name == "lessOrEquals" && left->value.get() > 0) - return true; + if (function->name == "equals" && left->value.get() != 0) + return true; + else if (function->name == "less" && left->value.get() >= 0) + return true; + else if (function->name == "lessOrEquals" && left->value.get() > 0) + return true; } else if (const ASTLiteral * right = function->arguments->children[1]->as()) { - if (function->name == "equals" && right->value.get() != 0) - return true; - else if (function->name == "greater" && right->value.get() >= 0) - return true; - else if (function->name == "greaterOrEquals" && right->value.get() > 0) - return true; + if (function->name == "equals" && right->value.get() != 0) + return true; + else if (function->name == "greater" && right->value.get() >= 0) + return true; + else if (function->name == "greaterOrEquals" && right->value.get() > 0) + return true; } } } @@ -366,6 +367,7 @@ static bool indexOfCanUseBloomFilter(const ASTPtr & parent) return false; } + bool MergeTreeIndexConditionBloomFilter::traverseASTEquals( const String & function_name, const ASTPtr & key_ast, const DataTypePtr & value_type, const Field & value_field, RPNElement & out, const ASTPtr & parent) { From 8264bd900b64519693a1a7102efb0123a9d2215a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 8 Nov 2020 16:17:03 +0300 Subject: [PATCH 053/114] Update MergeTreeIndexConditionBloomFilter.cpp --- src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 2cf10a9e677..03bc9e4e046 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -330,6 +330,8 @@ static bool indexOfCanUseBloomFilter(const ASTPtr & parent) if (!parent) return true; + /// `parent` is a function where `indexOf` is located. + /// Example: `indexOf(arr, x) = 1`, parent is a function named `equals`. if (const auto * function = parent->as()) { if (function->name == "and") @@ -343,6 +345,7 @@ static bool indexOfCanUseBloomFilter(const ASTPtr & parent) if (function->arguments->children.size() != 2) return false; + /// We don't allow constant expressions like `indexOf(arr, x) = 1 + 0` but it's neglible. if (const ASTLiteral * left = function->arguments->children[0]->as()) { if (function->name == "equals" && left->value.get() != 0) From 8e1a2fc9e4fc5c02fe947de79f3e62f2328fa771 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Nov 2020 16:41:25 +0300 Subject: [PATCH 054/114] Fix UB and overflow --- .../MergeTreeIndexConditionBloomFilter.cpp | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 03bc9e4e046..c709f8c3c99 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -1,17 +1,17 @@ -#include -#include -#include #include -#include -#include +#include #include #include #include -#include +#include +#include +#include +#include #include #include #include -#include +#include +#include #include #include @@ -106,11 +106,11 @@ bool MergeTreeIndexConditionBloomFilter::alwaysUnknownOrTrue() const rpn_stack.push_back(true); } else if (element.function == RPNElement::FUNCTION_EQUALS - || element.function == RPNElement::FUNCTION_NOT_EQUALS - || element.function == RPNElement::FUNCTION_HAS - || element.function == RPNElement::FUNCTION_IN - || element.function == RPNElement::FUNCTION_NOT_IN - || element.function == RPNElement::ALWAYS_FALSE) + || element.function == RPNElement::FUNCTION_NOT_EQUALS + || element.function == RPNElement::FUNCTION_HAS + || element.function == RPNElement::FUNCTION_IN + || element.function == RPNElement::FUNCTION_NOT_IN + || element.function == RPNElement::ALWAYS_FALSE) { rpn_stack.push_back(false); } @@ -338,7 +338,7 @@ static bool indexOfCanUseBloomFilter(const ASTPtr & parent) { return true; } - else if (function->name == "equals" + else if (function->name == "equals" /// notEquals is not applicable || function->name == "greater" || function->name == "greaterOrEquals" || function->name == "less" || function->name == "lessOrEquals") { @@ -346,24 +346,33 @@ static bool indexOfCanUseBloomFilter(const ASTPtr & parent) return false; /// We don't allow constant expressions like `indexOf(arr, x) = 1 + 0` but it's neglible. + + /// We should return true when the corresponding expression implies that the array contains the element. + /// Example: when `indexOf(arr, x)` > 10 is written, it means that arr definitely should contain the element + /// (at least at 11th position but it does not matter). + + bool reversed = false; + const ASTLiteral * constant = nullptr; + if (const ASTLiteral * left = function->arguments->children[0]->as()) { - if (function->name == "equals" && left->value.get() != 0) - return true; - else if (function->name == "less" && left->value.get() >= 0) - return true; - else if (function->name == "lessOrEquals" && left->value.get() > 0) - return true; + constant = left; + reversed = true; } else if (const ASTLiteral * right = function->arguments->children[1]->as()) { - if (function->name == "equals" && right->value.get() != 0) - return true; - else if (function->name == "greater" && right->value.get() >= 0) - return true; - else if (function->name == "greaterOrEquals" && right->value.get() > 0) - return true; + constant = right; } + else + return false; + + Field zero(0); + return (function->name == "equals" /// indexOf(...) = c, c != 0 + && !applyVisitor(FieldVisitorAccurateEquals(), constant->value, zero)) + || (function->name == (reversed ? "less" : "greater") /// indexOf(...) > c, c >= 0 + && !applyVisitor(FieldVisitorAccurateLess(), constant->value, zero)) + || (function->name == (reversed ? "lessOrEquals" : "greaterOrEquals") /// indexOf(...) >= c, c > 0 + && applyVisitor(FieldVisitorAccurateLess(), zero, constant->value)); } } From 3f5839c83cd3113545e323edcc5771cd1260df2b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Nov 2020 16:45:57 +0300 Subject: [PATCH 055/114] Add one more case --- src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index c709f8c3c99..fa0aa03e820 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -369,6 +369,8 @@ static bool indexOfCanUseBloomFilter(const ASTPtr & parent) Field zero(0); return (function->name == "equals" /// indexOf(...) = c, c != 0 && !applyVisitor(FieldVisitorAccurateEquals(), constant->value, zero)) + || (function->name == "notEquals" /// indexOf(...) != c, c = 0 + && applyVisitor(FieldVisitorAccurateEquals(), constant->value, zero)) || (function->name == (reversed ? "less" : "greater") /// indexOf(...) > c, c >= 0 && !applyVisitor(FieldVisitorAccurateLess(), constant->value, zero)) || (function->name == (reversed ? "lessOrEquals" : "greaterOrEquals") /// indexOf(...) >= c, c > 0 From d96914b044f51eb5145e0b8c30b44e4944a58e2d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 8 Nov 2020 16:51:03 +0300 Subject: [PATCH 056/114] Update bloom_filter.xml --- tests/performance/bloom_filter.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/bloom_filter.xml b/tests/performance/bloom_filter.xml index 00bdd594738..249ee4c511e 100644 --- a/tests/performance/bloom_filter.xml +++ b/tests/performance/bloom_filter.xml @@ -9,7 +9,6 @@ SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') = 2 SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0 - SELECT count() FROM test_bf_indexOf WHERE ary[indexOf(ary, '1')] = '1' SYSTEM START MERGES DROP TABLE IF EXISTS test_bf_indexOf From c7aa0c851aaae98f7affa3a8d8b3164030e87f64 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 8 Nov 2020 16:51:45 +0300 Subject: [PATCH 057/114] Update bloom_filter.xml --- tests/performance/bloom_filter.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/bloom_filter.xml b/tests/performance/bloom_filter.xml index 249ee4c511e..11ce38e647a 100644 --- a/tests/performance/bloom_filter.xml +++ b/tests/performance/bloom_filter.xml @@ -4,8 +4,8 @@ CREATE TABLE test_bf_indexOf (`id` int, `ary` Array(LowCardinality(Nullable(String))), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SYSTEM STOP MERGES - INSERT INTO test_bf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) - INSERT INTO test_bf_indexOf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) + INSERT INTO test_bf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) + INSERT INTO test_bf_indexOf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') = 2 SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0 From f4178dd91f248fa7fd3b93b58272b2c90b1f4b96 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Nov 2020 16:55:26 +0300 Subject: [PATCH 058/114] Fix perf tests --- tests/performance/bloom_filter_insert.xml | 10 ++++++++++ tests/performance/bloom_filter_select.xml | 11 +++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/performance/bloom_filter_insert.xml create mode 100644 tests/performance/bloom_filter_select.xml diff --git a/tests/performance/bloom_filter_insert.xml b/tests/performance/bloom_filter_insert.xml new file mode 100644 index 00000000000..010bce50516 --- /dev/null +++ b/tests/performance/bloom_filter_insert.xml @@ -0,0 +1,10 @@ + + DROP TABLE IF EXISTS test_bf + CREATE TABLE test_bf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 8192) ENGINE = MergeTree() ORDER BY id + SYSTEM STOP MERGES + + INSERT INTO test_bf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) + + SYSTEM START MERGES + DROP TABLE IF EXISTS test_bf + diff --git a/tests/performance/bloom_filter_select.xml b/tests/performance/bloom_filter_select.xml new file mode 100644 index 00000000000..f27a43c2991 --- /dev/null +++ b/tests/performance/bloom_filter_select.xml @@ -0,0 +1,11 @@ + + DROP TABLE IF EXISTS test_bf + CREATE TABLE test_bf_indexOf (`id` int, `ary` Array(LowCardinality(Nullable(String))), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id + + INSERT INTO test_bf_indexOf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) + + SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') = 2 + SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0 + + DROP TABLE IF EXISTS test_bf_indexOf + From 7257a5c3d2223b907a8a9371bd3b1a0c36ca9980 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Nov 2020 16:55:36 +0300 Subject: [PATCH 059/114] Fix perf tests --- tests/performance/bloom_filter.xml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 tests/performance/bloom_filter.xml diff --git a/tests/performance/bloom_filter.xml b/tests/performance/bloom_filter.xml deleted file mode 100644 index 11ce38e647a..00000000000 --- a/tests/performance/bloom_filter.xml +++ /dev/null @@ -1,16 +0,0 @@ - - DROP TABLE IF EXISTS test_bf - CREATE TABLE test_bf (`id` int, `ary` Array(String), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 8192) ENGINE = MergeTree() ORDER BY id - CREATE TABLE test_bf_indexOf (`id` int, `ary` Array(LowCardinality(Nullable(String))), INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY id - SYSTEM STOP MERGES - - INSERT INTO test_bf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) - INSERT INTO test_bf_indexOf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) - - SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') = 2 - SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0 - - SYSTEM START MERGES - DROP TABLE IF EXISTS test_bf_indexOf - DROP TABLE IF EXISTS test_bf - From 02da9af32d01fe91dd280361870e703ebaeb3225 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 17 Oct 2020 00:31:29 +0300 Subject: [PATCH 060/114] DateTime comparison with date string literal --- docs/en/operations/settings/settings.md | 2 +- docs/en/sql-reference/data-types/date.md | 37 ++++++++- src/IO/ReadHelpers.cpp | 28 +++++-- src/IO/ReadHelpers.h | 30 +++++-- ...e_time_compare_with_date_literal.reference | 80 +++++++++++++++++++ ...23_date_time_compare_with_date_literal.sql | 70 ++++++++++++++++ 6 files changed, 231 insertions(+), 16 deletions(-) create mode 100644 tests/queries/0_stateless/01523_date_time_compare_with_date_literal.reference create mode 100644 tests/queries/0_stateless/01523_date_time_compare_with_date_literal.sql diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index d83f7d6c219..f9c3c8a5d75 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -384,7 +384,7 @@ Possible values: - `'basic'` — Use basic parser. - ClickHouse can parse only the basic `YYYY-MM-DD HH:MM:SS` format. For example, `'2019-08-20 10:18:56'`. + ClickHouse can parse only the basic `YYYY-MM-DD HH:MM:SS` or `YYYY-MM-DD` format. For example, `'2019-08-20 10:18:56'` or `2019-08-20`. Default value: `'basic'`. diff --git a/docs/en/sql-reference/data-types/date.md b/docs/en/sql-reference/data-types/date.md index c686a987b2f..886e93f433c 100644 --- a/docs/en/sql-reference/data-types/date.md +++ b/docs/en/sql-reference/data-types/date.md @@ -3,10 +3,45 @@ toc_priority: 47 toc_title: Date --- -# Date {#date} +# Date {#data_type-date} A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105). The date value is stored without the time zone. +## Examples {#examples} + +**1.** Creating a table with a `DateTime`-type column and inserting data into it: + +``` sql +CREATE TABLE dt +( + `timestamp` Date, + `event_id` UInt8 +) +ENGINE = TinyLog; +``` + +``` sql +INSERT INTO dt Values (1546300800, 1), ('2019-01-01', 2); +``` + +``` sql +SELECT * FROM dt; +``` + +``` text +┌──timestamp─┬─event_id─┐ +│ 2019-01-01 │ 1 │ +│ 2019-01-01 │ 2 │ +└────────────┴──────────┘ +``` + +## See Also {#see-also} + +- [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md) +- [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime) +- [`DateTime` data type](../../sql-reference/data-types/datetime.md) + + [Original article](https://clickhouse.tech/docs/en/data_types/date/) diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index bf41de3959a..73ee0dfcd95 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -817,7 +817,11 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D { static constexpr bool throw_exception = std::is_same_v; + /// YYYY-MM-DD hh:mm:ss static constexpr auto date_time_broken_down_length = 19; + /// YYYY-MM-DD + static constexpr auto date_broken_down_length = 10; + /// unix timestamp max length static constexpr auto unix_timestamp_max_length = 10; char s[date_time_broken_down_length]; @@ -831,12 +835,15 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D ++buf.position(); } - /// 2015-01-01 01:02:03 + /// 2015-01-01 01:02:03 or 2015-01-01 if (s_pos == s + 4 && !buf.eof() && (*buf.position() < '0' || *buf.position() > '9')) { - const size_t remaining_size = date_time_broken_down_length - (s_pos - s); - size_t size = buf.read(s_pos, remaining_size); - if (remaining_size != size) + const auto already_read_length = s_pos - s; + const size_t remaining_date_time_size = date_time_broken_down_length - already_read_length; + const size_t remaining_date_size = date_broken_down_length - already_read_length; + + size_t size = buf.read(s_pos, remaining_date_time_size); + if (size != remaining_date_time_size && size != remaining_date_size) { s_pos[size] = 0; @@ -850,9 +857,16 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); - UInt8 hour = (s[11] - '0') * 10 + (s[12] - '0'); - UInt8 minute = (s[14] - '0') * 10 + (s[15] - '0'); - UInt8 second = (s[17] - '0') * 10 + (s[18] - '0'); + UInt8 hour = 0; + UInt8 minute = 0; + UInt8 second = 0; + + if (size == remaining_date_time_size) + { + hour = (s[11] - '0') * 10 + (s[12] - '0'); + minute = (s[14] - '0') * 10 + (s[15] - '0'); + second = (s[17] - '0') * 10 + (s[18] - '0'); + } if (unlikely(year == 0)) datetime = 0; diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 9ff1858c723..c79e260bf46 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -700,7 +700,7 @@ UInt128 stringToUUID(const String & str); template ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut); -/** In YYYY-MM-DD hh:mm:ss format, according to specified time zone. +/** In YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format, according to specified time zone. * As an exception, also supported parsing of unix timestamp in form of decimal number. */ template @@ -709,12 +709,21 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons /** Read 10 characters, that could represent unix timestamp. * Only unix timestamp of 5-10 characters is supported. * Then look at 5th character. If it is a number - treat whole as unix timestamp. - * If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss format. + * If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format. */ /// Optimistic path, when whole value is in buffer. const char * s = buf.position(); - if (s + 19 <= buf.buffer().end()) + + /// YYYY-MM-DD hh:mm:ss + static constexpr auto DateTimeStringInputSize = 19; + bool optimistic_path_for_date_time_input = s + DateTimeStringInputSize <= buf.buffer().end(); + + /// YYYY-MM-DD + static constexpr auto DateStringInputSize = 10; + bool optimistic_path_for_date_input = s + DateStringInputSize <= buf.buffer().end(); + + if (optimistic_path_for_date_time_input || optimistic_path_for_date_input) { if (s[4] < '0' || s[4] > '9') { @@ -722,16 +731,23 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); - UInt8 hour = (s[11] - '0') * 10 + (s[12] - '0'); - UInt8 minute = (s[14] - '0') * 10 + (s[15] - '0'); - UInt8 second = (s[17] - '0') * 10 + (s[18] - '0'); + UInt8 hour = 0; + UInt8 minute = 0; + UInt8 second = 0; + + if (optimistic_path_for_date_time_input) + { + hour = (s[11] - '0') * 10 + (s[12] - '0'); + minute = (s[14] - '0') * 10 + (s[15] - '0'); + second = (s[17] - '0') * 10 + (s[18] - '0'); + } if (unlikely(year == 0)) datetime = 0; else datetime = date_lut.makeDateTime(year, month, day, hour, minute, second); - buf.position() += 19; + buf.position() += optimistic_path_for_date_time_input ? DateTimeStringInputSize : DateStringInputSize; return ReturnType(true); } else diff --git a/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.reference b/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.reference new file mode 100644 index 00000000000..aad646c5a79 --- /dev/null +++ b/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.reference @@ -0,0 +1,80 @@ +DateTime +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 + +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 +2020-10-16 00:00:00 + + +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 + +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 + +2020-10-15 00:00:00 +2020-10-15 12:00:00 +2020-10-16 00:00:00 + +DateTime64 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 + +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + + +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 + +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 + +2020-10-15 00:00:00.000 +2020-10-15 12:00:00.000 +2020-10-16 00:00:00.000 + diff --git a/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.sql b/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.sql new file mode 100644 index 00000000000..5a3aa23bcbf --- /dev/null +++ b/tests/queries/0_stateless/01523_date_time_compare_with_date_literal.sql @@ -0,0 +1,70 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test(timestamp DateTime) ENGINE = MergeTree ORDER BY timestamp; + +INSERT INTO test VALUES ('2020-10-15 00:00:00'); +INSERT INTO test VALUES ('2020-10-15 12:00:00'); +INSERT INTO test VALUES ('2020-10-16 00:00:00'); + +SELECT 'DateTime'; +SELECT * FROM test WHERE timestamp != '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp == '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp > '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp >= '2020-10-15' ORDER by timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp < '2020-10-16' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp <= '2020-10-16' ORDER BY timestamp; +SELECT ''; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' != timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' == timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' < timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' <= timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-16' > timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-16' >= timestamp ORDER BY timestamp; +SELECT ''; + +DROP TABLE test; +CREATE TABLE test(timestamp DateTime64) ENGINE = MergeTree ORDER BY timestamp; + +INSERT INTO test VALUES ('2020-10-15 00:00:00'); +INSERT INTO test VALUES ('2020-10-15 12:00:00'); +INSERT INTO test VALUES ('2020-10-16 00:00:00'); + +SELECT 'DateTime64'; +SELECT * FROM test WHERE timestamp != '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp == '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp > '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp >= '2020-10-15' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp < '2020-10-16' ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE timestamp <= '2020-10-16' ORDER BY timestamp; +SELECT ''; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' != timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' == timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' < timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-15' <= timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-16' > timestamp ORDER BY timestamp; +SELECT ''; +SELECT * FROM test WHERE '2020-10-16' >= timestamp ORDER BY timestamp; +SELECT ''; + +DROP TABLE test; From 240bbd2cd728c588f0218d1246b2262c43c4a644 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 7 Nov 2020 16:21:56 +0300 Subject: [PATCH 061/114] Removed fast path for parsing DateTime in Date format --- src/IO/ReadHelpers.h | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index c79e260bf46..0e91bc1dd8e 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -719,11 +719,7 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons static constexpr auto DateTimeStringInputSize = 19; bool optimistic_path_for_date_time_input = s + DateTimeStringInputSize <= buf.buffer().end(); - /// YYYY-MM-DD - static constexpr auto DateStringInputSize = 10; - bool optimistic_path_for_date_input = s + DateStringInputSize <= buf.buffer().end(); - - if (optimistic_path_for_date_time_input || optimistic_path_for_date_input) + if (optimistic_path_for_date_time_input) { if (s[4] < '0' || s[4] > '9') { @@ -731,23 +727,16 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); - UInt8 hour = 0; - UInt8 minute = 0; - UInt8 second = 0; - - if (optimistic_path_for_date_time_input) - { - hour = (s[11] - '0') * 10 + (s[12] - '0'); - minute = (s[14] - '0') * 10 + (s[15] - '0'); - second = (s[17] - '0') * 10 + (s[18] - '0'); - } + UInt8 hour = (s[11] - '0') * 10 + (s[12] - '0'); + UInt8 minute = (s[14] - '0') * 10 + (s[15] - '0'); + UInt8 second = (s[17] - '0') * 10 + (s[18] - '0'); if (unlikely(year == 0)) datetime = 0; else datetime = date_lut.makeDateTime(year, month, day, hour, minute, second); - buf.position() += optimistic_path_for_date_time_input ? DateTimeStringInputSize : DateStringInputSize; + buf.position() += DateTimeStringInputSize; return ReturnType(true); } else From a06be511df8ebd0a963470026d8460257b891dc7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 9 Nov 2020 16:07:38 +0300 Subject: [PATCH 062/114] pcg serialization --- base/pcg-random/pcg_random.hpp | 9 +++++++ .../AggregateFunctionGroupArray.h | 27 +++++++++---------- src/AggregateFunctions/ReservoirSampler.h | 15 ++++++----- src/Client/MultiplexedConnections.cpp | 8 +++--- src/IO/Operators.h | 2 ++ src/IO/ReadHelpers.h | 20 ++++++++++++++ src/IO/WriteHelpers.h | 12 +++++++++ utils/check-style/check-style | 3 +++ 8 files changed, 71 insertions(+), 25 deletions(-) diff --git a/base/pcg-random/pcg_random.hpp b/base/pcg-random/pcg_random.hpp index d96d5895b31..f7778480c4c 100644 --- a/base/pcg-random/pcg_random.hpp +++ b/base/pcg-random/pcg_random.hpp @@ -113,6 +113,12 @@ #include "pcg_extras.hpp" +namespace DB +{ + struct PcgSerializer; + struct PcgDeserializer; +} + namespace pcg_detail { using namespace pcg_extras; @@ -557,6 +563,9 @@ public: engine& rng); + + friend ::DB::PcgSerializer; + friend ::DB::PcgDeserializer; }; template #include +#include +#include +#include #include #include @@ -244,10 +247,9 @@ public: if constexpr (Trait::sampler == Sampler::RNG) { DB::writeIntBinary(this->data(place).total_values, buf); - std::ostringstream rng_stream; - rng_stream.exceptions(std::ios::failbit); - rng_stream << this->data(place).rng; - DB::writeStringBinary(rng_stream.str(), buf); + WriteBufferFromOwnString rng_buf; + rng_buf << this->data(place).rng; + DB::writeStringBinary(rng_buf.str(), buf); } // TODO @@ -275,9 +277,8 @@ public: DB::readIntBinary(this->data(place).total_values, buf); std::string rng_string; DB::readStringBinary(rng_string, buf); - std::istringstream rng_stream(rng_string); - rng_stream.exceptions(std::ios::failbit); - rng_stream >> this->data(place).rng; + ReadBufferFromString rng_buf(rng_string); + rng_buf >> this->data(place).rng; } // TODO @@ -565,10 +566,9 @@ public: if constexpr (Trait::sampler == Sampler::RNG) { DB::writeIntBinary(data(place).total_values, buf); - std::ostringstream rng_stream; - rng_stream.exceptions(std::ios::failbit); - rng_stream << data(place).rng; - DB::writeStringBinary(rng_stream.str(), buf); + WriteBufferFromOwnString rng_buf; + rng_buf << data(place).rng; + DB::writeStringBinary(rng_buf.str(), buf); } // TODO @@ -600,9 +600,8 @@ public: DB::readIntBinary(data(place).total_values, buf); std::string rng_string; DB::readStringBinary(rng_string, buf); - std::istringstream rng_stream(rng_string); - rng_stream.exceptions(std::ios::failbit); - rng_stream >> data(place).rng; + ReadBufferFromString rng_buf(rng_string); + rng_buf >> data(place).rng; } // TODO diff --git a/src/AggregateFunctions/ReservoirSampler.h b/src/AggregateFunctions/ReservoirSampler.h index f82b0b856a0..c5d2158d59c 100644 --- a/src/AggregateFunctions/ReservoirSampler.h +++ b/src/AggregateFunctions/ReservoirSampler.h @@ -8,6 +8,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -190,9 +193,8 @@ public: std::string rng_string; DB::readStringBinary(rng_string, buf); - std::istringstream rng_stream(rng_string); - rng_stream.exceptions(std::ios::failbit); - rng_stream >> rng; + DB::ReadBufferFromString rng_buf(rng_string); + rng_buf >> rng; for (size_t i = 0; i < samples.size(); ++i) DB::readBinary(samples[i], buf); @@ -205,10 +207,9 @@ public: DB::writeIntBinary(sample_count, buf); DB::writeIntBinary(total_values, buf); - std::ostringstream rng_stream; - rng_stream.exceptions(std::ios::failbit); - rng_stream << rng; - DB::writeStringBinary(rng_stream.str(), buf); + DB::WriteBufferFromOwnString rng_buf; + rng_buf << rng; + DB::writeStringBinary(rng_buf.str(), buf); for (size_t i = 0; i < std::min(sample_count, total_values); ++i) DB::writeBinary(samples[i], buf); diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index b6cb55fa979..a99b0f9d7cc 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -222,19 +223,18 @@ std::string MultiplexedConnections::dumpAddresses() const std::string MultiplexedConnections::dumpAddressesUnlocked() const { bool is_first = true; - std::ostringstream os; - os.exceptions(std::ios::failbit); + WriteBufferFromOwnString buf; for (const ReplicaState & state : replica_states) { const Connection * connection = state.connection; if (connection) { - os << (is_first ? "" : "; ") << connection->getDescription(); + buf << (is_first ? "" : "; ") << connection->getDescription(); is_first = false; } } - return os.str(); + return buf.str(); } Packet MultiplexedConnections::receivePacketUnlocked() diff --git a/src/IO/Operators.h b/src/IO/Operators.h index 7e276ba9248..bd0b1de8fad 100644 --- a/src/IO/Operators.h +++ b/src/IO/Operators.h @@ -46,6 +46,7 @@ template WriteBuffer & operator<< (WriteBuffer & buf, const T & /// If you do not use the manipulators, the string is displayed without an escape, as is. template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const String & x) { writeString(x, buf); return buf; } template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const char & x) { writeChar(x, buf); return buf; } +template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const pcg32_fast & x) { PcgSerializer::serializePcg32(x, buf); return buf; } inline WriteBuffer & operator<< (WriteBuffer & buf, const char * x) { writeCString(x, buf); return buf; } @@ -73,6 +74,7 @@ inline WriteBuffer & operator<< (WriteBuffer & buf, FlushManip) { buf.next(); re template ReadBuffer & operator>> (ReadBuffer & buf, T & x) { readText(x, buf); return buf; } template <> inline ReadBuffer & operator>> (ReadBuffer & buf, String & x) { readString(x, buf); return buf; } template <> inline ReadBuffer & operator>> (ReadBuffer & buf, char & x) { readChar(x, buf); return buf; } +template <> inline ReadBuffer & operator>> (ReadBuffer & buf, pcg32_fast & x) { PcgDeserializer::deserializePcg32(x, buf); return buf; } /// If you specify a string literal for reading, this will mean - make sure there is a sequence of bytes and skip it. inline ReadBuffer & operator>> (ReadBuffer & buf, const char * x) { assertString(x, buf); return buf; } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 9ff1858c723..56ed4d20cef 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -53,6 +53,7 @@ namespace ErrorCodes extern const int CANNOT_READ_ARRAY_FROM_TEXT; extern const int CANNOT_PARSE_NUMBER; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; } /// Helper functions for formatted input. @@ -1228,4 +1229,23 @@ void saveUpToPosition(ReadBuffer & in, Memory<> & memory, char * current); */ bool loadAtPosition(ReadBuffer & in, Memory<> & memory, char * & current); + +struct PcgDeserializer +{ + static void deserializePcg32(const pcg32_fast & rng, ReadBuffer & buf) + { + decltype(rng.state_) multiplier, increment, state; + readText(multiplier, buf); + assertChar(' ', buf); + readText(increment, buf); + assertChar(' ', buf); + readText(state, buf); + + if (multiplier != rng.multiplier()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect multiplier in pcg32: expected {}, got {}", rng.multiplier(), multiplier); + if (increment != rng.increment()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect increment in pcg32: expected {}, got {}", rng.increment(), increment); + } +}; + } diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 9d517913582..a1d74e2fa6a 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -1093,4 +1093,16 @@ writeBinaryBigEndian(T x, WriteBuffer & buf) /// Assuming little endian archi writePODBinary(x, buf); } +struct PcgSerializer +{ + static void serializePcg32(const pcg32_fast & rng, WriteBuffer & buf) + { + writeText(rng.multiplier(), buf); + writeChar(' ', buf); + writeText(rng.increment(), buf); + writeChar(' ', buf); + writeText(rng.state_, buf); + } +}; + } diff --git a/utils/check-style/check-style b/utils/check-style/check-style index f9818a1e2bb..69d948c87fe 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -105,3 +105,6 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs # Trailing whitespaces find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P ' $' | grep -P '.' && echo "^ Trailing whitespaces." + +# Forbid stringstream because it's easy to use them incorrectly and hard to debug possible issues +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep 'std::ostringstream\|std::istringstream' && echo "Use WriteBufferFromString or ReadBufferFromString instead of std::ostringstream or std::istringstream" From c73c801825532e026ffa1908c800aca645e4de95 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Mon, 9 Nov 2020 17:49:48 +0300 Subject: [PATCH 063/114] Set sane default value for metric_log's collect inverval Otherwise server will crash on start if config value is missing. No test since we can't test configs. --- src/Interpreters/SystemLog.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 1e396a0fed1..01077602d90 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -25,6 +25,7 @@ namespace { constexpr size_t DEFAULT_SYSTEM_LOG_FLUSH_INTERVAL_MILLISECONDS = 7500; +constexpr size_t DEFAULT_METRIC_LOG_COLLECT_INTERVAL_MILLISECONDS = 1000; /// Creates a system log with MergeTree engine using parameters from config template @@ -125,7 +126,8 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi if (metric_log) { - size_t collect_interval_milliseconds = config.getUInt64("metric_log.collect_interval_milliseconds"); + size_t collect_interval_milliseconds = config.getUInt64("metric_log.collect_interval_milliseconds", + DEFAULT_METRIC_LOG_COLLECT_INTERVAL_MILLISECONDS); metric_log->startCollectMetric(collect_interval_milliseconds); } From 29944996807f41c5e5ad783a047ad687bcf936ae Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 9 Nov 2020 17:58:27 +0300 Subject: [PATCH 064/114] Update bloom_filter_select.xml --- tests/performance/bloom_filter_select.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/bloom_filter_select.xml b/tests/performance/bloom_filter_select.xml index f27a43c2991..90f737f8a7f 100644 --- a/tests/performance/bloom_filter_select.xml +++ b/tests/performance/bloom_filter_select.xml @@ -4,8 +4,8 @@ INSERT INTO test_bf_indexOf SELECT number AS id, [CAST(id, 'String'), CAST(id + 1, 'String'), CAST(id + 2, 'String')] FROM numbers(1000000) - SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') = 2 - SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0 + SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') = 2 + SELECT count() FROM test_bf_indexOf WHERE indexOf(ary, '1') > 0 DROP TABLE IF EXISTS test_bf_indexOf From dce9f26b024f845ec7ea02a1ce955921135be322 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 9 Nov 2020 18:12:40 +0300 Subject: [PATCH 065/114] Empty commit. --- src/Storages/IStorage.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 3b04c6cd632..ddd5b6727dc 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -80,6 +80,7 @@ TableExclusiveLockHolder IStorage::lockExclusively(const String & query_id, cons return result; } + Pipe IStorage::read( const Names & /*column_names*/, const StorageMetadataPtr & /*metadata_snapshot*/, From 62ff00ee8b2df2ce331e8cf4771d1bac9493c276 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 9 Nov 2020 19:05:40 +0300 Subject: [PATCH 066/114] use WriteBuffer in formatAST(...) --- programs/client/Client.cpp | 4 ++- programs/client/QueryFuzzer.cpp | 4 ++- programs/format/Format.cpp | 4 ++- programs/odbc-bridge/ColumnInfoHandler.cpp | 6 ++--- .../odbc-bridge/ODBCBlockOutputStream.cpp | 6 ++--- src/Access/DiskAccessStorage.cpp | 10 ++++--- src/Databases/DatabaseOnDisk.cpp | 16 +++++------- src/IO/Operators.h | 1 + src/IO/WriteHelpers.h | 5 ++++ src/Interpreters/AddDefaultDatabaseVisitor.h | 4 +-- .../ClusterProxy/SelectStreamFactory.cpp | 7 +++-- src/Interpreters/InDepthNodeVisitor.h | 4 +-- src/Interpreters/InterpreterCreateQuery.cpp | 14 ++++------ src/Interpreters/InterpreterExplainQuery.cpp | 20 ++++++-------- .../InterpreterShowAccessQuery.cpp | 9 +++---- ...InterpreterShowCreateAccessEntityQuery.cpp | 16 +++++------- .../InterpreterShowCreateQuery.cpp | 7 +++-- .../InterpreterShowGrantsQuery.cpp | 16 +++++------- src/Interpreters/QueryAliasesVisitor.cpp | 9 +++---- src/Parsers/ASTAlterQuery.cpp | 7 ++--- src/Parsers/ASTAsterisk.cpp | 1 + src/Parsers/ASTColumnDeclaration.cpp | 1 + src/Parsers/ASTColumnsMatcher.cpp | 1 + src/Parsers/ASTColumnsTransformers.cpp | 1 + src/Parsers/ASTConstraintDeclaration.cpp | 1 + src/Parsers/ASTCreateQuery.cpp | 1 + src/Parsers/ASTCreateQuotaQuery.cpp | 1 + src/Parsers/ASTCreateRoleQuery.cpp | 1 + src/Parsers/ASTCreateRowPolicyQuery.cpp | 10 +++---- src/Parsers/ASTCreateSettingsProfileQuery.cpp | 1 + src/Parsers/ASTCreateUserQuery.cpp | 1 + src/Parsers/ASTDictionary.cpp | 1 + .../ASTDictionaryAttributeDeclaration.cpp | 1 + src/Parsers/ASTDropAccessEntityQuery.cpp | 1 + src/Parsers/ASTDropQuery.cpp | 1 + src/Parsers/ASTExpressionList.cpp | 1 + src/Parsers/ASTFunction.cpp | 1 + .../ASTFunctionWithKeyValueArguments.cpp | 1 + src/Parsers/ASTGrantQuery.cpp | 1 + src/Parsers/ASTIdentifier.cpp | 1 + src/Parsers/ASTIndexDeclaration.cpp | 1 + src/Parsers/ASTInsertQuery.cpp | 1 + src/Parsers/ASTKillQueryQuery.cpp | 1 + src/Parsers/ASTLiteral.cpp | 5 ++++ src/Parsers/ASTLiteral.h | 5 +--- src/Parsers/ASTNameTypePair.cpp | 1 + src/Parsers/ASTOptimizeQuery.cpp | 1 + src/Parsers/ASTOrderByElement.cpp | 1 + src/Parsers/ASTPartition.cpp | 1 + src/Parsers/ASTQualifiedAsterisk.cpp | 1 + src/Parsers/ASTQueryParameter.cpp | 1 + src/Parsers/ASTQueryWithOnCluster.cpp | 1 + src/Parsers/ASTQueryWithOutput.h | 1 + src/Parsers/ASTQueryWithTableAndOutput.cpp | 1 + src/Parsers/ASTRenameQuery.h | 1 + src/Parsers/ASTRolesOrUsersSet.cpp | 1 + src/Parsers/ASTRowPolicyName.cpp | 1 + src/Parsers/ASTSampleRatio.cpp | 5 ++++ src/Parsers/ASTSampleRatio.h | 5 +--- src/Parsers/ASTSelectQuery.cpp | 1 + src/Parsers/ASTSelectWithUnionQuery.cpp | 1 + src/Parsers/ASTSetQuery.cpp | 1 + src/Parsers/ASTSetRoleQuery.cpp | 1 + src/Parsers/ASTSettingsProfileElement.cpp | 1 + src/Parsers/ASTShowAccessEntitiesQuery.cpp | 1 + .../ASTShowCreateAccessEntityQuery.cpp | 1 + src/Parsers/ASTShowGrantsQuery.cpp | 1 + src/Parsers/ASTShowPrivilegesQuery.cpp | 0 src/Parsers/ASTShowTablesQuery.cpp | 3 ++- src/Parsers/ASTSubquery.cpp | 1 + src/Parsers/ASTSystemQuery.cpp | 1 + src/Parsers/ASTTTLElement.cpp | 1 + src/Parsers/ASTTablesInSelectQuery.cpp | 1 + src/Parsers/ASTUseQuery.h | 1 + src/Parsers/ASTUserNameWithHost.cpp | 1 + src/Parsers/ASTWithAlias.cpp | 1 + src/Parsers/ASTWithElement.cpp | 1 + src/Parsers/CommonParsers.cpp | 1 + src/Parsers/DumpASTNode.h | 22 ++++++++-------- src/Parsers/IAST.cpp | 25 +++++++----------- src/Parsers/IAST.h | 17 ++++++------ src/Parsers/formatAST.cpp | 11 ++++---- src/Parsers/formatAST.h | 26 ++++++++++--------- src/Parsers/formatSettingName.cpp | 4 +-- src/Parsers/formatSettingName.h | 4 ++- src/Parsers/queryToString.cpp | 6 +---- src/Parsers/tests/create_parser.cpp | 4 ++- src/Parsers/tests/gtest_dictionary_parser.cpp | 8 +++--- src/Parsers/tests/select_parser.cpp | 4 ++- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 7 +++-- .../ReplicatedMergeTreeTableMetadata.cpp | 7 +++-- src/Storages/MutationCommands.cpp | 7 +++-- src/Storages/StorageMergeTree.cpp | 7 +++-- .../transformQueryForExternalDatabase.cpp | 3 +-- utils/db-generator/query_db_generator.cpp | 19 +++++++------- 95 files changed, 241 insertions(+), 193 deletions(-) delete mode 100644 src/Parsers/ASTShowPrivilegesQuery.cpp diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index fc452c165e7..f9e4b357aa9 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -54,6 +54,7 @@ #include #include #include +#include #include #include #include @@ -1529,7 +1530,8 @@ private: if (is_interactive) { std::cout << std::endl; - formatAST(*res, std::cout); + WriteBufferFromOStream res_buf(std::cout, 4096); + formatAST(*res, res_buf); std::cout << std::endl << std::endl; } diff --git a/programs/client/QueryFuzzer.cpp b/programs/client/QueryFuzzer.cpp index 3fe6b8087f0..cb058f6db00 100644 --- a/programs/client/QueryFuzzer.cpp +++ b/programs/client/QueryFuzzer.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -419,7 +420,8 @@ void QueryFuzzer::fuzzMain(ASTPtr & ast) fuzz(ast); std::cout << std::endl; - formatAST(*ast, std::cout, false /*highlight*/); + WriteBufferFromOStream ast_buf(std::cout, 4096); + formatAST(*ast, ast_buf, false /*highlight*/); std::cout << std::endl << std::endl; } diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 01f952bf95e..c9981e706b0 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -129,7 +130,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ASTPtr res = parseQueryAndMovePosition(parser, pos, end, "query", multiple, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (!quiet) { - formatAST(*res, std::cout, hilite, oneline); + WriteBufferFromOStream res_buf(std::cout, 4096); + formatAST(*res, res_buf, hilite, oneline); if (multiple) std::cout << "\n;\n"; std::cout << std::endl; diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 33723b4cc8e..9e2307715c5 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -113,16 +113,16 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques /// TODO Why not do SQLColumns instead? std::string name = schema_name.empty() ? backQuoteIfNeed(table_name) : backQuoteIfNeed(schema_name) + "." + backQuoteIfNeed(table_name); - std::stringstream ss; + WriteBufferFromOwnString buf; std::string input = "SELECT * FROM " + name + " WHERE 1 = 0"; ParserQueryWithOutput parser; ASTPtr select = parseQuery(parser, input.data(), input.data() + input.size(), "", context_settings.max_query_size, context_settings.max_parser_depth); - IAST::FormatSettings settings(ss, true); + IAST::FormatSettings settings(buf, true); settings.always_quote_identifiers = true; settings.identifier_quoting_style = getQuotingStyle(hdbc); select->format(settings); - std::string query = ss.str(); + std::string query = buf.str(); LOG_TRACE(log, "Inferring structure with query '{}'", query); diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp index 82ca861ea67..4d8b9fa6bdf 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -32,12 +32,12 @@ namespace for (const auto & column : columns) query.columns->children.emplace_back(std::make_shared(column.name)); - std::stringstream ss; - IAST::FormatSettings settings(ss, true); + WriteBufferFromOwnString buf; + IAST::FormatSettings settings(buf, true); settings.always_quote_identifiers = true; settings.identifier_quoting_style = quoting; query.IAST::format(settings); - return ss.str(); + return buf.str(); } std::string getQuestionMarks(size_t n) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index abf4ff12d5a..b03a2791560 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -197,11 +197,13 @@ namespace boost::range::push_back(queries, InterpreterShowGrantsQuery::getAttachGrantQueries(entity)); /// Serialize the list of ATTACH queries to a string. - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString buf; for (const ASTPtr & query : queries) - ss << *query << ";\n"; - String file_contents = std::move(ss).str(); + { + formatAST(*query, buf, false, true); + buf.write(";\n", 2); + } + String file_contents = buf.str(); /// First we save *.tmp file and then we rename if everything's ok. auto tmp_file_path = std::filesystem::path{file_path}.replace_extension(".tmp"); diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 83e70a25f87..1e6b4019c4b 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -94,10 +94,9 @@ String getObjectDefinitionFromCreateQuery(const ASTPtr & query) if (!create) { - std::ostringstream query_stream; - query_stream.exceptions(std::ios::failbit); - formatAST(*query, query_stream, true); - throw Exception("Query '" + query_stream.str() + "' is not CREATE query", ErrorCodes::LOGICAL_ERROR); + WriteBufferFromOwnString query_buf; + formatAST(*query, query_buf, true); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query '{}' is not CREATE query", query_buf.str()); } if (!create->is_dictionary) @@ -121,11 +120,10 @@ String getObjectDefinitionFromCreateQuery(const ASTPtr & query) if (create->uuid != UUIDHelpers::Nil) create->table = TABLE_WITH_UUID_NAME_PLACEHOLDER; - std::ostringstream statement_stream; - statement_stream.exceptions(std::ios::failbit); - formatAST(*create, statement_stream, false); - statement_stream << '\n'; - return statement_stream.str(); + WriteBufferFromOwnString statement_buf; + formatAST(*create, statement_buf, false); + writeChar('\n', statement_buf); + return statement_buf.str(); } DatabaseOnDisk::DatabaseOnDisk( diff --git a/src/IO/Operators.h b/src/IO/Operators.h index bd0b1de8fad..d1500aedd22 100644 --- a/src/IO/Operators.h +++ b/src/IO/Operators.h @@ -45,6 +45,7 @@ struct BinaryManipReadBuffer : std::reference_wrapper { usin template WriteBuffer & operator<< (WriteBuffer & buf, const T & x) { writeText(x, buf); return buf; } /// If you do not use the manipulators, the string is displayed without an escape, as is. template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const String & x) { writeString(x, buf); return buf; } +template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const std::string_view & x) { writeString(StringRef(x), buf); return buf; } template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const char & x) { writeChar(x, buf); return buf; } template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const pcg32_fast & x) { PcgSerializer::serializePcg32(x, buf); return buf; } diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index a1d74e2fa6a..da68c4aded5 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -271,6 +271,11 @@ inline void writeString(const StringRef & ref, WriteBuffer & buf) writeString(ref.data, ref.size, buf); } +//inline void writeString(const std::string_view & view, WriteBuffer & buf) +//{ +// writeString(view.data(), view.size(), buf); +//} + /** Writes a C-string without creating a temporary object. If the string is a literal, then `strlen` is executed at the compilation stage. * Use when the string is a literal. diff --git a/src/Interpreters/AddDefaultDatabaseVisitor.h b/src/Interpreters/AddDefaultDatabaseVisitor.h index bb684c5547a..7b72374c9c6 100644 --- a/src/Interpreters/AddDefaultDatabaseVisitor.h +++ b/src/Interpreters/AddDefaultDatabaseVisitor.h @@ -25,7 +25,7 @@ class AddDefaultDatabaseVisitor { public: explicit AddDefaultDatabaseVisitor( - const String & database_name_, bool only_replace_current_database_function_ = false, std::ostream * ostr_ = nullptr) + const String & database_name_, bool only_replace_current_database_function_ = false, WriteBuffer * ostr_ = nullptr) : database_name(database_name_) , only_replace_current_database_function(only_replace_current_database_function_) , visit_depth(0) @@ -66,7 +66,7 @@ private: const String database_name; bool only_replace_current_database_function = false; mutable size_t visit_depth; - std::ostream * ostr; + WriteBuffer * ostr; void visit(ASTSelectWithUnionQuery & select, ASTPtr &) const { diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 9e64695d1a0..f0f68c0d93f 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -106,10 +106,9 @@ String formattedAST(const ASTPtr & ast) { if (!ast) return {}; - std::stringstream ss; - ss.exceptions(std::ios::failbit); - formatAST(*ast, ss, false, true); - return ss.str(); + WriteBufferFromOwnString buf; + formatAST(*ast, buf, false, true); + return buf.str(); } } diff --git a/src/Interpreters/InDepthNodeVisitor.h b/src/Interpreters/InDepthNodeVisitor.h index 7b537f0daa0..7a793566cdd 100644 --- a/src/Interpreters/InDepthNodeVisitor.h +++ b/src/Interpreters/InDepthNodeVisitor.h @@ -16,7 +16,7 @@ class InDepthNodeVisitor public: using Data = typename Matcher::Data; - InDepthNodeVisitor(Data & data_, std::ostream * ostr_ = nullptr) + InDepthNodeVisitor(Data & data_, WriteBuffer * ostr_ = nullptr) : data(data_), visit_depth(0), ostr(ostr_) @@ -46,7 +46,7 @@ public: private: Data & data; size_t visit_depth; - std::ostream * ostr; + WriteBuffer * ostr; void visitChildren(T & ast) { diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 286d5269a64..d6e4dc666e7 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -135,10 +135,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) else if ((create.columns_list && create.columns_list->indices && !create.columns_list->indices->children.empty())) { /// Currently, there are no database engines, that support any arguments. - std::stringstream ostr; - ostr.exceptions(std::ios::failbit); - formatAST(*create.storage, ostr, false, false); - throw Exception("Unknown database engine: " + ostr.str(), ErrorCodes::UNKNOWN_DATABASE_ENGINE); + throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", serializeAST(*create.storage)); } if (create.storage->engine->name == "Atomic") @@ -182,11 +179,10 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) create.attach = true; create.if_not_exists = false; - std::ostringstream statement_stream; - statement_stream.exceptions(std::ios::failbit); - formatAST(create, statement_stream, false); - statement_stream << '\n'; - String statement = statement_stream.str(); + WriteBufferFromOwnString statement_buf; + formatAST(create, statement_buf, false); + writeChar('\n', statement_buf); + String statement = statement_buf.str(); /// Exclusive flag guarantees, that database is not created right now in another thread. WriteBufferFromFile out(metadata_file_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index ed791f0d592..ae8e2637cbb 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -222,15 +222,14 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() Block sample_block = getSampleBlock(); MutableColumns res_columns = sample_block.cloneEmptyColumns(); - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString buf; if (ast.getKind() == ASTExplainQuery::ParsedAST) { if (ast.getSettings()) throw Exception("Settings are not supported for EXPLAIN AST query.", ErrorCodes::UNKNOWN_SETTING); - dumpAST(*ast.getExplainedQuery(), ss); + dumpAST(*ast.getExplainedQuery(), buf); } else if (ast.getKind() == ASTExplainQuery::AnalyzedSyntax) { @@ -240,7 +239,7 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() ExplainAnalyzedSyntaxVisitor::Data data{.context = context}; ExplainAnalyzedSyntaxVisitor(data).visit(query); - ast.getExplainedQuery()->format(IAST::FormatSettings(ss, false)); + ast.getExplainedQuery()->format(IAST::FormatSettings(buf, false)); } else if (ast.getKind() == ASTExplainQuery::QueryPlan) { @@ -256,8 +255,7 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() if (settings.optimize) plan.optimize(); - WriteBufferFromOStream buffer(ss); - plan.explainPlan(buffer, settings.query_plan_options); + plan.explainPlan(buf, settings.query_plan_options); } else if (ast.getKind() == ASTExplainQuery::QueryPipeline) { @@ -271,8 +269,6 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() interpreter.buildQueryPlan(plan); auto pipeline = plan.buildQueryPipeline(); - WriteBufferFromOStream buffer(ss); - if (settings.graph) { /// Pipe holds QueryPlan, should not go out-of-scope @@ -280,17 +276,17 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() const auto & processors = pipe.getProcessors(); if (settings.compact) - printPipelineCompact(processors, buffer, settings.query_pipeline_options.header); + printPipelineCompact(processors, buf, settings.query_pipeline_options.header); else - printPipeline(processors, buffer); + printPipeline(processors, buf); } else { - plan.explainPipeline(buffer, settings.query_pipeline_options); + plan.explainPipeline(buf, settings.query_pipeline_options); } } - fillColumn(*res_columns[0], ss.str()); + fillColumn(*res_columns[0], buf.str()); return std::make_shared(sample_block.cloneWithColumns(std::move(res_columns))); } diff --git a/src/Interpreters/InterpreterShowAccessQuery.cpp b/src/Interpreters/InterpreterShowAccessQuery.cpp index 5f28c49c0bc..7bec7c411f0 100644 --- a/src/Interpreters/InterpreterShowAccessQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessQuery.cpp @@ -34,13 +34,12 @@ BlockInputStreamPtr InterpreterShowAccessQuery::executeImpl() const /// Build the result column. MutableColumnPtr column = ColumnString::create(); - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString buf; for (const auto & query : queries) { - ss.str(""); - formatAST(*query, ss, false, true); - column->insert(ss.str()); + buf.restart(); + formatAST(*query, buf, false, true); + column->insert(buf.str()); } String desc = "ACCESS"; diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 749a5811e13..a81245adfc9 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -238,21 +238,19 @@ BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() /// Build the result column. MutableColumnPtr column = ColumnString::create(); - std::stringstream create_query_ss; - create_query_ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString create_query_buf; for (const auto & create_query : create_queries) { - formatAST(*create_query, create_query_ss, false, true); - column->insert(create_query_ss.str()); - create_query_ss.str(""); + formatAST(*create_query, create_query_buf, false, true); + column->insert(create_query_buf.str()); + create_query_buf.restart(); } /// Prepare description of the result column. - std::stringstream desc_ss; - desc_ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString desc_buf; const auto & show_query = query_ptr->as(); - formatAST(show_query, desc_ss, false, true); - String desc = desc_ss.str(); + formatAST(show_query, desc_buf, false, true); + String desc = desc_buf.str(); String prefix = "SHOW "; if (startsWith(desc, prefix)) desc = desc.substr(prefix.length()); /// `desc` always starts with "SHOW ", so we can trim this prefix. diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 8861914a68a..32f461863c9 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -78,10 +78,9 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() create.uuid = UUIDHelpers::Nil; } - std::stringstream stream; - stream.exceptions(std::ios::failbit); - formatAST(*create_query, stream, false, false); - String res = stream.str(); + WriteBufferFromOwnString buf; + formatAST(*create_query, buf, false, false); + String res = buf.str(); MutableColumnPtr column = ColumnString::create(); column->insert(res); diff --git a/src/Interpreters/InterpreterShowGrantsQuery.cpp b/src/Interpreters/InterpreterShowGrantsQuery.cpp index 7de51b6a7ee..a2ddc5eec27 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/InterpreterShowGrantsQuery.cpp @@ -118,21 +118,19 @@ BlockInputStreamPtr InterpreterShowGrantsQuery::executeImpl() /// Build the result column. MutableColumnPtr column = ColumnString::create(); - std::stringstream grant_ss; - grant_ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString grant_buf; for (const auto & grant_query : grant_queries) { - grant_ss.str(""); - formatAST(*grant_query, grant_ss, false, true); - column->insert(grant_ss.str()); + grant_buf.restart(); + formatAST(*grant_query, grant_buf, false, true); + column->insert(grant_buf.str()); } /// Prepare description of the result column. - std::stringstream desc_ss; - desc_ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString desc_buf; const auto & show_query = query_ptr->as(); - formatAST(show_query, desc_ss, false, true); - String desc = desc_ss.str(); + formatAST(show_query, desc_buf, false, true); + String desc = desc_buf.str(); String prefix = "SHOW "; if (desc.starts_with(prefix)) desc = desc.substr(prefix.length()); /// `desc` always starts with "SHOW ", so we can trim this prefix. diff --git a/src/Interpreters/QueryAliasesVisitor.cpp b/src/Interpreters/QueryAliasesVisitor.cpp index 9de1d04990d..00c337920f4 100644 --- a/src/Interpreters/QueryAliasesVisitor.cpp +++ b/src/Interpreters/QueryAliasesVisitor.cpp @@ -20,13 +20,12 @@ namespace ErrorCodes static String wrongAliasMessage(const ASTPtr & ast, const ASTPtr & prev_ast, const String & alias) { - std::stringstream message; - message.exceptions(std::ios::failbit); - message << "Different expressions with the same alias " << backQuoteIfNeed(alias) << ":" << std::endl; + WriteBufferFromOwnString message; + message << "Different expressions with the same alias " << backQuoteIfNeed(alias) << ":\n"; formatAST(*ast, message, false, true); - message << std::endl << "and" << std::endl; + message << "\nand\n"; formatAST(*prev_ast, message, false, true); - message << std::endl; + message << '\n'; return message.str(); } diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index d74156d11d8..89130808e15 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -246,7 +247,7 @@ void ASTAlterCommand::formatImpl( << "PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); settings.ostr << (settings.hilite ? hilite_keyword : "") - << " FROM " << (settings.hilite ? hilite_none : "") << std::quoted(from, '\''); + << " FROM " << (settings.hilite ? hilite_none : "") << DB::quote << from; } else if (type == ASTAlterCommand::FREEZE_PARTITION) { @@ -256,7 +257,7 @@ void ASTAlterCommand::formatImpl( if (!with_name.empty()) { settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") - << " " << std::quoted(with_name, '\''); + << " " << DB::quote << with_name; } } else if (type == ASTAlterCommand::FREEZE_ALL) @@ -266,7 +267,7 @@ void ASTAlterCommand::formatImpl( if (!with_name.empty()) { settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") - << " " << std::quoted(with_name, '\''); + << " " << DB::quote << with_name; } } else if (type == ASTAlterCommand::DELETE) diff --git a/src/Parsers/ASTAsterisk.cpp b/src/Parsers/ASTAsterisk.cpp index 95a63586685..ed733e62ca3 100644 --- a/src/Parsers/ASTAsterisk.cpp +++ b/src/Parsers/ASTAsterisk.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index 27ece3e18c2..4c14230e926 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index aab8e841981..45799cb7ffe 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index a8f39079902..7c0fd0eb734 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTConstraintDeclaration.cpp b/src/Parsers/ASTConstraintDeclaration.cpp index 371bfa40f54..7d74837478c 100644 --- a/src/Parsers/ASTConstraintDeclaration.cpp +++ b/src/Parsers/ASTConstraintDeclaration.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 4efe5476395..a193433c988 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index 88516fb6eac..7e570b889e3 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTCreateRoleQuery.cpp b/src/Parsers/ASTCreateRoleQuery.cpp index 5ccfd9c6bd5..73b523a5bfe 100644 --- a/src/Parsers/ASTCreateRoleQuery.cpp +++ b/src/Parsers/ASTCreateRoleQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index 6224b534851..241d3ff051a 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -62,14 +63,13 @@ namespace void formatForClauses(const std::vector> & conditions, bool alter, const IAST::FormatSettings & settings) { std::vector> conditions_as_strings; - std::stringstream temp_sstream; - temp_sstream.exceptions(std::ios::failbit); - IAST::FormatSettings temp_settings(temp_sstream, settings); + WriteBufferFromOwnString temp_buf; + IAST::FormatSettings temp_settings(temp_buf, settings); for (const auto & [condition_type, condition] : conditions) { formatConditionalExpression(condition, temp_settings); - conditions_as_strings.emplace_back(condition_type, temp_sstream.str()); - temp_sstream.str(""); + conditions_as_strings.emplace_back(condition_type, temp_buf.str()); + temp_buf.restart(); } boost::container::flat_set commands; diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.cpp b/src/Parsers/ASTCreateSettingsProfileQuery.cpp index 77c2f1b22d7..84f8309462e 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ASTCreateSettingsProfileQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index 0ccc2232734..4b2aa70785a 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTDictionary.cpp b/src/Parsers/ASTDictionary.cpp index e0785f3bd49..878f6000aa9 100644 --- a/src/Parsers/ASTDictionary.cpp +++ b/src/Parsers/ASTDictionary.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { diff --git a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp index 05ba48ace7b..e9c50839a98 100644 --- a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp +++ b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTDropAccessEntityQuery.cpp b/src/Parsers/ASTDropAccessEntityQuery.cpp index fe98d8b4158..1df176c24ec 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.cpp +++ b/src/Parsers/ASTDropAccessEntityQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTDropQuery.cpp b/src/Parsers/ASTDropQuery.cpp index 0ced4a8ea96..b09b588ca6e 100644 --- a/src/Parsers/ASTDropQuery.cpp +++ b/src/Parsers/ASTDropQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTExpressionList.cpp b/src/Parsers/ASTExpressionList.cpp index de38e1fd7ea..2724465537f 100644 --- a/src/Parsers/ASTExpressionList.cpp +++ b/src/Parsers/ASTExpressionList.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 66565eeaf8f..76a52b8c641 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTFunctionWithKeyValueArguments.cpp b/src/Parsers/ASTFunctionWithKeyValueArguments.cpp index 5f1e78b61da..d94490ab8b3 100644 --- a/src/Parsers/ASTFunctionWithKeyValueArguments.cpp +++ b/src/Parsers/ASTFunctionWithKeyValueArguments.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace DB { diff --git a/src/Parsers/ASTGrantQuery.cpp b/src/Parsers/ASTGrantQuery.cpp index fefebc4ae4a..2610836c759 100644 --- a/src/Parsers/ASTGrantQuery.cpp +++ b/src/Parsers/ASTGrantQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 5a66bc7891d..d51f37a0047 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index e89f9bf26ed..0e8f0d0f7e8 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTInsertQuery.cpp b/src/Parsers/ASTInsertQuery.cpp index dc9b9f092ac..4096c484059 100644 --- a/src/Parsers/ASTInsertQuery.cpp +++ b/src/Parsers/ASTInsertQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTKillQueryQuery.cpp b/src/Parsers/ASTKillQueryQuery.cpp index 293b95b93bf..72bdd7d6b0b 100644 --- a/src/Parsers/ASTKillQueryQuery.cpp +++ b/src/Parsers/ASTKillQueryQuery.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { diff --git a/src/Parsers/ASTLiteral.cpp b/src/Parsers/ASTLiteral.cpp index cd9c389f336..ed6790499fb 100644 --- a/src/Parsers/ASTLiteral.cpp +++ b/src/Parsers/ASTLiteral.cpp @@ -73,4 +73,9 @@ void ASTLiteral::appendColumnNameImpl(WriteBuffer & ostr) const } } +void ASTLiteral::formatImplWithoutAlias(const FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +{ + settings.ostr << applyVisitor(FieldVisitorToString(), value); +} + } diff --git a/src/Parsers/ASTLiteral.h b/src/Parsers/ASTLiteral.h index 18f440a81a4..672bc6ddc3e 100644 --- a/src/Parsers/ASTLiteral.h +++ b/src/Parsers/ASTLiteral.h @@ -43,10 +43,7 @@ public: void updateTreeHashImpl(SipHash & hash_state) const override; protected: - void formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const override - { - settings.ostr << applyVisitor(FieldVisitorToString(), value); - } + void formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; }; diff --git a/src/Parsers/ASTNameTypePair.cpp b/src/Parsers/ASTNameTypePair.cpp index 80b0f7f8ec6..e4066081a9b 100644 --- a/src/Parsers/ASTNameTypePair.cpp +++ b/src/Parsers/ASTNameTypePair.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTOptimizeQuery.cpp b/src/Parsers/ASTOptimizeQuery.cpp index 92968f2b277..ae83952899d 100644 --- a/src/Parsers/ASTOptimizeQuery.cpp +++ b/src/Parsers/ASTOptimizeQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { diff --git a/src/Parsers/ASTOrderByElement.cpp b/src/Parsers/ASTOrderByElement.cpp index 48290b2669a..884d69a18e3 100644 --- a/src/Parsers/ASTOrderByElement.cpp +++ b/src/Parsers/ASTOrderByElement.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTPartition.cpp b/src/Parsers/ASTPartition.cpp index d24575b7f43..06bfe4f5217 100644 --- a/src/Parsers/ASTPartition.cpp +++ b/src/Parsers/ASTPartition.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { diff --git a/src/Parsers/ASTQualifiedAsterisk.cpp b/src/Parsers/ASTQualifiedAsterisk.cpp index 0cda01cecac..2491dcb36b7 100644 --- a/src/Parsers/ASTQualifiedAsterisk.cpp +++ b/src/Parsers/ASTQualifiedAsterisk.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { diff --git a/src/Parsers/ASTQueryParameter.cpp b/src/Parsers/ASTQueryParameter.cpp index 915ecd5e7e4..c10cced23ce 100644 --- a/src/Parsers/ASTQueryParameter.cpp +++ b/src/Parsers/ASTQueryParameter.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTQueryWithOnCluster.cpp b/src/Parsers/ASTQueryWithOnCluster.cpp index cb77d351d54..60e96b1dbe1 100644 --- a/src/Parsers/ASTQueryWithOnCluster.cpp +++ b/src/Parsers/ASTQueryWithOnCluster.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTQueryWithOutput.h b/src/Parsers/ASTQueryWithOutput.h index 9018d5661d9..92f9331f259 100644 --- a/src/Parsers/ASTQueryWithOutput.h +++ b/src/Parsers/ASTQueryWithOutput.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB diff --git a/src/Parsers/ASTQueryWithTableAndOutput.cpp b/src/Parsers/ASTQueryWithTableAndOutput.cpp index 3a776590f80..d44ba988d7a 100644 --- a/src/Parsers/ASTQueryWithTableAndOutput.cpp +++ b/src/Parsers/ASTQueryWithTableAndOutput.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTRenameQuery.h b/src/Parsers/ASTRenameQuery.h index 951abbe4419..8d300f430b3 100644 --- a/src/Parsers/ASTRenameQuery.h +++ b/src/Parsers/ASTRenameQuery.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTRolesOrUsersSet.cpp b/src/Parsers/ASTRolesOrUsersSet.cpp index a666d8ae1d5..1e7cd79f527 100644 --- a/src/Parsers/ASTRolesOrUsersSet.cpp +++ b/src/Parsers/ASTRolesOrUsersSet.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTRowPolicyName.cpp b/src/Parsers/ASTRowPolicyName.cpp index 5e3c494ccd3..3d1ac5621db 100644 --- a/src/Parsers/ASTRowPolicyName.cpp +++ b/src/Parsers/ASTRowPolicyName.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB diff --git a/src/Parsers/ASTSampleRatio.cpp b/src/Parsers/ASTSampleRatio.cpp index 8c5901d121d..03a4c9adf23 100644 --- a/src/Parsers/ASTSampleRatio.cpp +++ b/src/Parsers/ASTSampleRatio.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -34,5 +35,9 @@ String ASTSampleRatio::toString(Rational ratio) return toString(ratio.numerator) + " / " + toString(ratio.denominator); } +void ASTSampleRatio::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +{ + settings.ostr << toString(ratio); +} } diff --git a/src/Parsers/ASTSampleRatio.h b/src/Parsers/ASTSampleRatio.h index 787833eb4f3..e8953dec022 100644 --- a/src/Parsers/ASTSampleRatio.h +++ b/src/Parsers/ASTSampleRatio.h @@ -35,10 +35,7 @@ public: static String toString(BigNum num); static String toString(Rational ratio); - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override - { - settings.ostr << toString(ratio); - } + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 499761c4634..915d1f71925 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 96cac839c58..9c13a1a4ff3 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTSetQuery.cpp b/src/Parsers/ASTSetQuery.cpp index 8835d1dc7da..c8a2b3b37e8 100644 --- a/src/Parsers/ASTSetQuery.cpp +++ b/src/Parsers/ASTSetQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTSetRoleQuery.cpp b/src/Parsers/ASTSetRoleQuery.cpp index b5e0c05e083..e59e103b774 100644 --- a/src/Parsers/ASTSetRoleQuery.cpp +++ b/src/Parsers/ASTSetRoleQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTSettingsProfileElement.cpp b/src/Parsers/ASTSettingsProfileElement.cpp index c0fb2965a2d..2422126219f 100644 --- a/src/Parsers/ASTSettingsProfileElement.cpp +++ b/src/Parsers/ASTSettingsProfileElement.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/ASTShowAccessEntitiesQuery.cpp index e87baebba33..bacde098640 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ASTShowAccessEntitiesQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp index bc309ab5c44..f870c98071c 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTShowGrantsQuery.cpp b/src/Parsers/ASTShowGrantsQuery.cpp index 26ae506d7d4..4011cfc522c 100644 --- a/src/Parsers/ASTShowGrantsQuery.cpp +++ b/src/Parsers/ASTShowGrantsQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTShowPrivilegesQuery.cpp b/src/Parsers/ASTShowPrivilegesQuery.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/Parsers/ASTShowTablesQuery.cpp b/src/Parsers/ASTShowTablesQuery.cpp index 1e8dad13ad3..cd83bae06d9 100644 --- a/src/Parsers/ASTShowTablesQuery.cpp +++ b/src/Parsers/ASTShowTablesQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB { @@ -21,7 +22,7 @@ void ASTShowTablesQuery::formatLike(const FormatSettings & settings) const << (not_like ? " NOT" : "") << (case_insensitive_like ? " ILIKE " : " LIKE ") << (settings.hilite ? hilite_none : "") - << std::quoted(like, '\''); + << DB::quote << like; } void ASTShowTablesQuery::formatLimit(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const diff --git a/src/Parsers/ASTSubquery.cpp b/src/Parsers/ASTSubquery.cpp index 55ea89e3f07..bfe413908b9 100644 --- a/src/Parsers/ASTSubquery.cpp +++ b/src/Parsers/ASTSubquery.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 9cbb6ae94f6..4ed0ecd3a91 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp index f37631769b8..39283a3168e 100644 --- a/src/Parsers/ASTTTLElement.cpp +++ b/src/Parsers/ASTTTLElement.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp index eb3446ca1c4..8d131a848f7 100644 --- a/src/Parsers/ASTTablesInSelectQuery.cpp +++ b/src/Parsers/ASTTablesInSelectQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTUseQuery.h b/src/Parsers/ASTUseQuery.h index 2127bf9f2c0..4e4a13c2a7f 100644 --- a/src/Parsers/ASTUseQuery.h +++ b/src/Parsers/ASTUseQuery.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTUserNameWithHost.cpp b/src/Parsers/ASTUserNameWithHost.cpp index 13d34b99b3d..b99ea5ab8d4 100644 --- a/src/Parsers/ASTUserNameWithHost.cpp +++ b/src/Parsers/ASTUserNameWithHost.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB diff --git a/src/Parsers/ASTWithAlias.cpp b/src/Parsers/ASTWithAlias.cpp index ad93102e1b7..1feb89f4bdc 100644 --- a/src/Parsers/ASTWithAlias.cpp +++ b/src/Parsers/ASTWithAlias.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/ASTWithElement.cpp b/src/Parsers/ASTWithElement.cpp index 9d22286c2fd..00a82b703af 100644 --- a/src/Parsers/ASTWithElement.cpp +++ b/src/Parsers/ASTWithElement.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { diff --git a/src/Parsers/CommonParsers.cpp b/src/Parsers/CommonParsers.cpp index 47868f5df48..d7a9ed60ac3 100644 --- a/src/Parsers/CommonParsers.cpp +++ b/src/Parsers/CommonParsers.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include /// strncmp, strncasecmp diff --git a/src/Parsers/DumpASTNode.h b/src/Parsers/DumpASTNode.h index 01447850c74..1208aeca2a9 100644 --- a/src/Parsers/DumpASTNode.h +++ b/src/Parsers/DumpASTNode.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -14,7 +15,7 @@ namespace DB class DumpASTNode { public: - DumpASTNode(const IAST & ast_, std::ostream * ostr_, size_t & depth, const char * label_ = nullptr) + DumpASTNode(const IAST & ast_, WriteBuffer * ostr_, size_t & depth, const char * label_ = nullptr) : ast(ast_), ostr(ostr_), indent(depth), @@ -24,12 +25,12 @@ public: if (!ostr) return; if (label && visit_depth == 0) - (*ostr) << "-- " << label << std::endl; + (*ostr) << "-- " << label << '\n'; ++visit_depth; (*ostr) << String(indent, ' '); printNode(); - (*ostr) << std::endl; + (*ostr) << '\n'; } ~DumpASTNode() @@ -38,7 +39,7 @@ public: return; --visit_depth; if (label && visit_depth == 0) - (*ostr) << "--" << std::endl; + (*ostr) << "--\n"; } template @@ -50,14 +51,14 @@ public: (*ostr) << (str_indent ? String(str_indent) : String(indent, ' ')); (*ostr) << '(' << name << ' ' << value << ')'; if (!str_indent) - (*ostr) << std::endl; + (*ostr) << '\n'; } size_t & getDepth() { return visit_depth; } private: const IAST & ast; - std::ostream * ostr; + WriteBuffer * ostr; size_t indent; size_t & visit_depth; /// shared with children const char * label; @@ -77,7 +78,7 @@ private: } }; -inline void dumpAST(const IAST & ast, std::ostream & ostr, DumpASTNode * parent = nullptr) +inline void dumpAST(const IAST & ast, WriteBuffer & ostr, DumpASTNode * parent = nullptr) { size_t depth = 0; DumpASTNode dump(ast, &ostr, (parent ? parent->getDepth() : depth)); @@ -95,7 +96,6 @@ public: DebugASTLog() : log(nullptr) { - ss.exceptions(std::ios::failbit); if constexpr (_enable) log = &Poco::Logger::get("AST"); } @@ -103,14 +103,14 @@ public: ~DebugASTLog() { if constexpr (_enable) - LOG_DEBUG(log, ss.str()); + LOG_DEBUG(log, buf.str()); } - std::ostream * stream() { return (_enable ? &ss : nullptr); } + WriteBuffer * stream() { return (_enable ? &buf : nullptr); } private: Poco::Logger * log; - std::stringstream ss; + WriteBufferFromOwnString buf; }; diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index d716a796b77..4b7d3c2ea40 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -89,10 +89,9 @@ size_t IAST::checkDepthImpl(size_t max_depth, size_t level) const std::string IAST::formatForErrorMessage() const { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - format(FormatSettings(ss, true /* one line */)); - return ss.str(); + WriteBufferFromOwnString buf; + format(FormatSettings(buf, true /* one line */)); + return buf.str(); } void IAST::cloneChildren() @@ -112,8 +111,6 @@ String IAST::getColumnName() const void IAST::FormatSettings::writeIdentifier(const String & name) const { - WriteBufferFromOStream out(ostr, 32); - switch (identifier_quoting_style) { case IdentifierQuotingStyle::None: @@ -121,36 +118,34 @@ void IAST::FormatSettings::writeIdentifier(const String & name) const if (always_quote_identifiers) throw Exception("Incompatible arguments: always_quote_identifiers = true && identifier_quoting_style == IdentifierQuotingStyle::None", ErrorCodes::BAD_ARGUMENTS); - writeString(name, out); + writeString(name, ostr); break; } case IdentifierQuotingStyle::Backticks: { if (always_quote_identifiers) - writeBackQuotedString(name, out); + writeBackQuotedString(name, ostr); else - writeProbablyBackQuotedString(name, out); + writeProbablyBackQuotedString(name, ostr); break; } case IdentifierQuotingStyle::DoubleQuotes: { if (always_quote_identifiers) - writeDoubleQuotedString(name, out); + writeDoubleQuotedString(name, ostr); else - writeProbablyDoubleQuotedString(name, out); + writeProbablyDoubleQuotedString(name, ostr); break; } case IdentifierQuotingStyle::BackticksMySQL: { if (always_quote_identifiers) - writeBackQuotedStringMySQL(name, out); + writeBackQuotedStringMySQL(name, ostr); else - writeProbablyBackQuotedStringMySQL(name, out); + writeProbablyBackQuotedStringMySQL(name, ostr); break; } } - - out.next(); } } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index cc9e593d7cb..d9fd71378f7 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -161,7 +162,7 @@ public: /// Format settings. struct FormatSettings { - std::ostream & ostr; + WriteBuffer & ostr; bool hilite = false; bool one_line; bool always_quote_identifiers = false; @@ -169,13 +170,13 @@ public: char nl_or_ws; - FormatSettings(std::ostream & ostr_, bool one_line_) + FormatSettings(WriteBuffer & ostr_, bool one_line_) : ostr(ostr_), one_line(one_line_) { nl_or_ws = one_line ? ' ' : '\n'; } - FormatSettings(std::ostream & ostr_, const FormatSettings & other) + FormatSettings(WriteBuffer & ostr_, const FormatSettings & other) : ostr(ostr_), hilite(other.hilite), one_line(other.one_line), always_quote_identifiers(other.always_quote_identifiers), identifier_quoting_style(other.identifier_quoting_style) { @@ -242,17 +243,17 @@ private: template std::string IAST::formatForErrorMessage(const AstArray & array) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString buf; for (size_t i = 0; i < array.size(); ++i) { if (i > 0) { - ss << ", "; + const char * delim = ", "; + buf.write(delim, strlen(delim)); } - array[i]->format(IAST::FormatSettings(ss, true /* one line */)); + array[i]->format(IAST::FormatSettings(buf, true /* one line */)); } - return ss.str(); + return buf.str(); } } diff --git a/src/Parsers/formatAST.cpp b/src/Parsers/formatAST.cpp index e19dc715d51..3a258df099e 100644 --- a/src/Parsers/formatAST.cpp +++ b/src/Parsers/formatAST.cpp @@ -5,9 +5,9 @@ namespace DB { -void formatAST(const IAST & ast, std::ostream & s, bool hilite, bool one_line) +void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite, bool one_line) { - IAST::FormatSettings settings(s, one_line); + IAST::FormatSettings settings(buf, one_line); settings.hilite = hilite; ast.format(settings); @@ -15,10 +15,9 @@ void formatAST(const IAST & ast, std::ostream & s, bool hilite, bool one_line) String serializeAST(const IAST & ast, bool one_line) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - formatAST(ast, ss, false, one_line); - return ss.str(); + WriteBufferFromOwnString buf; + formatAST(ast, buf, false, one_line); + return buf.str(); } } diff --git a/src/Parsers/formatAST.h b/src/Parsers/formatAST.h index 685c504514e..bee89521812 100644 --- a/src/Parsers/formatAST.h +++ b/src/Parsers/formatAST.h @@ -7,23 +7,25 @@ namespace DB { +class WriteBuffer; + /** Takes a syntax tree and turns it back into text. * In case of INSERT query, the data will be missing. */ -void formatAST(const IAST & ast, std::ostream & s, bool hilite = true, bool one_line = false); +void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite = true, bool one_line = false); String serializeAST(const IAST & ast, bool one_line = true); -inline std::ostream & operator<<(std::ostream & os, const IAST & ast) -{ - formatAST(ast, os, false, true); - return os; -} - -inline std::ostream & operator<<(std::ostream & os, const ASTPtr & ast) -{ - formatAST(*ast, os, false, true); - return os; -} +//inline std::ostream & operator<<(std::ostream & os, const IAST & ast) +//{ +// formatAST(ast, os, false, true); +// return os; +//} +// +//inline std::ostream & operator<<(std::ostream & os, const ASTPtr & ast) +//{ +// formatAST(*ast, os, false, true); +// return os; +//} } diff --git a/src/Parsers/formatSettingName.cpp b/src/Parsers/formatSettingName.cpp index c305496fdb3..351a1eb8ade 100644 --- a/src/Parsers/formatSettingName.cpp +++ b/src/Parsers/formatSettingName.cpp @@ -2,13 +2,13 @@ #include #include #include -#include +#include namespace DB { -void formatSettingName(const String & setting_name, std::ostream & out) +void formatSettingName(const String & setting_name, WriteBuffer & out) { if (isValidIdentifier(setting_name)) { diff --git a/src/Parsers/formatSettingName.h b/src/Parsers/formatSettingName.h index a700d347a5f..ba819ee2b4c 100644 --- a/src/Parsers/formatSettingName.h +++ b/src/Parsers/formatSettingName.h @@ -7,9 +7,11 @@ namespace DB { +class WriteBuffer; + /// Outputs built-in or custom setting's name. /// The function is like backQuoteIfNeed() but didn't quote with backticks /// if the name consists of identifiers joined with dots. -void formatSettingName(const String & setting_name, std::ostream & out); +void formatSettingName(const String & setting_name, WriteBuffer & out); } diff --git a/src/Parsers/queryToString.cpp b/src/Parsers/queryToString.cpp index 44ea721485f..9721aa1f128 100644 --- a/src/Parsers/queryToString.cpp +++ b/src/Parsers/queryToString.cpp @@ -1,6 +1,5 @@ #include #include -#include namespace DB { @@ -11,9 +10,6 @@ namespace DB String queryToString(const IAST & query) { - std::ostringstream out; - out.exceptions(std::ios::failbit); - formatAST(query, out, false, true); - return out.str(); + return serializeAST(query); } } diff --git a/src/Parsers/tests/create_parser.cpp b/src/Parsers/tests/create_parser.cpp index fbdc967fa2a..c241b353b4f 100644 --- a/src/Parsers/tests/create_parser.cpp +++ b/src/Parsers/tests/create_parser.cpp @@ -4,6 +4,7 @@ #include #include #include +#include int main(int, char **) @@ -14,7 +15,8 @@ int main(int, char **) ParserCreateQuery parser; ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); - formatAST(*ast, std::cerr); + WriteBufferFromOStream out(std::cerr, 4096); + formatAST(*ast, out); std::cerr << std::endl; return 0; diff --git a/src/Parsers/tests/gtest_dictionary_parser.cpp b/src/Parsers/tests/gtest_dictionary_parser.cpp index c418759aa21..b051cedfb23 100644 --- a/src/Parsers/tests/gtest_dictionary_parser.cpp +++ b/src/Parsers/tests/gtest_dictionary_parser.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -17,10 +18,9 @@ using namespace DB; static String astToString(IAST * ast) { - std::ostringstream oss; - oss.exceptions(std::ios::failbit); - dumpAST(*ast, oss); - return oss.str(); + WriteBufferFromOwnString buf; + dumpAST(*ast, buf); + return buf.str(); } /// Tests for external dictionaries DDL parser diff --git a/src/Parsers/tests/select_parser.cpp b/src/Parsers/tests/select_parser.cpp index 7711f0d2527..7c18563659d 100644 --- a/src/Parsers/tests/select_parser.cpp +++ b/src/Parsers/tests/select_parser.cpp @@ -3,6 +3,7 @@ #include #include #include +#include int main(int, char **) @@ -25,7 +26,8 @@ try ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); std::cout << "Success." << std::endl; - formatAST(*ast, std::cerr); + WriteBufferFromOStream out(std::cerr, 4096); + formatAST(*ast, out); std::cout << std::endl; return 0; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 597ff5e8fee..aea740a3a26 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1693,13 +1693,12 @@ std::vector ReplicatedMergeTreeQueue::getMutationsStatu for (const MutationCommand & command : entry.commands) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - formatAST(*command.ast, ss, false, true); + WriteBufferFromOwnString buf; + formatAST(*command.ast, buf, false, true); result.push_back(MergeTreeMutationStatus { entry.znode_name, - ss.str(), + buf.str(), entry.create_time, entry.block_numbers, parts_to_mutate, diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 48f05b50675..d06706f9109 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -18,10 +18,9 @@ static String formattedAST(const ASTPtr & ast) { if (!ast) return ""; - std::stringstream ss; - ss.exceptions(std::ios::failbit); - formatAST(*ast, ss, false, true); - return ss.str(); + WriteBufferFromOwnString buf; + formatAST(*ast, buf, false, true); + return buf.str(); } ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTreeData & data, const StorageMetadataPtr & metadata_snapshot) diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index 53c9b50cb9d..c57ea0f1d77 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -126,10 +126,9 @@ std::shared_ptr MutationCommands::ast() const void MutationCommands::writeText(WriteBuffer & out) const { - std::stringstream commands_ss; - commands_ss.exceptions(std::ios::failbit); - formatAST(*ast(), commands_ss, /* hilite = */ false, /* one_line = */ true); - out << escape << commands_ss.str(); + WriteBufferFromOwnString commands_buf; + formatAST(*ast(), commands_buf, /* hilite = */ false, /* one_line = */ true); + out << escape << commands_buf.str(); } void MutationCommands::readText(ReadBuffer & in) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 4ec6d748738..fbeb188d649 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -538,13 +538,12 @@ std::vector StorageMergeTree::getMutationsStatus() cons for (const MutationCommand & command : entry.commands) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - formatAST(*command.ast, ss, false, true); + WriteBufferFromOwnString buf; + formatAST(*command.ast, buf, false, true); result.push_back(MergeTreeMutationStatus { entry.file_name, - ss.str(), + buf.str(), entry.create_time, block_numbers_map, parts_to_do_names, diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index 3148ab1112a..81d3303e262 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -220,8 +220,7 @@ String transformQueryForExternalDatabase( ASTPtr select_ptr = select; dropAliases(select_ptr); - std::stringstream out; - out.exceptions(std::ios::failbit); + WriteBufferFromOwnString out; IAST::FormatSettings settings(out, true); settings.identifier_quoting_style = identifier_quoting_style; settings.always_quote_identifiers = identifier_quoting_style != IdentifierQuotingStyle::None; diff --git a/utils/db-generator/query_db_generator.cpp b/utils/db-generator/query_db_generator.cpp index c8aae4a56f3..e3c5b862dbc 100644 --- a/utils/db-generator/query_db_generator.cpp +++ b/utils/db-generator/query_db_generator.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -828,9 +829,9 @@ FuncRet arithmeticFunc(DB::ASTPtr ch, std::map & columns) FuncRet r(ret_type, ""); if (no_indent) { - std::ostringstream ss; - formatAST(*ch, ss); - r.value = ss.str(); + DB::WriteBufferFromOwnString buf; + formatAST(*ch, buf); + r.value = buf.str(); } return r; } @@ -990,10 +991,10 @@ FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) { if (no_indent) { - std::ostringstream ss; - formatAST(*ch, ss); + DB::WriteBufferFromOwnString buf; + formatAST(*ch, buf); auto r = func_to_return_type[boost::algorithm::to_lower_copy(x->name)]; - r.value = ss.str(); + r.value = buf.str(); return r; } return func_to_return_type[boost::algorithm::to_lower_copy(x->name)]; @@ -1003,11 +1004,11 @@ FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) { if (no_indent) { - std::ostringstream ss; - formatAST(*ch, ss); + DB::WriteBufferFromOwnString buf; + formatAST(*ch, buf); return FuncRet( func_to_param_type[boost::algorithm::to_lower_copy(x->name)], - ss.str()); + buf.str()); } return FuncRet( func_to_param_type[boost::algorithm::to_lower_copy(x->name)], From 3c6794bf37bc126751f67aeca11c97df75c7b707 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Mon, 9 Nov 2020 20:22:05 +0300 Subject: [PATCH 067/114] Add Floyd-Rivest selection algorithm instead of std::partial_sort --- CMakeLists.txt | 1 + cmake/find/miniselect.cmake | 2 + contrib/miniselect/.clang-format | 1 + contrib/miniselect/.gitignore | 100 ++ contrib/miniselect/.travis.yml | 140 +++ contrib/miniselect/AUTHORS | 2 + contrib/miniselect/CMakeLists.txt | 52 + contrib/miniselect/CONTRIBUTORS | 1 + contrib/miniselect/LICENSE_1_0.txt | 23 + contrib/miniselect/README.md | 272 +++++ contrib/miniselect/benches/bench_common.h | 170 ++++ .../miniselect/benches/benchmark_select.cpp | 46 + contrib/miniselect/benches/benchmark_sort.cpp | 46 + contrib/miniselect/examples/example.cpp | 18 + contrib/miniselect/fuzz/CMakeLists.txt | 38 + .../miniselect/fuzz/build_like_oss_fuzz.sh | 22 + contrib/miniselect/fuzz/fuzz_select.cpp | 66 ++ contrib/miniselect/fuzz/fuzz_sort.cpp | 69 ++ .../miniselect/fuzz/fuzz_string_select.cpp | 70 ++ contrib/miniselect/fuzz/fuzz_string_sort.cpp | 73 ++ contrib/miniselect/fuzz/main.cpp | 22 + contrib/miniselect/fuzz/ossfuzz.sh | 23 + .../include/miniselect/floyd_rivest_select.h | 120 +++ .../include/miniselect/median_of_3_random.h | 69 ++ .../include/miniselect/median_of_medians.h | 71 ++ .../include/miniselect/median_of_ninthers.h | 190 ++++ .../miniselect/include/miniselect/pdqselect.h | 935 ++++++++++++++++++ .../miniselect/private/median_common.h | 437 ++++++++ contrib/miniselect/testing/test_common.h | 180 ++++ contrib/miniselect/testing/test_select.cpp | 231 +++++ contrib/miniselect/testing/test_sort.cpp | 161 +++ src/CMakeLists.txt | 1 + src/Columns/ColumnArray.cpp | 5 +- src/Columns/ColumnDecimal.cpp | 9 +- src/Columns/ColumnDecimal.h | 5 +- src/Columns/ColumnFixedString.cpp | 9 +- src/Columns/ColumnLowCardinality.cpp | 3 +- src/Columns/ColumnString.cpp | 5 +- src/Columns/ColumnTuple.cpp | 4 +- src/Columns/ColumnVector.cpp | 10 +- 40 files changed, 3680 insertions(+), 22 deletions(-) create mode 100644 cmake/find/miniselect.cmake create mode 100644 contrib/miniselect/.clang-format create mode 100644 contrib/miniselect/.gitignore create mode 100644 contrib/miniselect/.travis.yml create mode 100644 contrib/miniselect/AUTHORS create mode 100644 contrib/miniselect/CMakeLists.txt create mode 100644 contrib/miniselect/CONTRIBUTORS create mode 100644 contrib/miniselect/LICENSE_1_0.txt create mode 100644 contrib/miniselect/README.md create mode 100644 contrib/miniselect/benches/bench_common.h create mode 100644 contrib/miniselect/benches/benchmark_select.cpp create mode 100644 contrib/miniselect/benches/benchmark_sort.cpp create mode 100644 contrib/miniselect/examples/example.cpp create mode 100644 contrib/miniselect/fuzz/CMakeLists.txt create mode 100755 contrib/miniselect/fuzz/build_like_oss_fuzz.sh create mode 100644 contrib/miniselect/fuzz/fuzz_select.cpp create mode 100644 contrib/miniselect/fuzz/fuzz_sort.cpp create mode 100644 contrib/miniselect/fuzz/fuzz_string_select.cpp create mode 100644 contrib/miniselect/fuzz/fuzz_string_sort.cpp create mode 100644 contrib/miniselect/fuzz/main.cpp create mode 100755 contrib/miniselect/fuzz/ossfuzz.sh create mode 100644 contrib/miniselect/include/miniselect/floyd_rivest_select.h create mode 100644 contrib/miniselect/include/miniselect/median_of_3_random.h create mode 100644 contrib/miniselect/include/miniselect/median_of_medians.h create mode 100644 contrib/miniselect/include/miniselect/median_of_ninthers.h create mode 100644 contrib/miniselect/include/miniselect/pdqselect.h create mode 100644 contrib/miniselect/include/miniselect/private/median_common.h create mode 100644 contrib/miniselect/testing/test_common.h create mode 100644 contrib/miniselect/testing/test_select.cpp create mode 100644 contrib/miniselect/testing/test_sort.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 783a9f80b66..182d9989dc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -445,6 +445,7 @@ include (cmake/find/brotli.cmake) include (cmake/find/protobuf.cmake) include (cmake/find/grpc.cmake) include (cmake/find/pdqsort.cmake) +include (cmake/find/miniselect.cmake) include (cmake/find/hdfs3.cmake) # uses protobuf include (cmake/find/poco.cmake) include (cmake/find/curl.cmake) diff --git a/cmake/find/miniselect.cmake b/cmake/find/miniselect.cmake new file mode 100644 index 00000000000..0a50c9bf4a8 --- /dev/null +++ b/cmake/find/miniselect.cmake @@ -0,0 +1,2 @@ +set(MINISELECT_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/miniselect/include) +message(STATUS "Using miniselect: ${MINISELECT_INCLUDE_DIR}") diff --git a/contrib/miniselect/.clang-format b/contrib/miniselect/.clang-format new file mode 100644 index 00000000000..f6cb8ad931f --- /dev/null +++ b/contrib/miniselect/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/contrib/miniselect/.gitignore b/contrib/miniselect/.gitignore new file mode 100644 index 00000000000..f80f36759c8 --- /dev/null +++ b/contrib/miniselect/.gitignore @@ -0,0 +1,100 @@ +# eclipse project files +.cproject +.project +.settings + +# emacs temp files +*~ + +# vim temp files +.*.swp + +# XCode +^build/ +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 +xcuserdata +*.xccheckout +*.moved-aside +DerivedData +*.hmap +*.ipa +*.xcuserstate +*.DS_Store + +# IDE specific folder for JetBrains IDEs +.idea/ +cmake-build-debug/ +cmake-build-release/ + +# Visual Studio Code artifacts +.vscode/* +.history/ + +# Visual Studio artifacts +/VS/ + +# C/C++ build outputs +.build/ +bins +gens +libs +objs + +# C++ ignore from https://github.com/github/gitignore/blob/master/C%2B%2B.gitignore + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + + +# CMake files that may be specific to our installation + +# Build outputs +/build*/ +/visual_studio/ +/benchmark/ + +# Fuzzer outputs generated by instructions in fuzz/Fuzzing.md +/corpus.zip +/ossfuzz-out/ +/out/ + +# Generated docs +/doc/api +*.orig diff --git a/contrib/miniselect/.travis.yml b/contrib/miniselect/.travis.yml new file mode 100644 index 00000000000..a5036caf365 --- /dev/null +++ b/contrib/miniselect/.travis.yml @@ -0,0 +1,140 @@ +language: cpp + +dist: bionic + +matrix: + include: + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-8 + env: + - COMPILER="CC=gcc-8 && CXX=g++-8" + compiler: gcc-8 + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-9 + env: + - COMPILER="CC=gcc-9 && CXX=g++-9" + compiler: gcc-9 + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-10 + env: + - COMPILER="CC=gcc-10 && CXX=g++-10" + compiler: gcc-10 + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-10 + env: + - COMPILER="CC=gcc-10 && CXX=g++-10" + - SANITIZE="on" + compiler: gcc-10-sanitize + + - os: linux + addons: + apt: + sources: + - llvm-toolchain-bionic-6.0 + packages: + - clang-6.0 + env: + - COMPILER="CC=clang-6.0 && CXX=clang++-6.0" + compiler: clang-6 + + - os: linux + addons: + apt: + sources: + - llvm-toolchain-bionic-7 + packages: + - clang-7 + env: + - COMPILER="CC=clang-7 && CXX=clang++-7" + compiler: clang-7 + + - os: linux + addons: + apt: + sources: + - llvm-toolchain-bionic-8 + packages: + - clang-8 + env: + - COMPILER="CC=clang-8 && CXX=clang++-8" + compiler: clang-8 + + - os: linux + addons: + apt: + sources: + - llvm-toolchain-bionic-9 + packages: + - clang-9 + env: + - COMPILER="CC=clang-9 && CXX=clang++-9" + compiler: clang-9 + + - os: linux + addons: + apt: + packages: + - clang-10 + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + env: + - COMPILER="CC=clang-10 && CXX=clang++-10" + compiler: clang-10 + + - os: linux + addons: + apt: + packages: + - clang-10 + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + env: + - COMPILER="CC=clang-10 && CXX=clang++-10" + - SANITIZE="on" + compiler: clang-10-sanitize + +before_install: + - eval "${COMPILER}" + - git clone https://github.com/google/benchmark.git + - git clone https://github.com/google/googletest.git benchmark/googletest + +install: + - export CMAKE_FLAGS="-DMINISELECT_TESTING=on -DCMAKE_BUILD_TYPE=RelWithDebInfo"; + - if [[ "${SANITIZE}" == "on" ]]; then + export CMAKE_FLAGS="${CMAKE_FLAGS} -DMINISELECT_SANITIZE=on"; + fi + - export CTEST_FLAGS="-j4 --output-on-failure -E checkperf" + +script: + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . -- -j2 + - ctest $CTEST_FLAGS diff --git a/contrib/miniselect/AUTHORS b/contrib/miniselect/AUTHORS new file mode 100644 index 00000000000..896a8046a73 --- /dev/null +++ b/contrib/miniselect/AUTHORS @@ -0,0 +1,2 @@ +# List of authors for copyright purposes, in no particular order +Danila Kutenin diff --git a/contrib/miniselect/CMakeLists.txt b/contrib/miniselect/CMakeLists.txt new file mode 100644 index 00000000000..09e92031784 --- /dev/null +++ b/contrib/miniselect/CMakeLists.txt @@ -0,0 +1,52 @@ +cmake_minimum_required(VERSION 3.7) +project(miniselect) + +option(MINISELECT_TESTING "Building the tests." OFF) +option(MINISELECT_SANITIZE "Building the library with sanitizers." OFF) +option(MINISELECT_BUILD_LIBCXX "Building the library with libcxx." OFF) +option(MINISELECT_ENABLE_FUZZING "Building the library with fuzzing." OFF) + +include_directories(include) + +if (MINISELECT_TESTING) + enable_testing() + set(CMAKE_CXX_STANDARD 17) + if (NOT CMAKE_BUILD_TYPE) + message(STATUS "No build type selected, default to Release") + set(CMAKE_BUILD_TYPE "Release") + endif() + if (MINISELECT_SANITIZE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all") + endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Wextra -Wpedantic -Wno-gnu-zero-variadic-macro-arguments") + + if (MINISELECT_BUILD_LIBCXX AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + message(STATUS "Using libcxx as a default standard C++ library") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + endif() + + add_subdirectory(benchmark) + include_directories(testing) + include_directories(benches) + + add_executable(benchmark_sort benches/benchmark_sort.cpp) + target_link_libraries(benchmark_sort benchmark::benchmark gtest) + add_executable(benchmark_select benches/benchmark_select.cpp) + target_link_libraries(benchmark_select benchmark::benchmark gtest) + + set(TEST_SOURCES testing/test_select.cpp) + add_executable(test_select ${TEST_SOURCES}) + target_link_libraries(test_select gtest gmock gtest_main) + add_test(NAME test_select COMMAND test_select) + + set(TEST_SOURCES testing/test_sort.cpp) + add_executable(test_sort ${TEST_SOURCES}) + target_link_libraries(test_sort gtest gmock gtest_main) + add_test(NAME test_sort COMMAND test_sort) +endif() + +if(MINISELECT_ENABLE_FUZZING) + add_subdirectory(benchmark) + include_directories(testing) + add_subdirectory(fuzz) +endif() diff --git a/contrib/miniselect/CONTRIBUTORS b/contrib/miniselect/CONTRIBUTORS new file mode 100644 index 00000000000..75d47387e67 --- /dev/null +++ b/contrib/miniselect/CONTRIBUTORS @@ -0,0 +1 @@ +# contributors (in no particular order) diff --git a/contrib/miniselect/LICENSE_1_0.txt b/contrib/miniselect/LICENSE_1_0.txt new file mode 100644 index 00000000000..36b7cd93cdf --- /dev/null +++ b/contrib/miniselect/LICENSE_1_0.txt @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/miniselect/README.md b/contrib/miniselect/README.md new file mode 100644 index 00000000000..cbe576ddba8 --- /dev/null +++ b/contrib/miniselect/README.md @@ -0,0 +1,272 @@ +[![Build Status](https://travis-ci.com/danlark1/miniselect.svg?branch=main)](https://travis-ci.com/danlark1/miniselect) +[![License](https://img.shields.io/badge/License-Boost%201.0-lightblue.svg)](https://www.boost.org/LICENSE_1_0.txt) + +miniselect : Generic selection and partial ordering algorithms +============================================================== + +`miniselect` is a C++ header-only library that contains various generic selection +and partial sorting algorithms with the ease of use, testing, advice on usage and +benchmarking. + +Sorting is everywhere and there are many outstanding sorting algorithms that +compete in speed, comparison count and cache friendliness. However selection +algorithms are always a bit outside of the competition scope, however they are +pretty important, for example, in databases ORDER BY LIMIT N is used extremely +often which can benefit from more optimal selection and partial sorting +algorithms. This library tries to solve this problem with Modern C++. + +* **Easy:** First-class, easy to use dependency and carefully documented APIs and algorithm properties. +* **Fast:** We do care about speed of the algorithms and provide reasonable implementations. +* **Standard compliant:** We provide C++11 compatible APIs that are compliant to the standard [`std::nth_element`](https://en.cppreference.com/w/cpp/algorithm/nth_element) and [`std::partial_sort`](https://en.cppreference.com/w/cpp/algorithm/partial_sort) functions including custom comparators and order guarantees. Just replace the names of the functions in your project and it should work! +* **Well tested:** We test all algorithms with a unified framework, under sanitizers and fuzzing. +* **Benchmarked:** We gather benchmarks for all implementations to better understand good and bad spots. + +Table of Contents +----------------- + +* [Quick Start](#quick-start) +* [Testing](#testing) +* [Documentation](#documentation) +* [Performance results](#performance-results) +* [Real-world usage](#real-world-usage) +* [Contributing](#contributing) +* [Motivation](#motivation) +* [License](#license) + +Quick Start +----------- + +You can either include this project as a cmake dependency and then use the +headers that are provided in the [include](./include) folder or just pass the +[include](./include) folder to your compiler. + +```cpp +#include +#include + +#include "miniselect/median_of_ninthers.h" + +int main() { + std::vector v = {1, 8, 4, 3, 2, 9, 0, 7, 6, 5}; + miniselect::median_of_ninthers_select(v.begin(), v.begin() + 5, v.end()); + for (const int i : v) { + std::cout << i << ' '; + } + return 0; +} +// Compile it `clang++/g++ -I$DIRECTORY/miniselect/include/ example.cpp -std=c++11 -O3 -o example +// Possible output: 0 1 4 3 2 5 8 7 6 9 +``` + +Examples can be found in [examples](./examples). + +We support all compilers starting from GCC 7 and Clang 6. We are also planning +to support Windows, for now it is best effort but no issues are known so far. + +More on which algorithms are available, see [documentation](#documentation). + +Testing +------- + +To test and benchmark, we use [Google benchmark](https://github.com/google/benchmark) library. +Simply do in the root directory: + +```console +# Check out the library. +$ git clone https://github.com/google/benchmark.git +# Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory. +$ git clone https://github.com/google/googletest.git benchmark/googletest +$ mkdir build && cd build +$ cmake -DMINISELECT_TESTING=on .. +$ make -j +$ ctest -j4 --output-on-failure +``` + +It will create two tests and two benchmarks `test_sort`, `test_select`, +`benchmark_sort`, `benchmark_select`. Use them to validate or contribute. You +can also use `ctest` + +Documentation +------------- + +There are several selection algorithms available, further ![\large n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+n) is the number +of elements in the array, ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) is the selection element that is needed to be found (all algorithms are deterministic and not stable unless otherwise is specified): + + +| Name | Average | Best Case | Worst Case | Comparisons | Memory | +|------------------------- |--------------------------------------------------------------------------------------------------------- |--------------------------------------------------------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |--------------------------------------------------------------------------------------------------------------------------------- | +| [pdqselect](./include/miniselect/pdqselect.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5Clog+n%29) | At least ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n). Random data ![\large 2.5n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2.5n) | ![\large O(1)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%281%29) | +| [Floyd-Rivest](./include/miniselect/floyd_rivest_select.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n^2 )](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5E2+%29) | Avg: ![\large n + \min(k, n - k) + O(\sqrt{n \log n})](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+n+%2B+%5Cmin%28k%2C+n+-+k%29+%2B+O%28%5Csqrt%7Bn+%5Clog+n%7D%29) | ![\large O(\log \log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+%5Clog+n%29) | +| [Median Of Medians](./include/miniselect/median_of_medians.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | Between ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n) and ![\large 22n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+22n). Random data ![\large 2.5n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2.5n) | ![\large O(\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+n%29) | +| [Median Of Ninthers](./include/miniselect/median_of_ninthers.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | Between ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n) and ![\large 12n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+12.5n). Random data ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n) | ![\large O(\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+n%29) | +| [Median Of 3 Random](./include/miniselect/median_of_3_random.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n^2 )](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5E2+%29) | At least ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n). Random data ![\large 3n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+3n) | ![\large O(\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+n%29) | +| [libstdc++ (introselect)](https://github.com/gcc-mirror/gcc/blob/e0af865ab9d9d5b6b3ac7fdde26cf9bbf635b6b4/libstdc%2B%2B-v3/include/bits/stl_algo.h#L4748) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5Clog+n%29) | At least ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n). Random data ![\large 3n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+3n) | ![\large O(1)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%281%29) | +| [libc++ (median of 3)](https://github.com/llvm/llvm-project/blob/3ed89b51da38f081fedb57727076262abb81d149/libcxx/include/algorithm#L5159) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n^2 )](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5E2+%29) | At least ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n). Random data ![\large 3n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+3n) | ![\large O(1)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%281%29) | + +For sorting the situation is similar except every line adds ![\large O(k\log k)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28k%5Clog+k%29) comparisons and pdqselect is using ![\large O(\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+n%29) memory with one more general exception called partial sorting in C++ standard library. + +| Name | Average | Best Case | Worst Case | Comparisons | Memory | +|-------------------|-----------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------- | +| [std::partial_sort](https://github.com/llvm/llvm-project/blob/3ed89b51da38f081fedb57727076262abb81d149/libcxx/include/algorithm#L5074) | ![\large O(n\log k)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5Clog+k%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n\log k)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5Clog+k%29) | ![\large n\log k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+n%5Clog+k) on average, for some data patterns might be better | ![\large O(1)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%281%29) | + +## API + +All functions end either in `select`, either in `partial_sort` and +their behavior is exactly the same as for +[`std::nth_element`](https://en.cppreference.com/w/cpp/algorithm/nth_element) +and [`std::partial_sort`](https://en.cppreference.com/w/cpp/algorithm/partial_sort) +respectively, i.e. they accept 3 arguments as `first`, `middle`, `end` iterators +and an optional comparator. Several notes: + +* You should not throw exceptions from `Compare` function. Standard library +also does not specify the behavior in that matter. +* We don't support ParallelSTL for now. +* C++20 constexpr specifiers might be added but currently we don't have them +because of some floating point math in several algorithms. +* All functions are in the `miniselect` namespace. See the example for that. + +- pdqselect + - This algorithm is based on [`pdqsort`](https://github.com/orlp/pdqsort) which is acknowledged as one of the fastest generic sort algorithms. + - **Location:** [`miniselect/pdqselect.h`](./include/miniselect/pdqselect.h). + - **Functions:** `pdqselect`, `pdqselect_branchless`, `pdqpartial_sort`, `pdqpartial_sort_branchless`. Branchless version uses branchless partition algorithm provided by [`pdqsort`](https://github.com/orlp/pdqsort). Use it if your comparison function is branchless, it might give performance for very big ranges. + - **Performance advice:** Use it when you need to sort a big chunk so that ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) is close to ![\large n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+n). + +

+ +- Floyd-Rivest + - This algorithm is based on [Floyd-Rivest algorithm](https://en.wikipedia.org/wiki/Floyd%E2%80%93Rivest_algorithm). + - **Location:** [`miniselect/floyd_rivest_select.h`](./include/miniselect/floyd_rivest_select.h). + - **Functions:** `floyd_rivest_select`, `floyd_rivest_partial_sort`. + - **Performance advice:** Given that this algorithm performs as one of the best on average case in terms of comparisons and speed, we highly advise to + at least try this in your project. Especially it is good for small ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) or types that are expensive to compare (for example, strings). But even for median the benchmarks show it outperforms others. It is not easy for this algorithm to build a reasonable worst case but one of examples when this algorithm does not perform well is when there are lots of similar values of linear size (random01 dataset showed some moderate penalties). + +We present here two gifs, for median and for ![\large k = n / 10](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k+%3D+n+%2F+10) order statistic. + +

+ + +

+ +- Median Of Medians + - This algorithm is based on [Median of Medians](https://en.wikipedia.org/wiki/Median_of_medians) algorithm, one of the first deterministic linear time worst case median algorithm + - **Location:** [`miniselect/median_of_medians.h`](./include/miniselect/median_of_medians.h). + - **Functions:** `median_of_medians_select`, `median_of_medians_partial_sort`. + - **Performance advice:** This algorithm does not show advantages over others, implemented for historical reasons and for bechmarking. + +

+ +- Median Of Ninthers + - This algorithm is based on [Fast Deterministic Selection](https://erdani.com/research/sea2017.pdf) paper by Andrei Alexandrescu, one of the latest and fastest deterministic linear time worst case median algorithms + - **Location:** [`miniselect/median_of_ninthers.h`](./include/miniselect/median_of_ninthers.h). + - **Functions:** `median_of_ninthers_select`, `median_of_ninthers_partial_sort`. + - **Performance advice:** Use this algorithm if you absolutely need linear time worst case scenario for selection algorithm. This algorithm shows some strengths over other deterministic [`PICK`](https://en.wikipedia.org/wiki/Median_of_medians) algorithms and has lower constanst than MedianOfMedians. + +

+ +- Median Of 3 Random + - This algorithm is based on QuickSelect with the random median of 3 pivot choice algorithm (it chooses random 3 elements in the range and takes the middle value). It is a rando + - **Location:** [`miniselect/median_of_3_random.h`](./include/miniselect/median_of_3_random.h). + - **Functions:** `median_of_3_random_select`, `median_of_3_random_partial_sort`. + - **Performance advice:** This is a randomized algorithm and also it did not show any strengths against Median Of Ninthers. + +

+ +- Introselect + - This algorithm is based on [Introselect](https://en.wikipedia.org/wiki/Introselect) algorithm, it is used in libstdc++ in `std::nth_element`, however instead of falling back to MedianOfMedians it is using HeapSelect which adds logarithm to its worst complexity. + - **Location:** ``. + - **Functions:** `std::nth_element`. + - **Performance advice:** This algorithm is used in standard library and is not recommended to use if you are looking for performance. + +

+ +- Median Of 3 + - This algorithm is based on QuickSelect with median of 3 pivot choice algorithm (the middle value between begin, mid and end values), it is used in libc++ in `std::nth_element`. + - **Location:** ``. + - **Functions:** `std::nth_element`. + - **Performance advice:** This algorithm is used in standard library and is not recommended to use if you are looking for performance. + +

+ +- `std::partial_sort` + - This algorithm has [heap-based solutions](https://en.wikipedia.org/wiki/Partial_sorting) both in libc++ and libstdc++, from the first ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) elements the max heap is built, then one by one the elements are trying to be pushed to that heap with HeapSort in the end. + - **Location:** ``. + - **Functions:** `std::partial_sort`. + - **Performance advice:** This algorithm is very good for random data and small ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) and might outperform all selection+sort algorithms. However, for descending data it starts to significantly degrade and is not recommended for use if you have such patterns in real data. + +

+ +## Other algorithms to come + +* Kiwiel modification of FloydRivest algorithm which is described in [On Floyd and Rivest’s SELECT algorithm](https://core.ac.uk/download/pdf/82672439.pdf) with ternary and quintary pivots. +* Combination of FloydRivest and pdqsort pivot strategies, currently all experiments did not show any boost. + +Performance results +------------------- + +We use 10 datasets and 8 algorithms with 10000000 elements to find median and +other ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) on `Intel(R) Core(TM) i5-4200H CPU @ 2.80GHz` for `std::vector`, +for median the benchmarks are the following: + +![median](benches/plots/result_10000000_5000000.png) + +![median](benches/plots/result_comparisons_10000000_5000000.png) + +For smaller ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k), +for example, 1000, the results are the following + +![k equals 1000](benches/plots/result_10000000_1000.png) + +![k equals 1000](benches/plots/result_comparisons_10000000_1000.png) + +Other benchmarks can be found [here](https://drive.google.com/drive/folders/1DHEaeXgZuX6AJ9eByeZ8iQVQv0ueP8XM). + +The benchmarks for number of swaps will be later. + +Real-world usage +---------------- + +- [Yandex ClickHouse](https://github.com/yandex/ClickHouse) + +If you are planning to use miniselect in your product, please work from one of +our releases and if you wish, you can write the acknowledgment in this section +for visibility. + +Contributing +------------ + +Patches are welcome with new algorithms! You should add the selection algorithm +together with the partial sorting algorithm in [include](./include), add +tests in [testing](./testing) and ideally run benchmarks to see how it performs. +If you also have some data cases to test against, we would be more than happy +to merge them. + +Motivation +---------- + +Firstly the author was interested if any research had been done for small ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) +in selection algorithms and was struggling to find working implementations to +compare different approaches from standard library and quickselect algorithms. +After that it turned out that the problem is much more interesting than it looks +like and after reading The Art of Computer Programming from Donald Knuth about +minimum comparison sorting and selection algorithms the author decided to look +through all non-popular algorithms and try them out. + +The author have not found any decent library for selection algorithms and little +research is published in open source, so that they decided to merge all that +implementations and compare them with possible merging of different ideas +into a decent one algorithm for most needs. For a big story of adventures see +the author's blog post TODO. + +License +------- + +The code is made available under the [Boost License 1.0](https://boost.org/LICENSE_1_0.txt). + +Third-Party Libraries Used and Adjusted +--------------------------------------- + +| Library | License | +|---------------------|--------------------------------------------------------------------------------------------------| +| pdqsort | [MIT](https://github.com/orlp/pdqsort/blob/47a46767d76fc852284eaa083e4b7034ee6e2559/license.txt) | +| MedianOfNinthers | [Boost License 1.0](https://github.com/andralex/MedianOfNinthers/blob/master/LICENSE_1_0.txt) | + diff --git a/contrib/miniselect/benches/bench_common.h b/contrib/miniselect/benches/bench_common.h new file mode 100644 index 00000000000..b49b55dac9d --- /dev/null +++ b/contrib/miniselect/benches/bench_common.h @@ -0,0 +1,170 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include +#include +#include + +namespace miniselect { +namespace datagens { + +struct Random { + static std::vector Gen(size_t size) { + std::random_device rnd_device; + std::mt19937_64 mersenne_engine{rnd_device()}; + std::vector v; + v.reserve(size); + for (size_t i = 0; i < size; ++i) { + v.push_back(i); + } + std::shuffle(v.begin(), v.end(), mersenne_engine); + return v; + } +}; + +struct Shuffled16 { + static std::vector Gen(size_t size) { + std::random_device rnd_device; + std::mt19937_64 mersenne_engine{rnd_device()}; + std::vector v; + v.reserve(size); + for (size_t i = 0; i < size; ++i) { + v.push_back(i % 16); + } + std::shuffle(v.begin(), v.end(), mersenne_engine); + return v; + } +}; + +struct Random01 { + static std::vector Gen(size_t size) { + std::random_device rnd_device; + std::mt19937_64 mersenne_engine{rnd_device()}; + std::vector v; + v.reserve(size); + for (size_t i = 0; i < size; ++i) { + v.push_back(i % 2); + } + std::shuffle(v.begin(), v.end(), mersenne_engine); + return v; + } +}; + +struct Ascending { + static std::vector Gen(size_t size) { + std::vector v; + v.reserve(size); + for (size_t i = 0; i < size; ++i) { + v.push_back(i); + } + return v; + } +}; + +struct Descending { + static std::vector Gen(size_t size) { + std::vector v; + v.reserve(size); + for (int i = size - 1; i >= 0; --i) { + v.push_back(i); + } + return v; + } +}; + +struct PipeOrgan { + static std::vector Gen(size_t size) { + std::vector v; + v.reserve(size); + for (size_t i = 0; i < size / 2; ++i) { + v.push_back(i); + } + for (size_t i = size / 2; i < size; ++i) { + v.push_back(size - i); + } + return v; + } +}; + +struct PushFront { + static std::vector Gen(size_t size) { + std::vector v; + v.reserve(size); + for (size_t i = 1; i < size; ++i) { + v.push_back(i); + } + v.push_back(0); + return v; + } +}; + +struct PushMiddle { + static std::vector Gen(size_t size) { + std::vector v; + v.reserve(size); + for (size_t i = 0; i < size; ++i) { + if (i != size / 2) { + v.push_back(i); + } + } + v.push_back(size / 2); + return v; + } +}; + +struct Median3Killer { + static std::vector Gen(size_t size) { + size_t k = size / 2; + std::vector v; + v.reserve(size); + for (size_t i = 1; i < k + 1; ++i) { + if (i & 1) { + v.push_back(i); + } else { + v.push_back(k + i - 1); + } + } + for (size_t i = 1; i < k + 1; ++i) { + v.push_back(2 * i); + } + return v; + } +}; + +#define BENCH_IMPL(BENCH, GEN, IMPL) \ + BENCHMARK_TEMPLATE(BENCH, GEN, IMPL) \ + ->Unit(benchmark::kMicrosecond) \ + ->Arg(kSize - 10) \ + ->Arg(kSize / 2) \ + ->Arg(10000) \ + ->Arg(1000) \ + ->Arg(100) \ + ->Arg(10) \ + ->Arg(1) + +#define BENCH_GENS(BENCH, IMPL) \ + BENCH_IMPL(BENCH, datagens::Random, IMPL); \ + BENCH_IMPL(BENCH, datagens::Shuffled16, IMPL); \ + BENCH_IMPL(BENCH, datagens::Random01, IMPL); \ + BENCH_IMPL(BENCH, datagens::Ascending, IMPL); \ + BENCH_IMPL(BENCH, datagens::Descending, IMPL); \ + BENCH_IMPL(BENCH, datagens::PipeOrgan, IMPL); \ + BENCH_IMPL(BENCH, datagens::PushMiddle, IMPL); \ + BENCH_IMPL(BENCH, datagens::PushFront, IMPL); \ + BENCH_IMPL(BENCH, datagens::Median3Killer, IMPL) + +#define BENCH(NAME) \ + BENCH_GENS(NAME, algorithms::FloydRivest); \ + BENCH_GENS(NAME, algorithms::MedianOfNinthers); \ + BENCH_GENS(NAME, algorithms::MedianOfMedians); \ + BENCH_GENS(NAME, algorithms::MedianOf3Random); \ + BENCH_GENS(NAME, algorithms::PDQ); \ + BENCH_GENS(NAME, algorithms::PDQBranchless); \ + BENCH_GENS(NAME, algorithms::STD) + +} // namespace datagens +} // namespace miniselect diff --git a/contrib/miniselect/benches/benchmark_select.cpp b/contrib/miniselect/benches/benchmark_select.cpp new file mode 100644 index 00000000000..2a9b238c90a --- /dev/null +++ b/contrib/miniselect/benches/benchmark_select.cpp @@ -0,0 +1,46 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#include + +#include +#include +#include +#include +#include +#include + +#include "bench_common.h" +#include "test_common.h" + +namespace miniselect { +namespace { + +static constexpr size_t kSize = 65536; + +template +static void BM_sel(benchmark::State& state) { + auto vec = DataGen::Gen(kSize); + const size_t arg = state.range(0); + size_t cnt = 0; + size_t cmp = 0; + for (auto _ : state) { + Impl::Select(vec.begin(), vec.begin() + arg, vec.end(), + [&cmp](const auto& left, const auto& right) { + cmp++; + return left < right; + }); + ++cnt; + benchmark::DoNotOptimize(vec[arg]); + } + state.counters["Comparisons"] = 1.0 * cmp / cnt; +} + +BENCH(BM_sel); + +} // namespace +} // namespace miniselect + +BENCHMARK_MAIN(); diff --git a/contrib/miniselect/benches/benchmark_sort.cpp b/contrib/miniselect/benches/benchmark_sort.cpp new file mode 100644 index 00000000000..8b3bbd1a77f --- /dev/null +++ b/contrib/miniselect/benches/benchmark_sort.cpp @@ -0,0 +1,46 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#include + +#include +#include +#include +#include +#include +#include + +#include "bench_common.h" +#include "test_common.h" + +namespace miniselect { +namespace { + +static constexpr size_t kSize = 65536; + +template +static void BM_sort(benchmark::State& state) { + auto vec = DataGen::Gen(kSize); + const size_t arg = state.range(0); + size_t cnt = 0; + size_t cmp = 0; + for (auto _ : state) { + Impl::Sort(vec.begin(), vec.begin() + arg, vec.end(), + [&cmp](const auto& left, const auto& right) { + cmp++; + return left < right; + }); + ++cnt; + benchmark::DoNotOptimize(vec[arg]); + } + state.counters["Comparisons"] = 1.0 * cmp / cnt; +} + +BENCH(BM_sort); + +} // namespace +} // namespace miniselect + +BENCHMARK_MAIN(); diff --git a/contrib/miniselect/examples/example.cpp b/contrib/miniselect/examples/example.cpp new file mode 100644 index 00000000000..183e81ae1b6 --- /dev/null +++ b/contrib/miniselect/examples/example.cpp @@ -0,0 +1,18 @@ +#include +#include + +#include "miniselect/median_of_ninthers.h" + +int main() { + std::vector v = {1, 8, 4, 3, 2, 9, 0, 7, 6, 5}; + miniselect::median_of_ninthers_select(v.begin(), v.begin() + 5, v.end()); + for (const int i : v) { + std::cout << i << ' '; + } + return 0; +} + +// Compile it `clang++/g++ -I$DIRECTORY/miniselect/include/ example.cpp -std=c++11 -O3 -o example + +// Possible output: 0 1 4 3 2 5 8 7 6 9 +// ^ on the right place diff --git a/contrib/miniselect/fuzz/CMakeLists.txt b/contrib/miniselect/fuzz/CMakeLists.txt new file mode 100644 index 00000000000..38473bd78ad --- /dev/null +++ b/contrib/miniselect/fuzz/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 3.7) + +project(fuzz) + +option(ENABLE_FUZZING "enable building the fuzzers" ON) +set(CMAKE_CXX_STANDARD 17) + +if(ENABLE_FUZZING) + set(MINISELECT_FUZZ_LDFLAGS "" CACHE STRING "LDFLAGS for the fuzz targets") + + add_library(miniselect-fuzzer INTERFACE) + target_link_libraries(miniselect-fuzzer INTERFACE gtest) + target_link_libraries(miniselect-fuzzer INTERFACE ${MINISELECT_FUZZ_LDFLAGS}) + + if(MINISELECT_FUZZ_LINKMAIN) + target_sources(simdjson-fuzzer INTERFACE $/main.cpp) + endif() + + # Define the fuzzers + add_custom_target(all_fuzzers) + + set(fuzzernames) + function(implement_fuzzer name) + add_executable(${name} ${name}.cpp) + target_link_libraries(${name} PRIVATE miniselect-fuzzer) + add_dependencies(all_fuzzers ${name}) + set(fuzzernames ${fuzzernames} ${name} PARENT_SCOPE) + endfunction() + + implement_fuzzer(fuzz_select) + implement_fuzzer(fuzz_string_select) + implement_fuzzer(fuzz_sort) + implement_fuzzer(fuzz_string_sort) + + # to be able to get a list of all fuzzers from within a script + add_custom_target(print_all_fuzzernames + COMMAND ${CMAKE_COMMAND} -E echo ${fuzzernames}) +endif() diff --git a/contrib/miniselect/fuzz/build_like_oss_fuzz.sh b/contrib/miniselect/fuzz/build_like_oss_fuzz.sh new file mode 100755 index 00000000000..547348133a9 --- /dev/null +++ b/contrib/miniselect/fuzz/build_like_oss_fuzz.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# +# This script emulates how oss fuzz invokes the build +# process, handy for trouble shooting cmake issues and possibly +# recreating testcases. For proper debugging of the oss fuzz +# build, follow the procedure at https://google.github.io/oss-fuzz/getting-started/new-project-guide/#testing-locally + +set -eu + +ossfuzz=$(readlink -f $(dirname $0))/ossfuzz.sh + +mkdir -p ossfuzz-out +export OUT=$(pwd)/ossfuzz-out +export CC=clang +export CXX="clang++" +export CFLAGS="-fsanitize=fuzzer-no-link" +export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -O1" +export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + +$ossfuzz + +echo "look at the results in $OUT" diff --git a/contrib/miniselect/fuzz/fuzz_select.cpp b/contrib/miniselect/fuzz/fuzz_select.cpp new file mode 100644 index 00000000000..f70980bd0d9 --- /dev/null +++ b/contrib/miniselect/fuzz/fuzz_select.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include + +#include "test_common.h" + +template +void ChooseImplementation(uint8_t byte, std::vector& working, + Iter partition_iter, const ::testing::Types&) { + static_assert(sizeof...(T) < 256); + int i = 0; + constexpr size_t size = sizeof...(T); + ( + [&]() { + if (byte % size == i++) { + T::Select(working.begin(), partition_iter, working.end()); + } + }(), + ...); +} + +// Use the first element as a position into the data +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, + std::size_t size) { + if (size <= 3) return 0; + uint8_t impl = data[0]; + uint16_t partition_point = 0; + memcpy(&partition_point, data + 1, 2); + partition_point %= (size - 3); + std::vector working(data + 3, data + size); + auto canonical = working; + const auto partition_iter = working.begin() + partition_point; + ChooseImplementation(impl, working, partition_iter, + miniselect::algorithms::All{}); + + if (partition_iter != working.end()) { + const auto& nth = *partition_iter; + bool is_error = false; + if (!std::all_of(working.begin(), partition_iter, + [&](const auto& v) { return v <= nth; })) { + is_error = true; + } + if (!std::all_of(partition_iter, working.end(), + [&](const auto& v) { return v >= nth; })) { + is_error = true; + } + if (is_error) { + std::cerr << "FAILED!\nCanonical: "; + for (const auto& s : canonical) { + std::cerr << static_cast(s) << ' '; + } + std::cerr << std::endl; + std::cerr << "Got: "; + for (const auto& s : working) { + std::cerr << static_cast(s) << ' '; + } + std::cerr << std::endl; + std::cerr << "partition_iter = " << partition_iter - working.begin() + << std::endl; + std::abort(); + } + } + + return 0; +} diff --git a/contrib/miniselect/fuzz/fuzz_sort.cpp b/contrib/miniselect/fuzz/fuzz_sort.cpp new file mode 100644 index 00000000000..ba0a2b6ca3e --- /dev/null +++ b/contrib/miniselect/fuzz/fuzz_sort.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include + +#include "test_common.h" + +template +void ChooseImplementation(uint8_t byte, std::vector& working, + Iter partition_iter, const ::testing::Types&) { + static_assert(sizeof...(T) < 256); + int i = 0; + constexpr size_t size = sizeof...(T); + ( + [&]() { + if (byte % size == i++) { + T::Sort(working.begin(), partition_iter, working.end()); + } + }(), + ...); +} + +// Use the first element as a position into the data +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, + std::size_t size) { + if (size <= 3) return 0; + uint8_t impl = data[0]; + uint16_t partition_point = 0; + memcpy(&partition_point, data + 1, 2); + partition_point %= (size - 3); + std::vector working(data + 3, data + size); + auto canonical = working; + const auto partition_iter = working.begin() + partition_point; + ChooseImplementation(impl, working, partition_iter, + miniselect::algorithms::All{}); + + bool is_error = false; + if (partition_iter != working.end()) { + const auto& nth = *std::min_element(partition_iter, working.end()); + if (!std::all_of(working.begin(), partition_iter, + [&](const auto& v) { return v <= nth; })) { + is_error = true; + } + if (!std::all_of(partition_iter, working.end(), + [&](const auto& v) { return v >= nth; })) { + is_error = true; + } + } + if (!std::is_sorted(working.begin(), partition_iter)) { + is_error = true; + } + if (is_error) { + std::cerr << "FAILED!\nCanonical: "; + for (const auto& s : canonical) { + std::cerr << static_cast(s) << ' '; + } + std::cerr << std::endl; + std::cerr << "Got: "; + for (const auto& s : working) { + std::cerr << static_cast(s) << ' '; + } + std::cerr << std::endl; + std::cerr << "partition_iter = " << partition_iter - working.begin() + << std::endl; + std::abort(); + } + + return 0; +} diff --git a/contrib/miniselect/fuzz/fuzz_string_select.cpp b/contrib/miniselect/fuzz/fuzz_string_select.cpp new file mode 100644 index 00000000000..cd24b376d86 --- /dev/null +++ b/contrib/miniselect/fuzz/fuzz_string_select.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include + +#include "test_common.h" + +template +void ChooseImplementation(uint8_t byte, std::vector& working, + Iter partition_iter, const ::testing::Types&) { + static_assert(sizeof...(T) < 256); + int i = 0; + constexpr size_t size = sizeof...(T); + ( + [&]() { + if (byte % size == i++) { + T::Select(working.begin(), partition_iter, working.end()); + } + }(), + ...); +} + +// Use the first element as a position into the data +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, + std::size_t size) { + if (size <= 3) return 0; + uint8_t impl = data[0]; + uint16_t partition_point = 0; + memcpy(&partition_point, data + 1, 2); + partition_point %= (size - 3); + std::vector working; + for (auto i = data + 3; i < data + size; ++i) { + std::string s(1, *i); + working.push_back(s); + } + auto canonical = working; + const auto partition_iter = working.begin() + partition_point; + ChooseImplementation(impl, working, partition_iter, + miniselect::algorithms::All{}); + // nth may be the end iterator, in this case nth_element has no effect. + if (partition_iter != working.end()) { + const auto& nth = *partition_iter; + bool is_error = false; + if (!std::all_of(working.begin(), partition_iter, + [&](const auto& v) { return v <= nth; })) { + is_error = true; + } + if (!std::all_of(partition_iter, working.end(), + [&](const auto& v) { return v >= nth; })) { + is_error = true; + } + if (is_error) { + std::cerr << "FAILED!\nCanonical: "; + for (const auto& s : canonical) { + std::cerr << s << ' '; + } + std::cerr << std::endl; + std::cerr << "Got: "; + for (const auto& s : working) { + std::cerr << s << ' '; + } + std::cerr << std::endl; + std::cerr << "partition_iter = " << partition_iter - working.begin() + << std::endl; + std::abort(); + } + } + + return 0; +} diff --git a/contrib/miniselect/fuzz/fuzz_string_sort.cpp b/contrib/miniselect/fuzz/fuzz_string_sort.cpp new file mode 100644 index 00000000000..a797e0d7e22 --- /dev/null +++ b/contrib/miniselect/fuzz/fuzz_string_sort.cpp @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +#include "test_common.h" + +template +void ChooseImplementation(uint8_t byte, std::vector& working, + Iter partition_iter, const ::testing::Types&) { + static_assert(sizeof...(T) < 256); + int i = 0; + constexpr size_t size = sizeof...(T); + ( + [&]() { + if (byte % size == i++) { + T::Sort(working.begin(), partition_iter, working.end()); + } + }(), + ...); +} + +// Use the first element as a position into the data +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, + std::size_t size) { + if (size <= 3) return 0; + uint8_t impl = data[0]; + uint16_t partition_point = 0; + memcpy(&partition_point, data + 1, 2); + partition_point %= (size - 3); + std::vector working; + for (auto i = data + 3; i < data + size; ++i) { + std::string s(1, *i); + working.push_back(s); + } + auto canonical = working; + const auto partition_iter = working.begin() + partition_point; + ChooseImplementation(impl, working, partition_iter, + miniselect::algorithms::All{}); + // nth may be the end iterator, in this case nth_element has no effect. + bool is_error = false; + if (partition_iter != working.end()) { + const auto& nth = *std::min_element(partition_iter, working.end()); + if (!std::all_of(working.begin(), partition_iter, + [&](const auto& v) { return v <= nth; })) { + is_error = true; + } + if (!std::all_of(partition_iter, working.end(), + [&](const auto& v) { return v >= nth; })) { + is_error = true; + } + } + if (!std::is_sorted(working.begin(), partition_iter)) { + is_error = true; + } + if (is_error) { + std::cerr << "FAILED!\nCanonical: "; + for (const auto& s : canonical) { + std::cerr << s << ' '; + } + std::cerr << std::endl; + std::cerr << "Got: "; + for (const auto& s : working) { + std::cerr << s << ' '; + } + std::cerr << std::endl; + std::cerr << "partition_iter = " << partition_iter - working.begin() + << std::endl; + std::abort(); + } + + return 0; +} diff --git a/contrib/miniselect/fuzz/main.cpp b/contrib/miniselect/fuzz/main.cpp new file mode 100644 index 00000000000..e3377035f33 --- /dev/null +++ b/contrib/miniselect/fuzz/main.cpp @@ -0,0 +1,22 @@ +#include +#include +#include +#include + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, std::size_t Size); + +int main(int argc, char* argv[]) { + for (int i = 1; i < argc; ++i) { + std::ifstream in(argv[i]); + assert(in); + in.seekg(0, std::ios_base::end); + const auto pos = in.tellg(); + assert(pos >= 0); + in.seekg(0, std::ios_base::beg); + std::vector buf(static_cast(pos)); + in.read(buf.data(), static_cast(buf.size())); + assert(in.gcount() == pos); + LLVMFuzzerTestOneInput(reinterpret_cast(buf.data()), + buf.size()); + } +} diff --git a/contrib/miniselect/fuzz/ossfuzz.sh b/contrib/miniselect/fuzz/ossfuzz.sh new file mode 100755 index 00000000000..83f37f54eee --- /dev/null +++ b/contrib/miniselect/fuzz/ossfuzz.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# +# entry point for oss-fuzz, so that fuzzers +# and build invocation can be changed without having +# to modify the oss-fuzz repo. +# +# invoke it from the git root. + +# make sure to exit on problems +set -eux + +mkdir -p build +cd build + +cmake .. \ +-GNinja \ +-DCMAKE_BUILD_TYPE=Debug \ +-DENABLE_FUZZING=On \ +-DMINISELECT_FUZZ_LINKMAIN=off \ +-DMINISELECT_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE + +cmake --build . --target all_fuzzers + diff --git a/contrib/miniselect/include/miniselect/floyd_rivest_select.h b/contrib/miniselect/include/miniselect/floyd_rivest_select.h new file mode 100644 index 00000000000..e7d5f80f572 --- /dev/null +++ b/contrib/miniselect/include/miniselect/floyd_rivest_select.h @@ -0,0 +1,120 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace miniselect { +namespace floyd_rivest_detail { + +template +struct CompareRefType { + // Pass the comparator by lvalue reference. Or in debug mode, using a + // debugging wrapper that stores a reference. + using type = typename std::add_lvalue_reference::type; +}; + +template +inline void floyd_rivest_select_loop(Iter begin, Diff left, Diff right, Diff k, + Compare comp) { + while (right > left) { + Diff size = right - left; + if (size > 600) { + Diff n = right - left + 1; + Diff i = k - left + 1; + double z = log(n); + double s = 0.5 * exp(2 * z / 3); + double sd = 0.5 * sqrt(z * s * (n - s) / n); + if (i < n / 2) { + sd *= -1.0; + } + Diff newLeft = std::max(left, (Diff)(k - i * s / n + sd)); + Diff newRight = std::min(right, (Diff)(k + (n - i) * s / n + sd)); + floyd_rivest_select_loop(begin, newLeft, newRight, k, + comp); + } + Diff i = left; + Diff j = right; + std::swap(begin[left], begin[k]); + const bool to_swap = comp(begin[left], begin[right]); + if (to_swap) { + std::swap(begin[left], begin[right]); + } + // Make sure that non copyable types compile. + const auto& t = to_swap ? begin[left] : begin[right]; + while (i < j) { + std::swap(begin[i], begin[j]); + i++; + j--; + while (comp(begin[i], t)) { + i++; + } + while (comp(t, begin[j])) { + j--; + } + } + + if (to_swap) { + std::swap(begin[left], begin[j]); + } else { + j++; + std::swap(begin[right], begin[j]); + } + + if (j <= k) { + left = j + 1; + } + if (k <= j) { + right = j - 1; + } + } +} + +} // namespace floyd_rivest_detail + +template +inline void floyd_rivest_partial_sort(Iter begin, Iter mid, Iter end, + Compare comp) { + if (begin == end) return; + if (begin == mid) return; + using CompType = typename floyd_rivest_detail::CompareRefType::type; + + floyd_rivest_detail::floyd_rivest_select_loop< + Iter, CompType, typename std::iterator_traits::difference_type>( + begin, 0, end - begin - 1, mid - begin - 1, comp); + // std::sort proved to be better than other sorts because of pivoting. + std::sort(begin, mid, comp); +} + +template +inline void floyd_rivest_partial_sort(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + floyd_rivest_partial_sort(begin, mid, end, std::less()); +} + +template +inline void floyd_rivest_select(Iter begin, Iter mid, Iter end, Compare comp) { + if (mid == end) return; + using CompType = typename floyd_rivest_detail::CompareRefType::type; + + floyd_rivest_detail::floyd_rivest_select_loop< + Iter, CompType, typename std::iterator_traits::difference_type>( + begin, 0, end - begin - 1, mid - begin, comp); +} + +template +inline void floyd_rivest_select(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + floyd_rivest_select(begin, mid, end, std::less()); +} + +} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/median_of_3_random.h b/contrib/miniselect/include/miniselect/median_of_3_random.h new file mode 100644 index 00000000000..0f7b62fd61c --- /dev/null +++ b/contrib/miniselect/include/miniselect/median_of_3_random.h @@ -0,0 +1,69 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include +#include +#include +#include +#include + +#include "private/median_common.h" + +namespace miniselect { +namespace median_of_3_random_detail { + +template +static inline Iter partition(Iter r, Iter end, Compare&& comp) { + typedef typename std::iterator_traits::difference_type T; + const T len = end - r; + assert(len >= 3); + static std::mt19937_64 gen(1); + std::uniform_int_distribution dis(0, len - 1); + T x = dis(gen); + T y = dis(gen); + T z = dis(gen); + return median_common_detail::pivotPartition( + r, median_common_detail::medianIndex(r, x, y, z, comp), len, comp); +} + +} // namespace median_of_3_random_detail + +template +inline void median_of_3_random_select(Iter begin, Iter mid, Iter end, + Compare comp) { + if (mid == end) return; + using CompType = typename floyd_rivest_detail::CompareRefType::type; + + median_common_detail::quickselect< + Iter, CompType, &median_of_3_random_detail::partition>( + begin, mid, end, comp); +} + +template +inline void median_of_3_random_select(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + median_of_3_random_select(begin, mid, end, std::less()); +} + +template +inline void median_of_3_random_sort(Iter begin, Iter mid, Iter end, + Compare comp) { + if (begin == mid) return; + using CompType = typename floyd_rivest_detail::CompareRefType::type; + median_common_detail::quickselect< + Iter, CompType, &median_of_3_random_detail::partition>( + begin, mid - 1, end, comp); + std::sort(begin, mid, comp); +} + +template +inline void median_of_3_random_sort(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + median_of_3_random_sort(begin, mid, end, std::less()); +} + +} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/median_of_medians.h b/contrib/miniselect/include/miniselect/median_of_medians.h new file mode 100644 index 00000000000..922401b12d0 --- /dev/null +++ b/contrib/miniselect/include/miniselect/median_of_medians.h @@ -0,0 +1,71 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include +#include +#include +#include + +#include "private/median_common.h" + +namespace miniselect { +namespace median_of_medians_detail { + +template +static inline Iter partition(Iter r, Iter end, Compare&& comp) { + using CompType = typename median_common_detail::CompareRefType::type; + const size_t len = end - r; + if (len < 5) { + return median_common_detail::pivotPartition(r, len / 2, len, comp); + } + size_t j = 0; + for (size_t i = 4; i < len; i += 5, ++j) { + median_common_detail::partition5(r, i - 4, i - 3, i, i - 2, i - 1, comp); + std::swap(r[i], r[j]); + } + median_common_detail::quickselect(r, r + j / 2, + r + j, comp); + return median_common_detail::pivotPartition(r, j / 2, len, comp); +} + +} // namespace median_of_medians_detail + +template +inline void median_of_medians_select(Iter begin, Iter mid, Iter end, + Compare comp) { + if (mid == end) return; + using CompType = typename median_common_detail::CompareRefType::type; + + median_common_detail::quickselect< + Iter, CompType, &median_of_medians_detail::partition>( + begin, mid, end, comp); +} + +template +inline void median_of_medians_select(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + median_of_medians_select(begin, mid, end, std::less()); +} + +template +inline void median_of_medians_sort(Iter begin, Iter mid, Iter end, + Compare comp) { + if (begin == mid) return; + using CompType = typename median_common_detail::CompareRefType::type; + median_common_detail::quickselect< + Iter, CompType, &median_of_medians_detail::partition>( + begin, mid - 1, end, comp); + std::sort(begin, mid, comp); +} + +template +inline void median_of_medians_sort(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + median_of_medians_sort(begin, mid, end, std::less()); +} + +} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/median_of_ninthers.h b/contrib/miniselect/include/miniselect/median_of_ninthers.h new file mode 100644 index 00000000000..099786cf518 --- /dev/null +++ b/contrib/miniselect/include/miniselect/median_of_ninthers.h @@ -0,0 +1,190 @@ +/* Copyright Andrei Alexandrescu, 2016-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +// Adjusted from Alexandrescu paper to support arbitrary comparators. +#pragma once + +#include +#include +#include +#include +#include + +#include "private/median_common.h" + +namespace miniselect { +namespace median_of_ninthers_detail { + +template +void adaptiveQuickselect(Iter r, size_t n, size_t length, Compare&& comp); + +/** +Median of minima +*/ +template +size_t medianOfMinima(Iter const r, const size_t n, const size_t length, + Compare&& comp) { + assert(length >= 2); + assert(n * 4 <= length); + assert(n > 0); + const size_t subset = n * 2, computeMinOver = (length - subset) / subset; + assert(computeMinOver > 0); + for (size_t i = 0, j = subset; i < subset; ++i) { + const auto limit = j + computeMinOver; + size_t minIndex = j; + while (++j < limit) + if (comp(r[j], r[minIndex])) minIndex = j; + if (comp(r[minIndex], r[i])) std::swap(r[i], r[minIndex]); + assert(j < length || i + 1 == subset); + } + adaptiveQuickselect(r, n, subset, comp); + return median_common_detail::expandPartition(r, 0, n, subset, length, comp); +} + +/** +Median of maxima +*/ +template +size_t medianOfMaxima(Iter const r, const size_t n, const size_t length, + Compare&& comp) { + assert(length >= 2); + assert(n * 4 >= length * 3 && n < length); + const size_t subset = (length - n) * 2, subsetStart = length - subset, + computeMaxOver = subsetStart / subset; + assert(computeMaxOver > 0); + for (size_t i = subsetStart, j = i - subset * computeMaxOver; i < length; + ++i) { + const auto limit = j + computeMaxOver; + size_t maxIndex = j; + while (++j < limit) + if (comp(r[maxIndex], r[j])) maxIndex = j; + if (comp(r[i], r[maxIndex])) std::swap(r[i], r[maxIndex]); + assert(j != 0 || i + 1 == length); + } + adaptiveQuickselect(r + subsetStart, length - n, subset, comp); + return median_common_detail::expandPartition(r, subsetStart, n, length, + length, comp); +} + +/** +Partitions r[0 .. length] using a pivot of its own choosing. Attempts to pick a +pivot that approximates the median. Returns the position of the pivot. +*/ +template +size_t medianOfNinthers(Iter const r, const size_t length, Compare&& comp) { + assert(length >= 12); + const auto frac = length <= 1024 + ? length / 12 + : length <= 128 * 1024 ? length / 64 : length / 1024; + auto pivot = frac / 2; + const auto lo = length / 2 - pivot, hi = lo + frac; + assert(lo >= frac * 4); + assert(length - hi >= frac * 4); + assert(lo / 2 >= pivot); + const auto gap = (length - 9 * frac) / 4; + auto a = lo - 4 * frac - gap, b = hi + gap; + for (size_t i = lo; i < hi; ++i, a += 3, b += 3) { + median_common_detail::ninther(r, a, i - frac, b, a + 1, i, b + 1, a + 2, + i + frac, b + 2, comp); + } + + adaptiveQuickselect(r + lo, pivot, frac, comp); + return median_common_detail::expandPartition(r, lo, lo + pivot, hi, length, + comp); +} + +/** +Quickselect driver for medianOfNinthers, medianOfMinima, and medianOfMaxima. +Dispathes to each depending on the relationship between n (the sought order +statistics) and length. +*/ +template +void adaptiveQuickselect(Iter r, size_t n, size_t length, Compare&& comp) { + assert(n < length); + for (;;) { + // Decide strategy for partitioning + if (n == 0) { + // That would be the max + auto pivot = n; + for (++n; n < length; ++n) + if (comp(r[n], r[pivot])) pivot = n; + std::swap(r[0], r[pivot]); + return; + } + if (n + 1 == length) { + // That would be the min + auto pivot = 0; + for (n = 1; n < length; ++n) + if (comp(r[pivot], r[n])) pivot = n; + std::swap(r[pivot], r[length - 1]); + return; + } + assert(n < length); + size_t pivot; + if (length <= 16) + pivot = median_common_detail::pivotPartition(r, n, length, comp) - r; + else if (n * 6 <= length) + pivot = medianOfMinima(r, n, length, comp); + else if (n * 6 >= length * 5) + pivot = medianOfMaxima(r, n, length, comp); + else + pivot = medianOfNinthers(r, length, comp); + + // See how the pivot fares + if (pivot == n) { + return; + } + if (pivot > n) { + length = pivot; + } else { + ++pivot; + r += pivot; + length -= pivot; + n -= pivot; + } + } +} + +} // namespace median_of_ninthers_detail + +template +inline void median_of_ninthers_select(Iter begin, Iter mid, Iter end, + Compare comp) { + if (mid == end) return; + using CompType = typename median_common_detail::CompareRefType::type; + + median_of_ninthers_detail::adaptiveQuickselect( + begin, mid - begin, end - begin, comp); +} + +template +inline void median_of_ninthers_select(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + median_of_ninthers_select(begin, mid, end, std::less()); +} + +template +inline void median_of_ninthers_sort(Iter begin, Iter mid, Iter end, + Compare comp) { + if (begin == mid) return; + using CompType = typename median_common_detail::CompareRefType::type; + + median_of_ninthers_detail::adaptiveQuickselect( + begin, mid - begin - 1, end - begin, comp); + std::sort(begin, mid, comp); +} + +template +inline void median_of_ninthers_sort(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + median_of_ninthers_sort(begin, mid, end, std::less()); +} + +} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/pdqselect.h b/contrib/miniselect/include/miniselect/pdqselect.h new file mode 100644 index 00000000000..0a22d059103 --- /dev/null +++ b/contrib/miniselect/include/miniselect/pdqselect.h @@ -0,0 +1,935 @@ +/* + pdqsort.h - Pattern-defeating quicksort. + + Copyright (c) 2015 Orson Peters + + This software is provided 'as-is', without any express or implied warranty. + In no event will the authors be held liable for any damages arising from the + use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software in a + product, an acknowledgment in the product documentation would be appreciated + but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source distribution. +*/ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +// Adjusted by Danila Kutenin to support pdqselect and pdqpartial_sort. + +#ifndef PDQSORT_H +#define PDQSORT_H + +#include +#include +#include +#include +#include + +#if __cplusplus >= 201103L +#include +#include +#define PDQSORT_PREFER_MOVE(x) std::move(x) +#else +#define PDQSORT_PREFER_MOVE(x) (x) +#endif + +namespace miniselect { +namespace pdqsort_detail { + +template +struct CompareRefType { + // Pass the comparator by lvalue reference. Or in debug mode, using a + // debugging wrapper that stores a reference. + using type = typename std::add_lvalue_reference::type; +}; + +enum { + // Partitions below this size are sorted using insertion sort. + insertion_sort_threshold = 24, + + // Partitions above this size use Tukey's ninther to select the pivot. + ninther_threshold = 128, + + // When we detect an already sorted partition, attempt an insertion sort that + // allows this + // amount of element moves before giving up. + partial_insertion_sort_limit = 8, + + // Must be multiple of 8 due to loop unrolling, and < 256 to fit in unsigned + // char. + block_size = 64, + + // Cacheline size, assumes power of two. + cacheline_size = 64 + +}; + +#if __cplusplus >= 201103L +template +struct is_default_compare : std::false_type {}; +template +struct is_default_compare> : std::true_type {}; +template +struct is_default_compare> : std::true_type {}; +#endif + +// Returns floor(log2(n)), assumes n > 0. +template +inline int log2(T n) { + int log = 0; + while (n >>= 1) ++log; + return log; +} + +// Sorts [begin, end) using insertion sort with the given comparison function. +template +inline void insertion_sort(Iter begin, Iter end, Compare& comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return; + + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned + // correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { + *sift-- = PDQSORT_PREFER_MOVE(*sift_1); + } while (sift != begin && comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + } + } +} + +// Sorts [begin, end) using insertion sort with the given comparison function. +// Assumes +// *(begin - 1) is an element smaller than or equal to any element in [begin, +// end). +template +inline void unguarded_insertion_sort(Iter begin, Iter end, Compare& comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return; + + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned + // correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { + *sift-- = PDQSORT_PREFER_MOVE(*sift_1); + } while (comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + } + } +} + +// Attempts to use insertion sort on [begin, end). Will return false if more +// than partial_insertion_sort_limit elements were moved, and abort sorting. +// Otherwise it will successfully sort and return true. +template +inline bool partial_insertion_sort(Iter begin, Iter end, Compare& comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return true; + + std::size_t limit = 0; + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned + // correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { + *sift-- = PDQSORT_PREFER_MOVE(*sift_1); + } while (sift != begin && comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + limit += cur - sift; + } + + if (limit > partial_insertion_sort_limit) return false; + } + + return true; +} + +template +inline void sort2(Iter a, Iter b, Compare& comp) { + if (comp(*b, *a)) std::iter_swap(a, b); +} + +// Sorts the elements *a, *b and *c using comparison function comp. +template +inline void sort3(Iter a, Iter b, Iter c, Compare& comp) { + sort2(a, b, comp); + sort2(b, c, comp); + sort2(a, b, comp); +} + +template +inline T* align_cacheline(T* p) { +#if defined(UINTPTR_MAX) && __cplusplus >= 201103L + std::uintptr_t ip = reinterpret_cast(p); +#else + std::size_t ip = reinterpret_cast(p); +#endif + ip = (ip + cacheline_size - 1) & -cacheline_size; + return reinterpret_cast(ip); +} + +template +inline void swap_offsets(Iter first, Iter last, unsigned char* offsets_l, + unsigned char* offsets_r, int num, bool use_swaps) { + typedef typename std::iterator_traits::value_type T; + if (use_swaps) { + // This case is needed for the descending distribution, where we need + // to have proper swapping for pdqsort to remain O(n). + for (int i = 0; i < num; ++i) { + std::iter_swap(first + offsets_l[i], last - offsets_r[i]); + } + } else if (num > 0) { + Iter l = first + offsets_l[0]; + Iter r = last - offsets_r[0]; + T tmp(PDQSORT_PREFER_MOVE(*l)); + *l = PDQSORT_PREFER_MOVE(*r); + for (int i = 1; i < num; ++i) { + l = first + offsets_l[i]; + *r = PDQSORT_PREFER_MOVE(*l); + r = last - offsets_r[i]; + *l = PDQSORT_PREFER_MOVE(*r); + } + *r = PDQSORT_PREFER_MOVE(tmp); + } +} + +// Partitions [begin, end) around pivot *begin using comparison function comp. +// Elements equal to the pivot are put in the right-hand partition. Returns the +// position of the pivot after partitioning and whether the passed sequence +// already was correctly partitioned. Assumes the pivot is a median of at least +// 3 elements and that [begin, end) is at least insertion_sort_threshold long. +// Uses branchless partitioning. +template +inline std::pair partition_right_branchless(Iter begin, Iter end, + Compare& comp) { + typedef typename std::iterator_traits::value_type T; + + // Move pivot into local for speed. + T pivot(PDQSORT_PREFER_MOVE(*begin)); + Iter first = begin; + Iter last = end; + + // Find the first element greater than or equal than the pivot (the median of + // 3 guarantees this exists). + while (comp(*++first, pivot)) + ; + + // Find the first element strictly smaller than the pivot. We have to guard + // this search if there was no element before *first. + if (first - 1 == begin) + while (first < last && !comp(*--last, pivot)) + ; + else + while (!comp(*--last, pivot)) + ; + + // If the first pair of elements that should be swapped to partition are the + // same element, the passed in sequence already was correctly partitioned. + bool already_partitioned = first >= last; + if (!already_partitioned) { + std::iter_swap(first, last); + ++first; + } + + // The following branchless partitioning is derived from "BlockQuicksort: How + // Branch Mispredictions don’t affect Quicksort" by Stefan Edelkamp and Armin + // Weiss. + unsigned char offsets_l_storage[block_size + cacheline_size]; + unsigned char offsets_r_storage[block_size + cacheline_size]; + unsigned char* offsets_l = align_cacheline(offsets_l_storage); + unsigned char* offsets_r = align_cacheline(offsets_r_storage); + int num_l, num_r, start_l, start_r; + num_l = num_r = start_l = start_r = 0; + + while (last - first > 2 * block_size) { + // Fill up offset blocks with elements that are on the wrong side. + if (num_l == 0) { + start_l = 0; + Iter it = first; + for (unsigned char i = 0; i < block_size;) { + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + } + } + if (num_r == 0) { + start_r = 0; + Iter it = last; + for (unsigned char i = 0; i < block_size;) { + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + } + } + + // Swap elements and update block sizes and first/last boundaries. + int num = std::min(num_l, num_r); + swap_offsets(first, last, offsets_l + start_l, offsets_r + start_r, num, + num_l == num_r); + num_l -= num; + num_r -= num; + start_l += num; + start_r += num; + if (num_l == 0) first += block_size; + if (num_r == 0) last -= block_size; + } + + int l_size = 0, r_size = 0; + int unknown_left = (int)(last - first) - ((num_r || num_l) ? block_size : 0); + if (num_r) { + // Handle leftover block by assigning the unknown elements to the other + // block. + l_size = unknown_left; + r_size = block_size; + } else if (num_l) { + l_size = block_size; + r_size = unknown_left; + } else { + // No leftover block, split the unknown elements in two blocks. + l_size = unknown_left / 2; + r_size = unknown_left - l_size; + } + + // Fill offset buffers if needed. + if (unknown_left && !num_l) { + start_l = 0; + Iter it = first; + for (unsigned char i = 0; i < l_size;) { + offsets_l[num_l] = i++; + num_l += !comp(*it, pivot); + ++it; + } + } + if (unknown_left && !num_r) { + start_r = 0; + Iter it = last; + for (unsigned char i = 0; i < r_size;) { + offsets_r[num_r] = ++i; + num_r += comp(*--it, pivot); + } + } + + int num = std::min(num_l, num_r); + swap_offsets(first, last, offsets_l + start_l, offsets_r + start_r, num, + num_l == num_r); + num_l -= num; + num_r -= num; + start_l += num; + start_r += num; + if (num_l == 0) first += l_size; + if (num_r == 0) last -= r_size; + + // We have now fully identified [first, last)'s proper position. Swap the last + // elements. + if (num_l) { + offsets_l += start_l; + while (num_l--) std::iter_swap(first + offsets_l[num_l], --last); + first = last; + } + if (num_r) { + offsets_r += start_r; + while (num_r--) std::iter_swap(last - offsets_r[num_r], first), ++first; + last = first; + } + + // Put the pivot in the right place. + Iter pivot_pos = first - 1; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return std::make_pair(pivot_pos, already_partitioned); +} + +// Partitions [begin, end) around pivot *begin using comparison function comp. +// Elements equal to the pivot are put in the right-hand partition. Returns the +// position of the pivot after partitioning and whether the passed sequence +// already was correctly partitioned. Assumes the pivot is a median of at least +// 3 elements and that [begin, end) is at least insertion_sort_threshold long. +template +inline std::pair partition_right(Iter begin, Iter end, + Compare& comp) { + typedef typename std::iterator_traits::value_type T; + + // Move pivot into local for speed. + T pivot(PDQSORT_PREFER_MOVE(*begin)); + + Iter first = begin; + Iter last = end; + + // Find the first element greater than or equal than the pivot (the median of + // 3 guarantees this exists). + while (comp(*++first, pivot)) + ; + + // Find the first element strictly smaller than the pivot. We have to guard + // this search if there was no element before *first. + if (first - 1 == begin) + while (first < last && !comp(*--last, pivot)) + ; + else + while (!comp(*--last, pivot)) + ; + + // If the first pair of elements that should be swapped to partition are the + // same element, the passed in sequence already was correctly partitioned. + bool already_partitioned = first >= last; + + // Keep swapping pairs of elements that are on the wrong side of the pivot. + // Previously swapped pairs guard the searches, which is why the first + // iteration is special-cased above. + while (first < last) { + std::iter_swap(first, last); + while (comp(*++first, pivot)) + ; + while (!comp(*--last, pivot)) + ; + } + + // Put the pivot in the right place. + Iter pivot_pos = first - 1; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return std::make_pair(pivot_pos, already_partitioned); +} + +// Similar function to the one above, except elements equal to the pivot are put +// to the left of the pivot and it doesn't check or return if the passed +// sequence already was partitioned. Since this is rarely used (the many equal +// case), and in that case pdqsort already has O(n) performance, no block +// quicksort is applied here for simplicity. +template +inline Iter partition_left(Iter begin, Iter end, Compare& comp) { + typedef typename std::iterator_traits::value_type T; + + T pivot(PDQSORT_PREFER_MOVE(*begin)); + Iter first = begin; + Iter last = end; + + while (comp(pivot, *--last)) + ; + + if (last + 1 == end) + while (first < last && !comp(pivot, *++first)) + ; + else + while (!comp(pivot, *++first)) + ; + + while (first < last) { + std::iter_swap(first, last); + while (comp(pivot, *--last)) + ; + while (!comp(pivot, *++first)) + ; + } + + Iter pivot_pos = last; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return pivot_pos; +} + +template +inline void pdqsort_loop(Iter begin, Iter end, Compare& comp, int bad_allowed, + bool leftmost = true) { + typedef typename std::iterator_traits::difference_type diff_t; + + // Use a while loop for tail recursion elimination. + while (true) { + diff_t size = end - begin; + + // Insertion sort is faster for small arrays. + if (size < insertion_sort_threshold) { + if (leftmost) + insertion_sort(begin, end, comp); + else + unguarded_insertion_sort(begin, end, comp); + return; + } + + // Choose pivot as median of 3 or pseudomedian of 9. + diff_t s2 = size / 2; + if (size > ninther_threshold) { + sort3(begin, begin + s2, end - 1, comp); + sort3(begin + 1, begin + (s2 - 1), end - 2, comp); + sort3(begin + 2, begin + (s2 + 1), end - 3, comp); + sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); + std::iter_swap(begin, begin + s2); + } else + sort3(begin + s2, begin, end - 1, comp); + + // If *(begin - 1) is the end of the right partition of a previous partition + // operation there is no element in [begin, end) that is smaller than + // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change + // strategy, putting equal elements in the left partition, greater elements + // in the right partition. We do not have to recurse on the left partition, + // since it's sorted (all equal). + if (!leftmost && !comp(*(begin - 1), *begin)) { + begin = partition_left(begin, end, comp) + 1; + continue; + } + + // Partition and get results. + std::pair part_result = + Branchless ? partition_right_branchless(begin, end, comp) + : partition_right(begin, end, comp); + Iter pivot_pos = part_result.first; + bool already_partitioned = part_result.second; + + // Check for a highly unbalanced partition. + diff_t l_size = pivot_pos - begin; + diff_t r_size = end - (pivot_pos + 1); + bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; + + // If we got a highly unbalanced partition we shuffle elements to break many + // patterns. + if (highly_unbalanced) { + // If we had too many bad partitions, switch to heapsort to guarantee O(n + // log n). + if (--bad_allowed == 0) { + std::make_heap(begin, end, comp); + std::sort_heap(begin, end, comp); + return; + } + + if (l_size >= insertion_sort_threshold) { + std::iter_swap(begin, begin + l_size / 4); + std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); + + if (l_size > ninther_threshold) { + std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); + std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); + std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); + std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); + } + } + + if (r_size >= insertion_sort_threshold) { + std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); + std::iter_swap(end - 1, end - r_size / 4); + + if (r_size > ninther_threshold) { + std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); + std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); + std::iter_swap(end - 2, end - (1 + r_size / 4)); + std::iter_swap(end - 3, end - (2 + r_size / 4)); + } + } + } else { + // If we were decently balanced and we tried to sort an already + // partitioned sequence try to use insertion sort. + if (already_partitioned && + partial_insertion_sort(begin, pivot_pos, comp) && + partial_insertion_sort(pivot_pos + 1, end, comp)) + return; + } + + // Sort the left partition first using recursion and do tail recursion + // elimination for the right-hand partition. + pdqsort_loop(begin, pivot_pos, comp, bad_allowed, + leftmost); + begin = pivot_pos + 1; + leftmost = false; + } +} + +template +inline void pdqpartial_sort_loop(Iter begin, Iter mid, Iter end, Compare& comp, + int bad_allowed, bool leftmost = true) { + typedef typename std::iterator_traits::difference_type diff_t; + + // Use a while loop for tail recursion elimination. + while (true) { + diff_t size = end - begin; + + // Insertion sort is faster for small arrays. + if (size < insertion_sort_threshold) { + if (leftmost) + insertion_sort(begin, end, comp); + else + unguarded_insertion_sort(begin, end, comp); + return; + } + + // Choose pivot as median of 3 or pseudomedian of 9. + diff_t s2 = size / 2; + if (size > ninther_threshold) { + sort3(begin, begin + s2, end - 1, comp); + sort3(begin + 1, begin + (s2 - 1), end - 2, comp); + sort3(begin + 2, begin + (s2 + 1), end - 3, comp); + sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); + std::iter_swap(begin, begin + s2); + } else + sort3(begin + s2, begin, end - 1, comp); + + // If *(begin - 1) is the end of the right partition of a previous partition + // operation there is no element in [begin, end) that is smaller than + // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change + // strategy, putting equal elements in the left partition, greater elements + // in the right partition. We do not have to recurse on the left partition, + // since it's sorted (all equal). + if (!leftmost && !comp(*(begin - 1), *begin)) { + begin = partition_left(begin, end, comp) + 1; + continue; + } + + // Partition and get results. + std::pair part_result = + Branchless ? partition_right_branchless(begin, end, comp) + : partition_right(begin, end, comp); + Iter pivot_pos = part_result.first; + bool already_partitioned = part_result.second; + + // Check for a highly unbalanced partition. + diff_t l_size = pivot_pos - begin; + diff_t r_size = end - (pivot_pos + 1); + bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; + + // If we got a highly unbalanced partition we shuffle elements to break many + // patterns. + if (highly_unbalanced) { + // If we had too many bad partitions, switch to heapsort to guarantee O(n + // log n). + if (--bad_allowed == 0) { + std::make_heap(begin, end, comp); + std::sort_heap(begin, end, comp); + return; + } + + if (l_size >= insertion_sort_threshold) { + std::iter_swap(begin, begin + l_size / 4); + std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); + + if (l_size > ninther_threshold) { + std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); + std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); + std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); + std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); + } + } + + if (r_size >= insertion_sort_threshold) { + std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); + std::iter_swap(end - 1, end - r_size / 4); + + if (r_size > ninther_threshold) { + std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); + std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); + std::iter_swap(end - 2, end - (1 + r_size / 4)); + std::iter_swap(end - 3, end - (2 + r_size / 4)); + } + } + } else { + // If we were decently balanced and we tried to sort an already + // partitioned sequence try to use insertion sort. + if (already_partitioned && + partial_insertion_sort(begin, pivot_pos, comp) && + partial_insertion_sort(pivot_pos + 1, end, comp)) + return; + } + + // Sort the left partition first using recursion and do tail recursion + // elimination for the right-hand partition. + if (pivot_pos < mid) { + pdqsort_loop(begin, pivot_pos, comp, + bad_allowed, leftmost); + begin = pivot_pos + 1; + leftmost = false; + } else { + end = pivot_pos; + } + } +} + +template +inline void pdqselect_loop(Iter begin, Iter mid, Iter end, Compare& comp, + int bad_allowed, bool leftmost = true) { + typedef typename std::iterator_traits::difference_type diff_t; + + // Use a while loop for tail recursion elimination. + while (true) { + diff_t size = end - begin; + + // Insertion sort is faster for small arrays. + if (size < insertion_sort_threshold) { + if (leftmost) + insertion_sort(begin, end, comp); + else + unguarded_insertion_sort(begin, end, comp); + return; + } + + // Choose pivot as median of 3 or pseudomedian of 9. + diff_t s2 = size / 2; + if (size > ninther_threshold) { + sort3(begin, begin + s2, end - 1, comp); + sort3(begin + 1, begin + (s2 - 1), end - 2, comp); + sort3(begin + 2, begin + (s2 + 1), end - 3, comp); + sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); + std::iter_swap(begin, begin + s2); + } else + sort3(begin + s2, begin, end - 1, comp); + + // If *(begin - 1) is the end of the right partition of a previous partition + // operation there is no element in [begin, end) that is smaller than + // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change + // strategy, putting equal elements in the left partition, greater elements + // in the right partition. We do not have to recurse on the left partition, + // since it's sorted (all equal). + if (!leftmost && !comp(*(begin - 1), *begin)) { + begin = partition_left(begin, end, comp) + 1; + continue; + } + + // Partition and get results. + std::pair part_result = + Branchless ? partition_right_branchless(begin, end, comp) + : partition_right(begin, end, comp); + Iter pivot_pos = part_result.first; + bool already_partitioned = part_result.second; + + // Check for a highly unbalanced partition. + diff_t l_size = pivot_pos - begin; + diff_t r_size = end - (pivot_pos + 1); + bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; + + // If we got a highly unbalanced partition we shuffle elements to break many + // patterns. + if (highly_unbalanced) { + // If we had too many bad partitions, switch to heapsort to guarantee O(n + // log n). + if (--bad_allowed == 0) { + std::nth_element(begin, mid, end, comp); + return; + } + + if (l_size >= insertion_sort_threshold) { + std::iter_swap(begin, begin + l_size / 4); + std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); + + if (l_size > ninther_threshold) { + std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); + std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); + std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); + std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); + } + } + + if (r_size >= insertion_sort_threshold) { + std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); + std::iter_swap(end - 1, end - r_size / 4); + + if (r_size > ninther_threshold) { + std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); + std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); + std::iter_swap(end - 2, end - (1 + r_size / 4)); + std::iter_swap(end - 3, end - (2 + r_size / 4)); + } + } + } else { + // If we were decently balanced and we tried to sort an already + // partitioned sequence try to use insertion sort. + if (already_partitioned && + partial_insertion_sort(begin, pivot_pos, comp) && + partial_insertion_sort(pivot_pos + 1, end, comp)) + return; + } + // Sort the left partition first using recursion and do tail recursion + // elimination for the right-hand partition. + if (pivot_pos < mid) { + begin = pivot_pos + 1; + leftmost = false; + } else { + end = pivot_pos; + } + } +} +} // namespace pdqsort_detail + +template +inline void pdqsort(Iter begin, Iter end, Compare comp) { + if (begin == end) return; + +#if __cplusplus >= 201103L + pdqsort_detail::pdqsort_loop< + Iter, Compare, + pdqsort_detail::is_default_compare< + typename std::decay::type>::value && + std::is_arithmetic< + typename std::iterator_traits::value_type>::value>( + begin, end, comp, pdqsort_detail::log2(end - begin)); +#else + pdqsort_detail::pdqsort_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); +#endif +} + +template +inline void pdqsort(Iter begin, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqsort(begin, end, std::less()); +} + +template +inline void pdqsort_branchless(Iter begin, Iter end, Compare comp) { + if (begin == end) return; + pdqsort_detail::pdqsort_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); +} + +template +inline void pdqsort_branchless(Iter begin, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqsort_branchless(begin, end, std::less()); +} + +template +inline void pdqpartial_sort(Iter begin, Iter mid, Iter end, Compare comp) { + if (begin == end) return; + +#if __cplusplus >= 201103L + pdqsort_detail::pdqpartial_sort_loop< + Iter, Compare, + pdqsort_detail::is_default_compare< + typename std::decay::type>::value && + std::is_arithmetic< + typename std::iterator_traits::value_type>::value>( + begin, mid, end, comp, pdqsort_detail::log2(end - begin)); +#else + pdqsort_detail::pdqpartial_sort_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); +#endif +} + +template +inline void pdqpartial_sort(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqpartial_sort(begin, mid, end, std::less()); +} + +template +inline void pdqpartial_sort_branchless(Iter begin, Iter mid, Iter end, + Compare comp) { + if (begin == end) return; + pdqsort_detail::pdqpartial_sort_loop( + begin, mid, end, comp, pdqsort_detail::log2(end - begin)); +} + +template +inline void pdqpartial_sort_branchless(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqpartial_sort_branchless(begin, mid, end, std::less()); +} + +template +inline void pdqselect(Iter begin, Iter mid, Iter end, Compare comp) { + if (mid == end) return; + using CompType = typename median_common_detail::CompareRefType::type; + +#if __cplusplus >= 201103L + pdqsort_detail::pdqselect_loop< + Iter, CompType, + pdqsort_detail::is_default_compare< + typename std::decay::type>::value && + std::is_arithmetic< + typename std::iterator_traits::value_type>::value>( + begin, mid, end, comp, pdqsort_detail::log2(end - begin)); +#else + pdqsort_detail::pdqselect_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); +#endif +} + +template +inline void pdqselect(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqselect(begin, mid, end, std::less()); +} + +template +inline void pdqselect_branchless(Iter begin, Iter mid, Iter end, Compare comp) { + if (mid == end) return; + using CompType = typename median_common_detail::CompareRefType::type; + pdqsort_detail::pdqselect_loop( + begin, mid, end, comp, pdqsort_detail::log2(end - begin)); +} + +template +inline void pdqselect_branchless(Iter begin, Iter mid, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqselect_branchless(begin, mid, end, std::less()); +} + +#undef PDQSORT_PREFER_MOVE + +#endif + +} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/private/median_common.h b/contrib/miniselect/include/miniselect/private/median_common.h new file mode 100644 index 00000000000..30cb1323bbf --- /dev/null +++ b/contrib/miniselect/include/miniselect/private/median_common.h @@ -0,0 +1,437 @@ +/* Copyright Andrei Alexandrescu, 2016-, + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include +#include +#include + +namespace miniselect { +namespace median_common_detail { + +template +struct CompareRefType { + // Pass the comparator by lvalue reference. Or in debug mode, using a + // debugging wrapper that stores a reference. + using type = typename std::add_lvalue_reference::type; +}; +/** +Swaps the median of r[a], r[b], and r[c] into r[b]. +*/ +template +void median3(Iter r, size_t a, size_t b, size_t c, Compare&& comp) { + if (comp(r[b], r[a])) // b < a + { + if (comp(r[b], r[c])) // b < a, b < c + { + if (comp(r[c], r[a])) // b < c < a + std::swap(r[b], r[c]); + else // b < a <= c + std::swap(r[b], r[a]); + } + } else if (comp(r[c], r[b])) // a <= b, c < b + { + if (comp(r[c], r[a])) // c < a <= b + std::swap(r[b], r[a]); + else // a <= c < b + std::swap(r[b], r[c]); + } +} + +/** +Sorts in place r[a], r[b], and r[c]. +*/ +template +void sort3(Iter r, size_t a, size_t b, size_t c, Compare&& comp) { + typedef typename std::iterator_traits::value_type T; + if (comp(r[b], r[a])) // b < a + { + if (comp(r[c], r[b])) // c < b < a + { + std::swap(r[a], r[c]); // a < b < c + } else // b < a, b <= c + { + T t = std::move(r[a]); + r[a] = std::move(r[b]); + if (comp(r[c], t)) // b <= c < a + { + r[b] = std::move(r[c]); + r[c] = std::move(t); + } else // b < a <= c + { + r[b] = std::move(t); + } + } + } else if (comp(r[c], r[b])) // a <= b, c < b + { + T t = std::move(r[c]); + r[c] = std::move(r[b]); + if (comp(t, r[a])) // c < a < b + { + r[b] = std::move(r[a]); + r[a] = std::move(t); + } else // a <= c < b + { + r[b] = std::move(t); + } + } + + assert(!comp(r[b], r[a]) && !comp(r[c], r[b])); +} + +/** +If leanRight == false, swaps the lower median of r[a]...r[d] into r[b] and +the minimum into r[a]. If leanRight == true, swaps the upper median of +r[a]...r[d] into r[c] and the minimum into r[d]. +*/ +template +void partition4(Iter r, size_t a, size_t b, size_t c, size_t d, + Compare&& comp) { + assert(a != b && a != c && a != d && b != c && b != d && c != d); + /* static */ if (leanRight) { + // In the median of 5 algorithm, consider r[e] infinite + if (comp(r[c], r[a])) { + std::swap(r[a], r[c]); + } // a <= c + if (comp(r[d], r[b])) { + std::swap(r[b], r[d]); + } // a <= c, b <= d + if (comp(r[d], r[c])) { + std::swap(r[c], r[d]); // a <= d, b <= c < d + std::swap(r[a], r[b]); // b <= d, a <= c < d + } // a <= c <= d, b <= d + if (comp(r[c], r[b])) { // a <= c <= d, c < b <= d + std::swap(r[b], r[c]); // a <= b <= c <= d + } // a <= b <= c <= d + } else { + // In the median of 5 algorithm consider r[a] infinitely small, then + // change b->a. c->b, d->c, e->d + if (comp(r[c], r[a])) { + std::swap(r[a], r[c]); + } + if (comp(r[c], r[b])) { + std::swap(r[b], r[c]); + } + if (comp(r[d], r[a])) { + std::swap(r[a], r[d]); + } + if (comp(r[d], r[b])) { + std::swap(r[b], r[d]); + } else { + if (comp(r[b], r[a])) { + std::swap(r[a], r[b]); + } + } + } +} + +/** +Places the median of r[a]...r[e] in r[c] and partitions the other elements +around it. +*/ +template +void partition5(Iter r, size_t a, size_t b, size_t c, size_t d, size_t e, + Compare&& comp) { + assert(a != b && a != c && a != d && a != e && b != c && b != d && b != e && + c != d && c != e && d != e); + if (comp(r[c], r[a])) { + std::swap(r[a], r[c]); + } + if (comp(r[d], r[b])) { + std::swap(r[b], r[d]); + } + if (comp(r[d], r[c])) { + std::swap(r[c], r[d]); + std::swap(r[a], r[b]); + } + if (comp(r[e], r[b])) { + std::swap(r[b], r[e]); + } + if (comp(r[e], r[c])) { + std::swap(r[c], r[e]); + if (comp(r[c], r[a])) { + std::swap(r[a], r[c]); + } + } else { + if (comp(r[c], r[b])) { + std::swap(r[b], r[c]); + } + } +} + +/** +Implements Hoare partition. +*/ +template +Iter pivotPartition(Iter r, size_t k, size_t length, Compare&& comp) { + assert(k < length); + std::swap(*r, r[k]); + size_t lo = 1, hi = length - 1; + for (;; ++lo, --hi) { + for (;; ++lo) { + if (lo > hi) goto loop_done; + if (!comp(r[lo], *r)) break; + } + // found the left bound: r[lo] >= r[0] + assert(lo <= hi); + for (; comp(*r, r[hi]); --hi) { + } + if (lo >= hi) break; + // found the right bound: r[hi] <= r[0], swap & make progress + std::swap(r[lo], r[hi]); + } +loop_done: + --lo; + std::swap(r[lo], *r); + return r + lo; +} + +/** +Implements the quickselect algorithm, parameterized with a partition function. +*/ +template +void quickselect(Iter r, Iter mid, Iter end, Compare&& comp) { + if (r == end || mid >= end) return; + assert(r <= mid && mid < end); + for (;;) switch (end - r) { + case 1: + return; + case 2: + if (comp(r[1], *r)) std::swap(*r, r[1]); + return; + case 3: + sort3(r, 0, 1, 2, comp); + return; + case 4: + switch (mid - r) { + case 0: + goto select_min; + case 1: + partition4(r, 0, 1, 2, 3, comp); + break; + case 2: + partition4(r, 0, 1, 2, 3, comp); + break; + case 3: + goto select_max; + default: + assert(false); + } + return; + default: + assert(end - r > 4); + if (r == mid) { + select_min: + auto pivot = r; + for (++mid; mid < end; ++mid) + if (comp(*mid, *pivot)) pivot = mid; + std::swap(*r, *pivot); + return; + } + if (mid + 1 == end) { + select_max: + auto pivot = r; + for (mid = r + 1; mid < end; ++mid) + if (comp(*pivot, *mid)) pivot = mid; + std::swap(*pivot, end[-1]); + return; + } + auto pivot = partition(r, end, comp); + if (pivot == mid) return; + if (mid < pivot) { + end = pivot; + } else { + r = pivot + 1; + } + } +} + +/** +Returns the index of the median of r[a], r[b], and r[c] without writing +anything. +*/ +template +size_t medianIndex(const Iter r, size_t a, size_t b, size_t c, Compare&& comp) { + if (r[a] > r[c]) std::swap(a, c); + if (r[b] > r[c]) return c; + if (comp(r[b], r[a])) return a; + return b; +} + +/** +Returns the index of the median of r[a], r[b], r[c], and r[d] without writing +anything. If leanRight is true, computes the upper median. Otherwise, conputes +the lower median. +*/ +template +static size_t medianIndex(Iter r, size_t a, size_t b, size_t c, size_t d, + Compare&& comp) { + if (comp(r[d], r[c])) std::swap(c, d); + assert(r[c] <= r[d]); + /* static */ if (leanRight) { + if (comp(r[c], r[a])) { + assert(comp(r[c], r[a]) && !comp(r[d], r[c])); // so r[c]) is out + return medianIndex(r, a, b, d, comp); + } + } else { + if (!comp(r[d], r[a])) { + return medianIndex(r, a, b, c, comp); + } + } + // Could return medianIndex(r, b, c, d) but we already know r[c] <= r[d] + if (!comp(r[c], r[b])) return c; + if (comp(r[d], r[b])) return d; + return b; +} + +/** +Tukey's Ninther: compute the median of r[_1], r[_2], r[_3], then the median of +r[_4], r[_5], r[_6], then the median of r[_7], r[_8], r[_9], and then swap the +median of those three medians into r[_5]. +*/ +template +void ninther(Iter r, size_t _1, size_t _2, size_t _3, size_t _4, size_t _5, + size_t _6, size_t _7, size_t _8, size_t _9, Compare&& comp) { + _2 = medianIndex(r, _1, _2, _3, comp); + _8 = medianIndex(r, _7, _8, _9, comp); + if (comp(r[_8], r[_2])) std::swap(_2, _8); + if (comp(r[_6], r[_4])) std::swap(_4, _6); + // Here we know that r[_2] and r[_8] are the other two medians and that + // r[_2] <= r[_8]. We also know that r[_4] <= r[_6] + if (comp(r[_5], r[_4])) { + // r[_4] is the median of r[_4], r[_5], r[_6] + } else if (comp(r[_6], r[_5])) { + // r[_6] is the median of r[_4], r[_5], r[_6] + _4 = _6; + } else { + // Here we know r[_5] is the median of r[_4], r[_5], r[_6] + if (comp(r[_5], r[_2])) return std::swap(r[_5], r[_2]); + if (comp(r[_8], r[_5])) return std::swap(r[_5], r[_8]); + // This is the only path that returns with no swap + return; + } + // Here we know r[_4] is the median of r[_4], r[_5], r[_6] + if (comp(r[_4], r[_2])) + _4 = _2; + else if (comp(r[_8], r[_4])) + _4 = _8; + std::swap(r[_5], r[_4]); +} + +/** +Input assumptions: +(a) hi <= rite +(c) the range r[0 .. hi] contains elements no smaller than r[0] +Output guarantee: same as Hoare partition using r[0] as pivot. Returns the new +position of the pivot. +*/ +template +size_t expandPartitionRight(Iter r, size_t hi, size_t rite, Compare&& comp) { + size_t pivot = 0; + assert(pivot <= hi); + assert(hi <= rite); + // First loop: spend r[pivot .. hi] + for (; pivot < hi; --rite) { + if (rite == hi) goto done; + if (!comp(r[rite], r[0])) continue; + ++pivot; + std::swap(r[rite], r[pivot]); + } + // Second loop: make left and pivot meet + for (; rite > pivot; --rite) { + if (!comp(r[rite], r[0])) continue; + while (rite > pivot) { + ++pivot; + if (comp(r[0], r[pivot])) { + std::swap(r[rite], r[pivot]); + break; + } + } + } + +done: + std::swap(r[0], r[pivot]); + return pivot; +} + +/** +Input assumptions: +(a) lo > 0, lo <= pivot +(b) the range r[lo .. pivot] already contains elements no greater than r[pivot] +Output guarantee: Same as Hoare partition around r[pivot]. Returns the new +position of the pivot. +*/ +template +size_t expandPartitionLeft(Iter r, size_t lo, size_t pivot, Compare&& comp) { + assert(lo > 0 && lo <= pivot); + size_t left = 0; + const auto oldPivot = pivot; + for (; lo < pivot; ++left) { + if (left == lo) goto done; + if (!comp(r[oldPivot], r[left])) continue; + --pivot; + std::swap(r[left], r[pivot]); + } + // Second loop: make left and pivot meet + for (;; ++left) { + if (left == pivot) break; + if (!comp(r[oldPivot], r[left])) continue; + for (;;) { + if (left == pivot) goto done; + --pivot; + if (comp(r[pivot], r[oldPivot])) { + std::swap(r[left], r[pivot]); + break; + } + } + } + +done: + std::swap(r[oldPivot], r[pivot]); + return pivot; +} + +/** +Input assumptions: +(a) lo <= pivot, pivot < hi, hi <= length +(b) the range r[lo .. pivot] already contains elements no greater than +r[pivot] +(c) the range r[pivot .. hi] already contains elements no smaller than +r[pivot] +Output guarantee: Same as Hoare partition around r[pivot], returning the new +position of the pivot. +*/ +template +size_t expandPartition(Iter r, size_t lo, size_t pivot, size_t hi, + size_t length, Compare&& comp) { + assert(lo <= pivot && pivot < hi && hi <= length); + --hi; + --length; + size_t left = 0; + for (;; ++left, --length) { + for (;; ++left) { + if (left == lo) + return pivot + expandPartitionRight(r + pivot, hi - pivot, + length - pivot, comp); + if (comp(r[pivot], r[left])) break; + } + for (;; --length) { + if (length == hi) + return left + + expandPartitionLeft(r + left, lo - left, pivot - left, comp); + if (!comp(r[pivot], r[length])) break; + } + std::swap(r[left], r[length]); + } +} + +} // namespace median_common_detail +} // namespace miniselect diff --git a/contrib/miniselect/testing/test_common.h b/contrib/miniselect/testing/test_common.h new file mode 100644 index 00000000000..df0c179c840 --- /dev/null +++ b/contrib/miniselect/testing/test_common.h @@ -0,0 +1,180 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#pragma once + +#include + +#include + +#include "miniselect/floyd_rivest_select.h" +#include "miniselect/median_of_3_random.h" +#include "miniselect/median_of_medians.h" +#include "miniselect/median_of_ninthers.h" +#include "miniselect/pdqselect.h" + +namespace miniselect { +namespace algorithms { + +struct STD { + template + static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { + std::partial_sort(begin, mid, end, std::move(comp)); + } + + template + static void Sort(Iter begin, Iter mid, Iter end) { + std::partial_sort(begin, mid, end); + } + + template + static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { + std::nth_element(begin, mid, end, std::move(comp)); + } + + template + static void Select(Iter begin, Iter mid, Iter end) { + std::nth_element(begin, mid, end); + } +}; + +struct PDQ { + template + static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { + pdqpartial_sort(begin, mid, end, std::move(comp)); + } + + template + static void Sort(Iter begin, Iter mid, Iter end) { + pdqpartial_sort(begin, mid, end); + } + + template + static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { + pdqselect(begin, mid, end, std::move(comp)); + } + + template + static void Select(Iter begin, Iter mid, Iter end) { + pdqselect(begin, mid, end); + } +}; + +struct PDQBranchless { + template + static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { + pdqpartial_sort_branchless(begin, mid, end, std::move(comp)); + } + + template + static void Sort(Iter begin, Iter mid, Iter end) { + pdqpartial_sort_branchless(begin, mid, end); + } + + template + static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { + pdqselect_branchless(begin, mid, end, std::move(comp)); + } + + template + static void Select(Iter begin, Iter mid, Iter end) { + pdqselect_branchless(begin, mid, end); + } +}; + +struct FloydRivest { + template + static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { + floyd_rivest_partial_sort(begin, mid, end, std::move(comp)); + } + + template + static void Sort(Iter begin, Iter mid, Iter end) { + floyd_rivest_partial_sort(begin, mid, end); + } + + template + static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { + floyd_rivest_select(begin, mid, end, std::move(comp)); + } + + template + static void Select(Iter begin, Iter mid, Iter end) { + floyd_rivest_select(begin, mid, end); + } +}; + +struct MedianOfNinthers { + template + static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { + median_of_ninthers_sort(begin, mid, end, std::move(comp)); + } + + template + static void Sort(Iter begin, Iter mid, Iter end) { + median_of_ninthers_sort(begin, mid, end); + } + + template + static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { + median_of_ninthers_select(begin, mid, end, std::move(comp)); + } + + template + static void Select(Iter begin, Iter mid, Iter end) { + median_of_ninthers_select(begin, mid, end); + } +}; + +struct MedianOfMedians { + template + static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { + median_of_medians_sort(begin, mid, end, std::move(comp)); + } + + template + static void Sort(Iter begin, Iter mid, Iter end) { + median_of_medians_sort(begin, mid, end); + } + + template + static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { + median_of_medians_select(begin, mid, end, std::move(comp)); + } + + template + static void Select(Iter begin, Iter mid, Iter end) { + median_of_medians_select(begin, mid, end); + } +}; + +struct MedianOf3Random { + template + static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { + median_of_3_random_sort(begin, mid, end, std::move(comp)); + } + + template + static void Sort(Iter begin, Iter mid, Iter end) { + median_of_3_random_sort(begin, mid, end); + } + + template + static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { + median_of_3_random_select(begin, mid, end, std::move(comp)); + } + + template + static void Select(Iter begin, Iter mid, Iter end) { + median_of_3_random_select(begin, mid, end); + } +}; + +using All = + ::testing::Types; + +} // namespace algorithms +} // namespace miniselect diff --git a/contrib/miniselect/testing/test_select.cpp b/contrib/miniselect/testing/test_select.cpp new file mode 100644 index 00000000000..9b8e9dce970 --- /dev/null +++ b/contrib/miniselect/testing/test_select.cpp @@ -0,0 +1,231 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#include +#include + +#include +#include +#include +#include +#include + +#include "test_common.h" + +using ::testing::Eq; + +namespace miniselect { +namespace { + +struct IndirectLess { + // Non const comparator with deleted copy. + template + bool operator()(const P &x, const P &y) const { + return *x < *y; + } + IndirectLess(const IndirectLess &) = default; + IndirectLess &operator=(const IndirectLess &) = default; + IndirectLess(IndirectLess &&) = default; + IndirectLess &operator=(IndirectLess &&) = default; +}; + +template +class SelectTest : public ::testing::Test { + public: + using Base = Selector; + + static void TestSelects(size_t N, size_t M) { + ASSERT_NE(N, 0); + ASSERT_GT(N, M); + SCOPED_TRACE(N); + SCOPED_TRACE(M); + std::vector array(N); + for (size_t i = 0; i < N; ++i) { + array[i] = i; + } + auto array_smaller = array; + std::mt19937_64 mersenne_engine; + std::shuffle(array.begin(), array.end(), mersenne_engine); + Selector::Select(array.begin(), array.begin() + M, array.end(), + std::greater()); + EXPECT_EQ(array[M], N - M - 1); + for (size_t i = 0; i < M; ++i) { + EXPECT_GE(array[i], array[M]); + } + for (size_t i = M; i < N; ++i) { + EXPECT_LE(array[i], array[M]); + } + std::shuffle(array_smaller.begin(), array_smaller.end(), mersenne_engine); + Selector::Select(array_smaller.begin(), array_smaller.begin() + M, + array_smaller.end()); + EXPECT_EQ(array_smaller[M], M); + for (size_t i = 0; i < M; ++i) { + EXPECT_LE(array_smaller[i], array_smaller[M]); + } + for (size_t i = M; i < N; ++i) { + EXPECT_GE(array_smaller[i], array_smaller[M]); + } + } + + static void TestSelects(size_t N) { + TestSelects(N, 0); + TestSelects(N, 1); + TestSelects(N, 2); + TestSelects(N, 3); + TestSelects(N, N / 2 - 1); + TestSelects(N, N / 2); + TestSelects(N, N / 2 + 1); + TestSelects(N, N - 2); + TestSelects(N, N - 1); + } + + static void TestManySelects() { + TestSelects(10); + TestSelects(256); + TestSelects(257); + TestSelects(499); + TestSelects(500); + TestSelects(997); + TestSelects(1000); + TestSelects(1000 * 100); + TestSelects(1009); + TestSelects(1009 * 109); + } + + static void TestCustomComparators() { + std::vector> v(1000); + for (int i = 0; static_cast(i) < v.size(); ++i) { + v[i] = std::make_unique(i); + } + Selector::Select(v.begin(), v.begin() + v.size() / 2, v.end(), + IndirectLess{}); + EXPECT_EQ(*v[v.size() / 2], v.size() / 2); + for (size_t i = 0; i < v.size() / 2; ++i) { + ASSERT_NE(v[i], nullptr); + EXPECT_LE(*v[i], v.size() / 2); + } + for (size_t i = v.size() / 2; i < v.size(); ++i) { + ASSERT_NE(v[i], nullptr); + EXPECT_GE(*v[i], v.size() / 2); + } + } + + static void TestRepeat(size_t N, size_t M) { + ASSERT_NE(N, 0); + ASSERT_GT(N, M); + SCOPED_TRACE(N); + SCOPED_TRACE(M); + std::mt19937_64 mersenne_engine(10); + std::vector array(N); + for (size_t i = 0; i < M; ++i) { + array[i] = false; + } + for (size_t i = M; i < N; ++i) { + array[i] = true; + } + std::shuffle(array.begin(), array.end(), mersenne_engine); + Selector::Select(array.begin(), array.begin() + M, array.end()); + EXPECT_EQ(array[M], true); + for (size_t i = 0; i < M; ++i) { + EXPECT_EQ(array[i], false); + } + for (size_t i = M; i < N; ++i) { + EXPECT_EQ(array[i], true); + } + std::shuffle(array.begin(), array.end(), mersenne_engine); + Selector::Select(array.begin(), array.begin() + M / 2, array.end()); + EXPECT_EQ(array[M / 2], false); + for (size_t i = 0; i < M / 2; ++i) { + EXPECT_EQ(array[i], false); + } + std::shuffle(array.begin(), array.end(), mersenne_engine); + Selector::Select(array.begin(), array.begin() + M - 1, array.end()); + EXPECT_EQ(array[M - 1], false); + for (size_t i = 0; i < M - 1; ++i) { + EXPECT_EQ(array[i], false); + } + } + + static void TestRepeats(size_t N) { + TestRepeat(N, 1); + TestRepeat(N, 2); + TestRepeat(N, 3); + TestRepeat(N, N / 2 - 1); + TestRepeat(N, N / 2); + TestRepeat(N, N / 2 + 1); + TestRepeat(N, N - 2); + TestRepeat(N, N - 1); + } + + static void TestManyRepeats() { + TestRepeats(10); + TestRepeats(100); + TestRepeats(257); + TestRepeats(1000); + TestRepeats(100000); + } +}; + +TYPED_TEST_SUITE(SelectTest, algorithms::All); + +TYPED_TEST(SelectTest, TestSmall) { + std::vector v = {"ab", "aaa", "ab"}; + TypeParam::Select(v.begin(), v.begin() + 1, v.end()); + EXPECT_THAT(v, Eq(std::vector{"aaa", "ab", "ab"})); + v = {"aba"}; + TypeParam::Select(v.begin(), v.begin(), v.end()); + EXPECT_THAT(v, Eq(std::vector{"aba"})); + v.clear(); + TypeParam::Select(v.begin(), v.end(), v.end()); + EXPECT_TRUE(v.empty()); +} + +TYPED_TEST(SelectTest, TestAnotherSmall) { + std::vector v = {"ab", "ab", "aaa"}; + TypeParam::Select(v.begin(), v.begin() + 1, v.end()); + EXPECT_THAT(v, Eq(std::vector{"aaa", "ab", "ab"})); +} + +TYPED_TEST(SelectTest, TestEmptySmall) { + std::vector v = {"", ""}; + TypeParam::Select(v.begin(), v.begin() + 1, v.end()); + EXPECT_THAT(v, Eq(std::vector{"", ""})); +} + +TYPED_TEST(SelectTest, TestBasic) { TestFixture::TestManySelects(); } + +TYPED_TEST(SelectTest, TestComparators) { + TestFixture::TestCustomComparators(); +} + +TYPED_TEST(SelectTest, TestRepeats) { TestFixture::TestManyRepeats(); } + +TYPED_TEST(SelectTest, TestLast) { + std::vector array(100); + for (size_t i = 0; i < 100; ++i) { + array[i] = i; + } + auto array_smaller = array; + std::mt19937_64 mersenne_engine; + std::shuffle(array.begin(), array.end(), mersenne_engine); + auto copy_array = array; + // Should be no effect. + size_t cmp = 0; + TypeParam::Select(array.begin(), array.end(), array.end(), + [&cmp](const auto &lhs, const auto &rhs) { + ++cmp; + return lhs < rhs; + }); + EXPECT_EQ(cmp, 0); + EXPECT_EQ(copy_array, array); +} + +} // namespace +} // namespace miniselect + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/contrib/miniselect/testing/test_sort.cpp b/contrib/miniselect/testing/test_sort.cpp new file mode 100644 index 00000000000..19c6ff036fe --- /dev/null +++ b/contrib/miniselect/testing/test_sort.cpp @@ -0,0 +1,161 @@ +/* Copyright Danila Kutenin, 2020-. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * https://boost.org/LICENSE_1_0.txt) + */ +#include +#include + +#include +#include +#include +#include +#include + +#include "test_common.h" + +using ::testing::Eq; + +namespace miniselect { +namespace { + +struct IndirectLess { + // Non const comparator with deleted copy. + template + bool operator()(const P &x, const P &y) const { + return *x < *y; + } + IndirectLess(const IndirectLess &) = default; + IndirectLess &operator=(const IndirectLess &) = default; + IndirectLess(IndirectLess &&) = default; + IndirectLess &operator=(IndirectLess &&) = default; +}; + +template +class PartialSortTest : public ::testing::Test { + public: + static void TestSorts(size_t N, size_t M) { + ASSERT_NE(N, 0); + ASSERT_GE(N, M); + SCOPED_TRACE(N); + SCOPED_TRACE(M); + std::vector array(N); + for (size_t i = 0; i < N; ++i) { + array[i] = i; + } + auto array_smaller = array; + std::mt19937_64 mersenne_engine; + std::shuffle(array.begin(), array.end(), mersenne_engine); + Sorter::Sort(array.begin(), array.begin() + M, array.end(), + std::greater()); + for (size_t i = 0; i < M; ++i) { + EXPECT_EQ(array[i], N - i - 1); + } + std::shuffle(array_smaller.begin(), array_smaller.end(), mersenne_engine); + Sorter::Sort(array_smaller.begin(), array_smaller.begin() + M, + array_smaller.end()); + for (size_t i = 0; i < M; ++i) { + EXPECT_EQ(array_smaller[i], i); + } + } + + static void TestSorts(size_t N) { + TestSorts(N, 0); + TestSorts(N, 1); + TestSorts(N, 2); + TestSorts(N, 3); + TestSorts(N, N / 2 - 1); + TestSorts(N, N / 2); + TestSorts(N, N / 2 + 1); + TestSorts(N, N - 2); + TestSorts(N, N - 1); + TestSorts(N, N); + } + + static void TestManySorts() { + TestSorts(10); + TestSorts(256); + TestSorts(257); + TestSorts(499); + TestSorts(500); + TestSorts(997); + TestSorts(1000); + TestSorts(1000 * 100); + TestSorts(1009); + TestSorts(1009 * 109); + } + + static void TestCustomComparators() { + std::vector> v(1000); + for (int i = 0; static_cast(i) < v.size(); ++i) { + v[i] = std::make_unique(i); + } + Sorter::Sort(v.begin(), v.begin() + v.size() / 2, v.end(), IndirectLess{}); + for (int i = 0; static_cast(i) < v.size() / 2; ++i) { + ASSERT_NE(v[i], nullptr); + EXPECT_EQ(*v[i], i); + } + } +}; + +TYPED_TEST_SUITE(PartialSortTest, algorithms::All); + +TYPED_TEST(PartialSortTest, TestSmall) { + std::vector v = {"ab", "aaa", "ab"}; + TypeParam::Sort(v.begin(), v.begin() + 1, v.end()); + EXPECT_THAT(v, Eq(std::vector{"aaa", "ab", "ab"})); + v = {"aba"}; + TypeParam::Sort(v.begin(), v.begin(), v.end()); + EXPECT_THAT(v, Eq(std::vector{"aba"})); + v.clear(); + TypeParam::Sort(v.begin(), v.end(), v.end()); + EXPECT_TRUE(v.empty()); +} + +TYPED_TEST(PartialSortTest, TestAnotherSmall) { + std::vector v = {"ab", "ab", "aaa"}; + TypeParam::Sort(v.begin(), v.begin() + 1, v.end()); + EXPECT_THAT(v, Eq(std::vector{"aaa", "ab", "ab"})); +} + +TYPED_TEST(PartialSortTest, TestEmptySmall) { + std::vector v = {"", ""}; + TypeParam::Sort(v.begin(), v.begin() + 1, v.end()); + EXPECT_THAT(v, Eq(std::vector{"", ""})); +} + +TYPED_TEST(PartialSortTest, TestBasic) { TestFixture::TestManySorts(); } + +TYPED_TEST(PartialSortTest, TestComparators) { + TestFixture::TestCustomComparators(); +} + +// The standard says that the order of other elements is unspecified even if +// nothing should be sorted so it fails for libcxx and PDQ which is Ok. Saving +// this test for a reference. +TYPED_TEST(PartialSortTest, DISABLED_TestEmpty) { + std::vector array(100); + for (size_t i = 0; i < 100; ++i) { + array[i] = i; + } + std::mt19937_64 mersenne_engine; + std::shuffle(array.begin(), array.end(), mersenne_engine); + size_t cmp = 0; + auto copy_array = array; + // Should be no effect. + TypeParam::Sort(array.begin(), array.begin(), array.end(), + [&cmp](const auto &lhs, const auto &rhs) { + ++cmp; + return lhs < rhs; + }); + EXPECT_EQ(cmp, 0); + EXPECT_EQ(copy_array, array); +} + +} // namespace +} // namespace miniselect + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 085269847e4..7b9c05a1adc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -321,6 +321,7 @@ target_include_directories(clickhouse_common_io PUBLIC ${CMAKE_CURRENT_BINARY_DI dbms_target_include_directories(PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/Core/include) dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) +dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR}) if (ZSTD_LIBRARY) dbms_target_link_libraries(PRIVATE ${ZSTD_LIBRARY}) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 9b948236943..6dbe755f0ba 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -20,6 +20,7 @@ #include #include +#include namespace DB { @@ -782,7 +783,7 @@ void ColumnArray::getPermutationImpl(size_t limit, Permutation & res, Comparator auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; }; if (limit) - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); else std::sort(res.begin(), res.end(), less); } @@ -835,7 +836,7 @@ void ColumnArray::updatePermutationImpl(size_t limit, Permutation & res, EqualRa /// Since then we are working inside the interval. - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); auto new_first = first; for (auto j = first + 1; j < limit; ++j) { diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index b9549175f6c..b27506c1cfb 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -8,6 +8,7 @@ #include #include +#include #include @@ -162,10 +163,10 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum { const auto& [first, last] = equal_ranges[i]; if (reverse) - std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; @@ -193,10 +194,10 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum /// Since then we are working inside the interval. if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index c33ab34b541..128b595803f 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -253,9 +254,9 @@ protected: sort_end = res.begin() + limit; if (reverse) - std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; }); + miniselect::floyd_rivest_partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; }); else - std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; }); + miniselect::floyd_rivest_partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; }); } }; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 0e44b83791c..41e46a7fa98 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -10,6 +10,7 @@ #include #include +#include #include @@ -157,9 +158,9 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir if (limit) { if (reverse) - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); else - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); } else { @@ -217,9 +218,9 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu /// Since then we are working inside the interval. if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); auto new_first = first; for (auto j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 3f03734b738..9b87a409aaa 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -8,6 +8,7 @@ #include #include +#include namespace DB { @@ -393,7 +394,7 @@ void ColumnLowCardinality::updatePermutationImpl(size_t limit, Permutation & res /// Since then we are working inside the interval. - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); auto new_first = first; for (auto j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 23798f64a9c..541863486a6 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -10,6 +10,7 @@ #include #include +#include namespace DB { @@ -313,7 +314,7 @@ void ColumnString::getPermutationImpl(size_t limit, Permutation & res, Comparato auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; }; if (limit) - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); else std::sort(res.begin(), res.end(), less); } @@ -365,7 +366,7 @@ void ColumnString::updatePermutationImpl(size_t limit, Permutation & res, EqualR /// Since then we are working inside the interval. - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); size_t new_first = first; for (size_t j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index d6e1ca982d6..cbe5c7e11cd 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -9,7 +9,7 @@ #include #include #include - +#include namespace DB { @@ -352,7 +352,7 @@ void ColumnTuple::getPermutationImpl(size_t limit, Permutation & res, LessOperat if (limit) { - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); } else { diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 733a1510f93..e9af38d6984 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -17,7 +17,7 @@ #include #include #include - +#include #ifdef __SSE2__ #include @@ -156,9 +156,9 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi res[i] = i; if (reverse) - std::partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint)); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint)); else - std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint)); + miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint)); } else { @@ -254,9 +254,9 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire /// Since then, we are working inside the interval. if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint)); + miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint)); size_t new_first = first; for (size_t j = first + 1; j < limit; ++j) From 8c6471f7c3ac3d0a89b948a2b154aba207992013 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Mon, 9 Nov 2020 21:37:05 +0300 Subject: [PATCH 068/114] Retry CI From 14b4c19919da77177e78029530837946fff4811e Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Mon, 9 Nov 2020 20:00:00 +0100 Subject: [PATCH 069/114] Create adding_test_queries.md --- docs/en/development/adding_test_queries.md | 141 +++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 docs/en/development/adding_test_queries.md diff --git a/docs/en/development/adding_test_queries.md b/docs/en/development/adding_test_queries.md new file mode 100644 index 00000000000..710f6467ce7 --- /dev/null +++ b/docs/en/development/adding_test_queries.md @@ -0,0 +1,141 @@ +# How to add test queries to ClickHouse CI + +ClickHouse has hundreds (or even thousands) of features. Every commit get checked by a complex set of tests containing many thousands of test cases. + +The core functionality is very well tested, but some corner-cases and different combinations of features can be uncovered with ClickHouse CI. + +Most of the bugs/regressions we see happen in that 'grey area' where test coverage is poor. + +And we are very interested in covering most of the possible scenarios and feature combinations used in real life by tests. + +## Why adding tests + +Why/when you should add a test case into ClickHouse code: +1) you use some complicated scenarios / feature combinations / you have some corner case which is probably not widely used +2) you see that certain behavior gets changed between version w/o notifications in the changelog +3) you just want to help to improve ClickHouse quality and ensure the features you use will not be broken in the future releases +4) once the test is added/accepted, you can be sure the corner case you check will never be accidentally broken. +5) you will be a part of great open-source community +6) your name will be visible in the `system.contributors` table! +7) you will make a world bit better :) + +### Steps to do + +#### Prerequisite + +I assume you run some Linux machine (you can use docker / virtual machines on other OS) and any modern browser / internet connection, and you have some basic Linux & SQL skills. + +Any highly specialized knowledge are not needed (so you don't need to know C++ or know something about how ClickHouse CI works). + + +#### Preparation + +1) [create GitHub account](https://github.com/join) (if you haven't one yet) +2) [setup git](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/set-up-git) +```bash +# for Ubuntu +sudo apt-get update +sudo apt-get install git + +git config --global user.name "John Doe" # fill with your name +git config --global user.email "email@example.com" # fill with your email + +``` +3) [fork ClickHouse project](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/fork-a-repo) - just open [https://github.com/ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse) and press fork button in the top right corner: +![fork repo](https://github-images.s3.amazonaws.com/help/bootcamp/Bootcamp-Fork.png) + +4) clone your fork to some folder on your PC, for example, `~/workspace/ClickHouse` +``` +mkdir ~/workspace && cd ~/workspace +git clone https://github.com/< your GitHub username>/ClickHouse +cd ClickHouse +git remote add upstream https://github.com/ClickHouse/ClickHouse +``` + +#### New branch for the test + +1) create a new branch from the latest clickhouse master +``` +cd ~/workspace/ClickHouse +git fetch upstream +git checkout -b name_for_a_branch_with_my_test upstream/master +``` + +#### Install & run clickhouse + +1) install `clickhouse-server` (follow [official docs](https://clickhouse.tech/docs/en/getting-started/install/)) +2) install test configurations (it will use Zookeeper mock implementation and adjust some settings) +``` +cd ~/workspace/ClickHouse/tests/config +sudo ./install.sh +``` +3) run clickhouse-server +``` +sudo systemctl restart clickhouse-server +``` + +#### Creating the test file + + +1) find the number for your test - find the file with the biggest number in `tests/queries/0_stateless/` + +```sh +$ cd ~/workspace/ClickHouse +$ ls tests/queries/0_stateless/[0-9]*.reference | tail -n 1 +tests/queries/0_stateless/01520_client_print_query_id.reference +``` +Currently, the last number for the test is `01520`, so my test will have the number `01521` + +2) create an SQL file with the next number and name of the feature you test + +```sh +touch tests/queries/0_stateless/01521_dummy_test.sql +``` + +3) edit SQL file with your favorite editor (see hint of creating tests below) +```sh +vim tests/queries/0_stateless/01521_dummy_test.sql +``` + + +4) run the test, and put the result of that into the reference file: +``` +clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee tests/queries/0_stateless/01521_dummy_test.reference +``` + +5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor. + +#### How create good test + +- test should be + - minimal - create only tables related to tested functionality, remove unrelated columns and parts of query + - fast - should not take longer than few seconds (better subseconds) + - correct - fails then feature is not working + - deteministic + - isolated / stateless + - don't rely on some environment things + - don't rely on timing when possible +- try to cover corner cases (zeros / Nulls / empty sets / throwing exceptions) +- to test that query return errors, you can put special comment after the query: `-- { serverError 60 }` or `-- { clientError 20 }` +- don't switch databases (unless necessary) +- you can create several table replicas on the same node if needed +- you can use one of the test cluster definitions when needed (see system.clusters) +- use `number` / `numbers_mt` / `zeros` / `zeros_mt` and similar for queries / to initialize data when appliable +- clean up the created objects after test and before the test (DROP IF EXISTS) - in case of some dirty state +- prefer sync mode of operations (mutations, merges, etc.) +- use other SQL files in the `0_stateless` folder as a reference +- ensure the feature / feature combination you want to tests is not covered yet with existsing tests + +#### Commit / push / create PR. + +1) commit & push your changes +```sh +cd ~/workspace/ClickHouse +git add tests/queries/0_stateless/01521_dummy_test.sql +git add tests/queries/0_stateless/01521_dummy_test.reference +git commit # use some nice commit message when possible +git push origin HEAD +``` +2) use a link which was shown during the push, to create a PR into the main repo +3) adjust the PR title and contents, in `Changelog category (leave one)` keep +`Build/Testing/Packaging Improvement`, fill the rest of the fields if you want. From 5cdfcfb3077884357b8eed9b8a11b856b81b54e7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 9 Nov 2020 22:07:38 +0300 Subject: [PATCH 070/114] remove other stringstreams --- base/mysqlxx/Query.h | 2 +- programs/client/Client.cpp | 7 ++-- programs/format/Format.cpp | 1 + src/Access/DiskAccessStorage.cpp | 2 +- src/Access/LDAPAccessStorage.cpp | 2 +- src/Access/UsersConfigAccessStorage.cpp | 2 +- src/Common/HTMLForm.h | 4 +- src/Common/ThreadProfileEvents.cpp | 3 +- src/Common/UInt128.h | 8 ++-- src/Common/parseGlobs.cpp | 33 +++++++-------- .../gtest_getMultipleValuesFromConfig.cpp | 3 +- .../tests/gtest_sensitive_data_masker.cpp | 12 ++++-- .../tests/gtest_data_type_get_common_type.cpp | 2 +- src/Dictionaries/DictionaryStructure.cpp | 4 +- .../tests/gtest_dictionary_configuration.cpp | 2 +- src/IO/WriteHelpers.cpp | 8 ++++ src/IO/WriteHelpers.h | 2 + src/Parsers/IAST.cpp | 10 +++++ src/Parsers/IAST.h | 8 +--- .../MySQL/tests/gtest_create_parser.cpp | 4 +- src/Parsers/iostream_debug_helpers.cpp | 9 +++-- src/Parsers/ya.make | 1 - .../Formats/Impl/AvroRowInputFormat.cpp | 2 +- src/Storages/MergeTree/KeyCondition.cpp | 40 +++++++++---------- src/Storages/MergeTree/MergeTreeData.cpp | 40 +++++++++---------- src/Storages/StorageDistributed.cpp | 16 ++++---- src/Storages/StorageS3.cpp | 16 +++----- src/Storages/System/StorageSystemUsers.cpp | 2 +- src/Storages/tests/gtest_storage_log.cpp | 6 +-- utils/check-style/check-style | 2 +- 30 files changed, 136 insertions(+), 117 deletions(-) diff --git a/base/mysqlxx/Query.h b/base/mysqlxx/Query.h index 1d3ab9678d5..d0a905e2031 100644 --- a/base/mysqlxx/Query.h +++ b/base/mysqlxx/Query.h @@ -77,7 +77,7 @@ public: private: Connection * conn; - std::ostringstream query_buf; + std::ostringstream query_buf; // STYLE_CHECK_ALLOW_STD_STRING_STREAM void executeImpl(); }; diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index f9e4b357aa9..98c1463ab23 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1159,13 +1159,13 @@ private: ASTPtr ast_to_process; try { - std::stringstream dump_before_fuzz; + WriteBufferFromOwnString dump_before_fuzz; fuzz_base->dumpTree(dump_before_fuzz); auto base_before_fuzz = fuzz_base->formatForErrorMessage(); ast_to_process = fuzz_base->clone(); - std::stringstream dump_of_cloned_ast; + WriteBufferFromOwnString dump_of_cloned_ast; ast_to_process->dumpTree(dump_of_cloned_ast); // Run the original query as well. @@ -1187,7 +1187,8 @@ private: fprintf(stderr, "dump of cloned ast:\n%s\n", dump_of_cloned_ast.str().c_str()); fprintf(stderr, "dump after fuzz:\n"); - fuzz_base->dumpTree(std::cerr); + WriteBufferFromOStream cerr_buf(std::cerr, 4096); + fuzz_base->dumpTree(cerr_buf); fmt::print(stderr, "IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly."); diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index c9981e706b0..986dc67a798 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -132,6 +132,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) { WriteBufferFromOStream res_buf(std::cout, 4096); formatAST(*res, res_buf, hilite, oneline); + res_buf.next(); if (multiple) std::cout << "\n;\n"; std::cout << std::endl; diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index b03a2791560..426c27ea799 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -355,7 +355,7 @@ String DiskAccessStorage::getStorageParamsJSON() const json.set("path", directory_path); if (readonly) json.set("readonly", readonly.load()); - std::ostringstream oss; + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(json, oss); return oss.str(); diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index c9f00f2f4ab..92de7fce8d7 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -150,7 +150,7 @@ String LDAPAccessStorage::getStorageParamsJSON() const params_json.set("server", ldap_server); params_json.set("roles", default_role_names); - std::ostringstream oss; + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(params_json, oss); diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index ba3c2d5b8e5..eb993d696c6 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -460,7 +460,7 @@ String UsersConfigAccessStorage::getStorageParamsJSON() const Poco::JSON::Object json; if (!path.empty()) json.set("path", path); - std::ostringstream oss; + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(json, oss); return oss.str(); diff --git a/src/Common/HTMLForm.h b/src/Common/HTMLForm.h index 2490d613160..2b62167dce7 100644 --- a/src/Common/HTMLForm.h +++ b/src/Common/HTMLForm.h @@ -16,13 +16,13 @@ struct HTMLForm : public Poco::Net::HTMLForm HTMLForm(const Poco::Net::HTTPRequest & request) { Poco::URI uri(request.getURI()); - std::istringstream istr(uri.getRawQuery()); + std::istringstream istr(uri.getRawQuery()); // STYLE_CHECK_ALLOW_STD_STRING_STREAM readUrl(istr); } HTMLForm(const Poco::URI & uri) { - std::istringstream istr(uri.getRawQuery()); + std::istringstream istr(uri.getRawQuery()); // STYLE_CHECK_ALLOW_STD_STRING_STREAM readUrl(istr); } diff --git a/src/Common/ThreadProfileEvents.cpp b/src/Common/ThreadProfileEvents.cpp index 7b94ca0f2b2..c8a4555dabf 100644 --- a/src/Common/ThreadProfileEvents.cpp +++ b/src/Common/ThreadProfileEvents.cpp @@ -413,7 +413,8 @@ std::vector PerfEventsCounters::eventIndicesFromString(const std::string return result; } - std::istringstream iss(events_list); + + std::istringstream iss(events_list); // STYLE_CHECK_ALLOW_STD_STRING_STREAM std::string event_name; while (std::getline(iss, event_name, ',')) { diff --git a/src/Common/UInt128.h b/src/Common/UInt128.h index 7b6f8e7c7be..735b287f90f 100644 --- a/src/Common/UInt128.h +++ b/src/Common/UInt128.h @@ -6,6 +6,7 @@ #include #include +#include #ifdef __SSE4_2__ #include @@ -48,10 +49,9 @@ struct UInt128 String toHexString() const { - std::ostringstream os; - os.exceptions(std::ios::failbit); - os << std::setw(16) << std::setfill('0') << std::hex << high << low; - return String(os.str()); + String res(2 * sizeof(UInt128), 0); + writeHexUIntLowercase(*this, res.data()); + return res; } bool inline operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); } diff --git a/src/Common/parseGlobs.cpp b/src/Common/parseGlobs.cpp index f04ad1ea8a0..75539512b6d 100644 --- a/src/Common/parseGlobs.cpp +++ b/src/Common/parseGlobs.cpp @@ -1,4 +1,7 @@ #include +#include +#include +#include #include #include #include @@ -18,21 +21,21 @@ namespace DB */ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_globs) { - std::ostringstream oss_for_escaping; - oss_for_escaping.exceptions(std::ios::failbit); + /// FIXME make it better + WriteBufferFromOwnString buf_for_escaping; /// Escaping only characters that not used in glob syntax for (const auto & letter : initial_str_with_globs) { - if ((letter == '[') || (letter == ']') || (letter == '|') || (letter == '+') || (letter == '-') || (letter == '(') || (letter == ')')) - oss_for_escaping << '\\'; - oss_for_escaping << letter; + if ((letter == '[') || (letter == ']') || (letter == '|') || (letter == '+') || (letter == '-') || (letter == '(') || (letter == ')') || (letter == '\\')) + buf_for_escaping << '\\'; + buf_for_escaping << letter; } - std::string escaped_with_globs = oss_for_escaping.str(); + std::string escaped_with_globs = buf_for_escaping.str(); static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})"); /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without {}*, re2::StringPiece input(escaped_with_globs); re2::StringPiece matched; - std::ostringstream oss_for_replacing; + std::ostringstream oss_for_replacing; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss_for_replacing.exceptions(std::ios::failbit); size_t current_index = 0; while (RE2::FindAndConsume(&input, enum_or_range, &matched)) @@ -45,9 +48,8 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob size_t range_begin = 0; size_t range_end = 0; char point; - std::istringstream iss_range(buffer); - iss_range.exceptions(std::ios::failbit); - iss_range >> range_begin >> point >> point >> range_end; + ReadBufferFromString buf_range(buffer); + buf_range >> range_begin >> point >> point >> range_end; bool leading_zeros = buffer[0] == '0'; size_t num_len = std::to_string(range_end).size(); if (leading_zeros) @@ -71,20 +73,19 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob } oss_for_replacing << escaped_with_globs.substr(current_index); std::string almost_res = oss_for_replacing.str(); - std::ostringstream oss_final_processing; - oss_final_processing.exceptions(std::ios::failbit); + WriteBufferFromOwnString buf_final_processing; for (const auto & letter : almost_res) { if ((letter == '?') || (letter == '*')) { - oss_final_processing << "[^/]"; /// '?' is any symbol except '/' + buf_final_processing << "[^/]"; /// '?' is any symbol except '/' if (letter == '?') continue; } if ((letter == '.') || (letter == '{') || (letter == '}')) - oss_final_processing << '\\'; - oss_final_processing << letter; + buf_final_processing << '\\'; + buf_final_processing << letter; } - return oss_final_processing.str(); + return buf_final_processing.str(); } } diff --git a/src/Common/tests/gtest_getMultipleValuesFromConfig.cpp b/src/Common/tests/gtest_getMultipleValuesFromConfig.cpp index 4756043acbf..30aba9418ba 100644 --- a/src/Common/tests/gtest_getMultipleValuesFromConfig.cpp +++ b/src/Common/tests/gtest_getMultipleValuesFromConfig.cpp @@ -9,7 +9,8 @@ using namespace DB; TEST(Common, getMultipleValuesFromConfig) { - std::istringstream xml_isteam(R"END( + std::istringstream // STYLE_CHECK_ALLOW_STD_STRING_STREAM + xml_isteam(R"END( 0 diff --git a/src/Common/tests/gtest_sensitive_data_masker.cpp b/src/Common/tests/gtest_sensitive_data_masker.cpp index 67ad5be2f52..b8125c77b9b 100644 --- a/src/Common/tests/gtest_sensitive_data_masker.cpp +++ b/src/Common/tests/gtest_sensitive_data_masker.cpp @@ -102,7 +102,8 @@ TEST(Common, SensitiveDataMasker) EXPECT_EQ(maskerbad.wipeSensitiveData(x), 0); { - std::istringstream xml_isteam(R"END( + std::istringstream // STYLE_CHECK_ALLOW_STD_STRING_STREAM + xml_isteam(R"END( @@ -152,7 +153,8 @@ TEST(Common, SensitiveDataMasker) try { - std::istringstream xml_isteam_bad(R"END( + std::istringstream // STYLE_CHECK_ALLOW_STD_STRING_STREAM + xml_isteam_bad(R"END( @@ -181,7 +183,8 @@ TEST(Common, SensitiveDataMasker) try { - std::istringstream xml_isteam_bad(R"END( + std::istringstream // STYLE_CHECK_ALLOW_STD_STRING_STREAM + xml_isteam_bad(R"END( test @@ -203,7 +206,8 @@ TEST(Common, SensitiveDataMasker) try { - std::istringstream xml_isteam_bad(R"END( + std::istringstream // STYLE_CHECK_ALLOW_STD_STRING_STREAM + xml_isteam_bad(R"END( test())( diff --git a/src/DataTypes/tests/gtest_data_type_get_common_type.cpp b/src/DataTypes/tests/gtest_data_type_get_common_type.cpp index 8212555e8bc..5c258f0b42e 100644 --- a/src/DataTypes/tests/gtest_data_type_get_common_type.cpp +++ b/src/DataTypes/tests/gtest_data_type_get_common_type.cpp @@ -26,7 +26,7 @@ static auto typeFromString(const std::string & str) static auto typesFromString(const std::string & str) { - std::istringstream data_types_stream(str); + std::istringstream data_types_stream(str); // STYLE_CHECK_ALLOW_STD_STRING_STREAM DataTypes data_types; std::string data_type; while (data_types_stream >> data_type) diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index 4c7cc5b4118..44924b69363 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -230,8 +231,7 @@ std::string DictionaryStructure::getKeyDescription() const if (id) return "UInt64"; - std::ostringstream out; - out.exceptions(std::ios::failbit); + WriteBufferFromOwnString out; out << '('; diff --git a/src/Dictionaries/tests/gtest_dictionary_configuration.cpp b/src/Dictionaries/tests/gtest_dictionary_configuration.cpp index 62422124bd8..be0b0d7c6d2 100644 --- a/src/Dictionaries/tests/gtest_dictionary_configuration.cpp +++ b/src/Dictionaries/tests/gtest_dictionary_configuration.cpp @@ -18,7 +18,7 @@ static bool registered = false; static std::string configurationToString(const DictionaryConfigurationPtr & config) { const Poco::Util::XMLConfiguration * xml_config = dynamic_cast(config.get()); - std::ostringstream oss; + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); xml_config->save(oss); return oss.str(); diff --git a/src/IO/WriteHelpers.cpp b/src/IO/WriteHelpers.cpp index a0a2a45c791..61bfc281050 100644 --- a/src/IO/WriteHelpers.cpp +++ b/src/IO/WriteHelpers.cpp @@ -89,4 +89,12 @@ void writeProbablyBackQuotedStringMySQL(const StringRef & s, WriteBuffer & buf) writeProbablyQuotedStringImpl(s, buf, [](const StringRef & s_, WriteBuffer & buf_) { return writeBackQuotedStringMySQL(s_, buf_); }); } +void writePointerHex(const void * ptr, WriteBuffer & buf) +{ + writeString("0x", buf); + char hex_str[2 * sizeof(ptr)]; + writeHexUIntLowercase(reinterpret_cast(ptr), hex_str); + buf.write(hex_str, 2 * sizeof(ptr)); +} + } diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index da68c4aded5..4a117fefbee 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -1110,4 +1110,6 @@ struct PcgSerializer } }; +void writePointerHex(const void * ptr, WriteBuffer & buf); + } diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index 4b7d3c2ea40..2a74c484187 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -148,4 +148,14 @@ void IAST::FormatSettings::writeIdentifier(const String & name) const } } +void IAST::dumpTree(WriteBuffer & ostr, size_t indent) const +{ + String indent_str(indent, '-'); + ostr << indent_str << getID() << ", "; + writePointerHex(this, ostr); + writeChar('\n', ostr); + for (const auto & child : children) + child->dumpTree(ostr, indent + 1); +} + } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index d9fd71378f7..9428f312106 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -78,13 +78,7 @@ public: void updateTreeHash(SipHash & hash_state) const; virtual void updateTreeHashImpl(SipHash & hash_state) const; - void dumpTree(std::ostream & ostr, size_t indent = 0) const - { - String indent_str(indent, '-'); - ostr << indent_str << getID() << ", " << this << std::endl; - for (const auto & child : children) - child->dumpTree(ostr, indent + 1); - } + void dumpTree(WriteBuffer & ostr, size_t indent = 0) const; /** Check the depth of the tree. * If max_depth is specified and the depth is greater - throw an exception. diff --git a/src/Parsers/MySQL/tests/gtest_create_parser.cpp b/src/Parsers/MySQL/tests/gtest_create_parser.cpp index 1aaba8d67e4..554b3f0a67d 100644 --- a/src/Parsers/MySQL/tests/gtest_create_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_create_parser.cpp @@ -6,6 +6,7 @@ #include #include #include +#include using namespace DB; using namespace DB::MySQLParser; @@ -37,6 +38,7 @@ TEST(CreateTableParser, SS) ParserCreateQuery p_create_query; String input = "CREATE TABLE `test_table_1` (`a` int DEFAULT NULL, `b` int DEFAULT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci"; ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); - ast->dumpTree(std::cerr); + WriteBufferFromOStream buf(std::cerr, 4096); + ast->dumpTree(buf); } diff --git a/src/Parsers/iostream_debug_helpers.cpp b/src/Parsers/iostream_debug_helpers.cpp index 66666f0dbfc..b74d337b22d 100644 --- a/src/Parsers/iostream_debug_helpers.cpp +++ b/src/Parsers/iostream_debug_helpers.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace DB { @@ -23,9 +25,10 @@ std::ostream & operator<<(std::ostream & stream, const Expected & what) std::ostream & operator<<(std::ostream & stream, const IAST & what) { - stream << "IAST{"; - what.dumpTree(stream); - stream << "}"; + WriteBufferFromOStream buf(stream, 4096); + buf << "IAST{"; + what.dumpTree(buf); + buf << "}"; return stream; } diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 3fd173d8e18..0bef6699266 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -52,7 +52,6 @@ SRCS( ASTShowAccessEntitiesQuery.cpp ASTShowCreateAccessEntityQuery.cpp ASTShowGrantsQuery.cpp - ASTShowPrivilegesQuery.cpp ASTShowTablesQuery.cpp ASTSubquery.cpp ASTSystemQuery.cpp diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 8a416ade740..787059ddcc8 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -160,7 +160,7 @@ static void insertNumber(IColumn & column, WhichDataType type, T value) static std::string nodeToJson(avro::NodePtr root_node) { - std::ostringstream ss; + std::ostringstream ss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM ss.exceptions(std::ios::failbit); root_node->printJson(ss, 0); return ss.str(); diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 4caeafc093f..78e43bad73a 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include @@ -33,8 +35,7 @@ namespace ErrorCodes String Range::toString() const { - std::stringstream str; - str.exceptions(std::ios::failbit); + WriteBufferFromOwnString str; if (!left_bounded) str << "(-inf, "; @@ -1559,19 +1560,18 @@ bool KeyCondition::mayBeTrueAfter( String KeyCondition::RPNElement::toString() const { - auto print_wrapped_column = [this](std::ostringstream & ss) + auto print_wrapped_column = [this](WriteBuffer & buf) { for (auto it = monotonic_functions_chain.rbegin(); it != monotonic_functions_chain.rend(); ++it) - ss << (*it)->getName() << "("; + buf << (*it)->getName() << "("; - ss << "column " << key_column; + buf << "column " << key_column; for (auto it = monotonic_functions_chain.rbegin(); it != monotonic_functions_chain.rend(); ++it) - ss << ")"; + buf << ")"; }; - std::ostringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString buf; switch (function) { case FUNCTION_AND: @@ -1585,24 +1585,24 @@ String KeyCondition::RPNElement::toString() const case FUNCTION_NOT_IN_SET: case FUNCTION_IN_SET: { - ss << "("; - print_wrapped_column(ss); - ss << (function == FUNCTION_IN_SET ? " in " : " notIn "); + buf << "("; + print_wrapped_column(buf); + buf << (function == FUNCTION_IN_SET ? " in " : " notIn "); if (!set_index) - ss << "unknown size set"; + buf << "unknown size set"; else - ss << set_index->size() << "-element set"; - ss << ")"; - return ss.str(); + buf << set_index->size() << "-element set"; + buf << ")"; + return buf.str(); } case FUNCTION_IN_RANGE: case FUNCTION_NOT_IN_RANGE: { - ss << "("; - print_wrapped_column(ss); - ss << (function == FUNCTION_NOT_IN_RANGE ? " not" : "") << " in " << range.toString(); - ss << ")"; - return ss.str(); + buf << "("; + print_wrapped_column(buf); + buf << (function == FUNCTION_NOT_IN_RANGE ? " not" : "") << " in " << range.toString(); + buf << ")"; + return buf.str(); } case ALWAYS_FALSE: return "false"; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 7884dc7beaa..062a0611501 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3205,13 +3205,12 @@ void MergeTreeData::Transaction::rollbackPartsToTemporaryState() { if (!isEmpty()) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << " Rollbacking parts state to temporary and removing from working set:"; + WriteBufferFromOwnString buf; + buf << " Rollbacking parts state to temporary and removing from working set:"; for (const auto & part : precommitted_parts) - ss << " " << part->relative_path; - ss << "."; - LOG_DEBUG(data.log, "Undoing transaction.{}", ss.str()); + buf << " " << part->relative_path; + buf << "."; + LOG_DEBUG(data.log, "Undoing transaction.{}", buf.str()); data.removePartsFromWorkingSetImmediatelyAndSetTemporaryState( DataPartsVector(precommitted_parts.begin(), precommitted_parts.end())); @@ -3224,13 +3223,12 @@ void MergeTreeData::Transaction::rollback() { if (!isEmpty()) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << " Removing parts:"; + WriteBufferFromOwnString buf; + buf << " Removing parts:"; for (const auto & part : precommitted_parts) - ss << " " << part->relative_path; - ss << "."; - LOG_DEBUG(data.log, "Undoing transaction.{}", ss.str()); + buf << " " << part->relative_path; + buf << "."; + LOG_DEBUG(data.log, "Undoing transaction.{}", buf.str()); data.removePartsFromWorkingSet( DataPartsVector(precommitted_parts.begin(), precommitted_parts.end()), @@ -3760,15 +3758,15 @@ bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, S if (out_reason && (settings.min_rows_for_wide_part != 0 || settings.min_bytes_for_wide_part != 0 || settings.min_rows_for_compact_part != 0 || settings.min_bytes_for_compact_part != 0)) { - std::ostringstream message; - message.exceptions(std::ios::failbit); - message << "Table can't create parts with adaptive granularity, but settings" - << " min_rows_for_wide_part = " << settings.min_rows_for_wide_part - << ", min_bytes_for_wide_part = " << settings.min_bytes_for_wide_part - << ", min_rows_for_compact_part = " << settings.min_rows_for_compact_part - << ", min_bytes_for_compact_part = " << settings.min_bytes_for_compact_part - << ". Parts with non-adaptive granularity can be stored only in Wide (default) format."; - *out_reason = message.str(); + *out_reason = fmt::format( + "Table can't create parts with adaptive granularity, but settings" + " min_rows_for_wide_part = {}" + ", min_bytes_for_wide_part = {}" + ", min_rows_for_compact_part = {}" + ", min_bytes_for_compact_part = {}" + ". Parts with non-adaptive granularity can be stored only in Wide (default) format.", + settings.min_rows_for_wide_part, settings.min_bytes_for_wide_part, + settings.min_rows_for_compact_part, settings.min_bytes_for_compact_part); } return false; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 9316a5742c1..0a266433ebc 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -49,6 +49,8 @@ #include #include +#include +#include #include @@ -175,19 +177,18 @@ UInt64 getMaximumFileNumber(const std::string & dir_path) std::string makeFormattedListOfShards(const ClusterPtr & cluster) { - std::ostringstream os; - os.exceptions(std::ios::failbit); + WriteBufferFromOwnString buf; bool head = true; - os << "["; + buf << "["; for (const auto & shard_info : cluster->getShardsInfo()) { - (head ? os : os << ", ") << shard_info.shard_num; + (head ? buf : buf << ", ") << shard_info.shard_num; head = false; } - os << "]"; + buf << "]"; - return os.str(); + return buf.str(); } ExpressionActionsPtr buildShardingKeyExpression(const ASTPtr & sharding_key, const Context & context, const NamesAndTypesList & columns, bool project) @@ -744,8 +745,7 @@ ClusterPtr StorageDistributed::getOptimizedCluster(const Context & context, cons UInt64 force = settings.force_optimize_skip_unused_shards; if (force) { - std::stringstream exception_message; - exception_message.exceptions(std::ios::failbit); + WriteBufferFromOwnString exception_message; if (!has_sharding_key) exception_message << "No sharding key"; else if (!sharding_key_is_usable) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index ce9ebbd53b3..66c688f3195 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -254,18 +254,14 @@ Strings listFilesWithRegexpMatching(Aws::S3::S3Client & client, const S3::URI & outcome = client.ListObjectsV2(request); if (!outcome.IsSuccess()) { - std::ostringstream message; - message.exceptions(std::ios::failbit); - message << "Could not list objects in bucket " << quoteString(request.GetBucket()) - << " with prefix " << quoteString(request.GetPrefix()); - if (page > 1) - message << ", page " << std::to_string(page); + throw Exception(ErrorCodes::S3_ERROR, "Could not list objects in bucket {} with prefix {}, page {}, S3 exception: {}, message: {}", + quoteString(request.GetBucket()), quoteString(request.GetPrefix()), page, + backQuote(outcome.GetError().GetExceptionName()), quoteString(outcome.GetError().GetMessage())); - message << ", S3 exception: " + backQuote(outcome.GetError().GetExceptionName()) - << ", message: " + quoteString(outcome.GetError().GetMessage()); - - throw Exception(message.str(), ErrorCodes::S3_ERROR); + throw Exception(ErrorCodes::S3_ERROR, "Could not list objects in bucket {} with prefix {}, S3 exception: {}, message: {}", + quoteString(request.GetBucket()), quoteString(request.GetPrefix()), + backQuote(outcome.GetError().GetExceptionName()), quoteString(outcome.GetError().GetMessage())); } for (const auto & row : outcome.GetResult().GetContents()) diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index 70c67683b25..675fee84746 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -96,7 +96,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & auth_params_json.set("server", authentication.getServerName()); - std::ostringstream oss; + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(auth_params_json, oss); const auto str = oss.str(); diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 821fbc4b279..0fb418e8413 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -133,16 +133,14 @@ std::string readData(DB::StoragePtr & table, const DB::Context & context) tryRegisterFormats(); - std::ostringstream ss; - ss.exceptions(std::ios::failbit); - WriteBufferFromOStream out_buf(ss); + WriteBufferFromOwnString out_buf; BlockOutputStreamPtr output = FormatFactory::instance().getOutput("Values", out_buf, sample, context); copyData(*in, *output); output->flush(); - return ss.str(); + return out_buf.str(); } TYPED_TEST(StorageLogTest, testReadWrite) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 69d948c87fe..65ed4cec67a 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -107,4 +107,4 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P ' $' | grep -P '.' && echo "^ Trailing whitespaces." # Forbid stringstream because it's easy to use them incorrectly and hard to debug possible issues -find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep 'std::ostringstream\|std::istringstream' && echo "Use WriteBufferFromString or ReadBufferFromString instead of std::ostringstream or std::istringstream" +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep 'std::ostringstream\|std::istringstream' | grep -v "STYLE_CHECK_ALLOW_STD_STRING_STREAM" && echo "Use WriteBufferFromString or ReadBufferFromString instead of std::ostringstream or std::istringstream" From fdce810237a3cab731ebbd4a4b9af73cfd06f9d7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 9 Nov 2020 22:32:18 +0300 Subject: [PATCH 071/114] Add setting back. --- src/Core/Settings.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index c8521d2f91b..580756361b1 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -399,6 +399,7 @@ class IColumn; \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ + M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing. Will be removed after 2021-03-31", 0) \ M(Bool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) \ M(Bool, force_optimize_skip_unused_shards_no_nested, false, "Obsolete setting, does nothing. Will be removed after 2020-12-01. Use force_optimize_skip_unused_shards_nesting instead.", 0) \ From f7f7fde3a8d4707accf8477c96c4d2925f33923a Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Mon, 9 Nov 2020 22:48:39 +0300 Subject: [PATCH 072/114] Fix fast test to see all others --- .../0_stateless/01525_select_with_offset_fetch_clause.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference index 709116b4746..422a076b0cb 100644 --- a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference +++ b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference @@ -3,7 +3,7 @@ 5 1 1 2 1 -3 4 +3 3 1 1 2 1 3 4 From d0840bfc5db83a8fe3a459c3ea30a7a7b374c116 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 00:53:43 +0300 Subject: [PATCH 073/114] Move miniselect to submodule and replace quantile exact with better algorithm --- .gitmodules | 3 + contrib/miniselect | 1 + contrib/miniselect/.clang-format | 1 - contrib/miniselect/.gitignore | 100 -- contrib/miniselect/.travis.yml | 140 --- contrib/miniselect/AUTHORS | 2 - contrib/miniselect/CMakeLists.txt | 52 - contrib/miniselect/CONTRIBUTORS | 1 - contrib/miniselect/LICENSE_1_0.txt | 23 - contrib/miniselect/README.md | 272 ----- contrib/miniselect/benches/bench_common.h | 170 ---- .../miniselect/benches/benchmark_select.cpp | 46 - contrib/miniselect/benches/benchmark_sort.cpp | 46 - contrib/miniselect/examples/example.cpp | 18 - contrib/miniselect/fuzz/CMakeLists.txt | 38 - .../miniselect/fuzz/build_like_oss_fuzz.sh | 22 - contrib/miniselect/fuzz/fuzz_select.cpp | 66 -- contrib/miniselect/fuzz/fuzz_sort.cpp | 69 -- .../miniselect/fuzz/fuzz_string_select.cpp | 70 -- contrib/miniselect/fuzz/fuzz_string_sort.cpp | 73 -- contrib/miniselect/fuzz/main.cpp | 22 - contrib/miniselect/fuzz/ossfuzz.sh | 23 - .../include/miniselect/floyd_rivest_select.h | 120 --- .../include/miniselect/median_of_3_random.h | 69 -- .../include/miniselect/median_of_medians.h | 71 -- .../include/miniselect/median_of_ninthers.h | 190 ---- .../miniselect/include/miniselect/pdqselect.h | 935 ------------------ .../miniselect/private/median_common.h | 437 -------- contrib/miniselect/testing/test_common.h | 180 ---- contrib/miniselect/testing/test_select.cpp | 231 ----- contrib/miniselect/testing/test_sort.cpp | 161 --- src/AggregateFunctions/QuantileExact.h | 13 +- src/AggregateFunctions/QuantileTiming.h | 5 +- src/Columns/ColumnDecimal.cpp | 4 +- 34 files changed, 16 insertions(+), 3658 deletions(-) create mode 160000 contrib/miniselect delete mode 100644 contrib/miniselect/.clang-format delete mode 100644 contrib/miniselect/.gitignore delete mode 100644 contrib/miniselect/.travis.yml delete mode 100644 contrib/miniselect/AUTHORS delete mode 100644 contrib/miniselect/CMakeLists.txt delete mode 100644 contrib/miniselect/CONTRIBUTORS delete mode 100644 contrib/miniselect/LICENSE_1_0.txt delete mode 100644 contrib/miniselect/README.md delete mode 100644 contrib/miniselect/benches/bench_common.h delete mode 100644 contrib/miniselect/benches/benchmark_select.cpp delete mode 100644 contrib/miniselect/benches/benchmark_sort.cpp delete mode 100644 contrib/miniselect/examples/example.cpp delete mode 100644 contrib/miniselect/fuzz/CMakeLists.txt delete mode 100755 contrib/miniselect/fuzz/build_like_oss_fuzz.sh delete mode 100644 contrib/miniselect/fuzz/fuzz_select.cpp delete mode 100644 contrib/miniselect/fuzz/fuzz_sort.cpp delete mode 100644 contrib/miniselect/fuzz/fuzz_string_select.cpp delete mode 100644 contrib/miniselect/fuzz/fuzz_string_sort.cpp delete mode 100644 contrib/miniselect/fuzz/main.cpp delete mode 100755 contrib/miniselect/fuzz/ossfuzz.sh delete mode 100644 contrib/miniselect/include/miniselect/floyd_rivest_select.h delete mode 100644 contrib/miniselect/include/miniselect/median_of_3_random.h delete mode 100644 contrib/miniselect/include/miniselect/median_of_medians.h delete mode 100644 contrib/miniselect/include/miniselect/median_of_ninthers.h delete mode 100644 contrib/miniselect/include/miniselect/pdqselect.h delete mode 100644 contrib/miniselect/include/miniselect/private/median_common.h delete mode 100644 contrib/miniselect/testing/test_common.h delete mode 100644 contrib/miniselect/testing/test_select.cpp delete mode 100644 contrib/miniselect/testing/test_sort.cpp diff --git a/.gitmodules b/.gitmodules index fdd48fcce01..0e4291eac9e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -190,3 +190,6 @@ path = contrib/croaring url = https://github.com/RoaringBitmap/CRoaring branch = v0.2.66 +[submodule "contrib/miniselect"] + path = contrib/miniselect + url = https://github.com/danlark1/miniselect diff --git a/contrib/miniselect b/contrib/miniselect new file mode 160000 index 00000000000..be0af6bd0b6 --- /dev/null +++ b/contrib/miniselect @@ -0,0 +1 @@ +Subproject commit be0af6bd0b6eb044d1acc4f754b229972d99903a diff --git a/contrib/miniselect/.clang-format b/contrib/miniselect/.clang-format deleted file mode 100644 index f6cb8ad931f..00000000000 --- a/contrib/miniselect/.clang-format +++ /dev/null @@ -1 +0,0 @@ -BasedOnStyle: Google diff --git a/contrib/miniselect/.gitignore b/contrib/miniselect/.gitignore deleted file mode 100644 index f80f36759c8..00000000000 --- a/contrib/miniselect/.gitignore +++ /dev/null @@ -1,100 +0,0 @@ -# eclipse project files -.cproject -.project -.settings - -# emacs temp files -*~ - -# vim temp files -.*.swp - -# XCode -^build/ -*.pbxuser -!default.pbxuser -*.mode1v3 -!default.mode1v3 -*.mode2v3 -!default.mode2v3 -*.perspectivev3 -!default.perspectivev3 -xcuserdata -*.xccheckout -*.moved-aside -DerivedData -*.hmap -*.ipa -*.xcuserstate -*.DS_Store - -# IDE specific folder for JetBrains IDEs -.idea/ -cmake-build-debug/ -cmake-build-release/ - -# Visual Studio Code artifacts -.vscode/* -.history/ - -# Visual Studio artifacts -/VS/ - -# C/C++ build outputs -.build/ -bins -gens -libs -objs - -# C++ ignore from https://github.com/github/gitignore/blob/master/C%2B%2B.gitignore - -# Prerequisites -*.d - -# Compiled Object files -*.slo -*.lo -*.o -*.obj - -# Precompiled Headers -*.gch -*.pch - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - - -# CMake files that may be specific to our installation - -# Build outputs -/build*/ -/visual_studio/ -/benchmark/ - -# Fuzzer outputs generated by instructions in fuzz/Fuzzing.md -/corpus.zip -/ossfuzz-out/ -/out/ - -# Generated docs -/doc/api -*.orig diff --git a/contrib/miniselect/.travis.yml b/contrib/miniselect/.travis.yml deleted file mode 100644 index a5036caf365..00000000000 --- a/contrib/miniselect/.travis.yml +++ /dev/null @@ -1,140 +0,0 @@ -language: cpp - -dist: bionic - -matrix: - include: - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-8 - env: - - COMPILER="CC=gcc-8 && CXX=g++-8" - compiler: gcc-8 - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-9 - env: - - COMPILER="CC=gcc-9 && CXX=g++-9" - compiler: gcc-9 - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-10 - env: - - COMPILER="CC=gcc-10 && CXX=g++-10" - compiler: gcc-10 - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-10 - env: - - COMPILER="CC=gcc-10 && CXX=g++-10" - - SANITIZE="on" - compiler: gcc-10-sanitize - - - os: linux - addons: - apt: - sources: - - llvm-toolchain-bionic-6.0 - packages: - - clang-6.0 - env: - - COMPILER="CC=clang-6.0 && CXX=clang++-6.0" - compiler: clang-6 - - - os: linux - addons: - apt: - sources: - - llvm-toolchain-bionic-7 - packages: - - clang-7 - env: - - COMPILER="CC=clang-7 && CXX=clang++-7" - compiler: clang-7 - - - os: linux - addons: - apt: - sources: - - llvm-toolchain-bionic-8 - packages: - - clang-8 - env: - - COMPILER="CC=clang-8 && CXX=clang++-8" - compiler: clang-8 - - - os: linux - addons: - apt: - sources: - - llvm-toolchain-bionic-9 - packages: - - clang-9 - env: - - COMPILER="CC=clang-9 && CXX=clang++-9" - compiler: clang-9 - - - os: linux - addons: - apt: - packages: - - clang-10 - sources: - - ubuntu-toolchain-r-test - - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' - key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - env: - - COMPILER="CC=clang-10 && CXX=clang++-10" - compiler: clang-10 - - - os: linux - addons: - apt: - packages: - - clang-10 - sources: - - ubuntu-toolchain-r-test - - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' - key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - env: - - COMPILER="CC=clang-10 && CXX=clang++-10" - - SANITIZE="on" - compiler: clang-10-sanitize - -before_install: - - eval "${COMPILER}" - - git clone https://github.com/google/benchmark.git - - git clone https://github.com/google/googletest.git benchmark/googletest - -install: - - export CMAKE_FLAGS="-DMINISELECT_TESTING=on -DCMAKE_BUILD_TYPE=RelWithDebInfo"; - - if [[ "${SANITIZE}" == "on" ]]; then - export CMAKE_FLAGS="${CMAKE_FLAGS} -DMINISELECT_SANITIZE=on"; - fi - - export CTEST_FLAGS="-j4 --output-on-failure -E checkperf" - -script: - - mkdir build - - cd build - - cmake $CMAKE_FLAGS .. - - cmake --build . -- -j2 - - ctest $CTEST_FLAGS diff --git a/contrib/miniselect/AUTHORS b/contrib/miniselect/AUTHORS deleted file mode 100644 index 896a8046a73..00000000000 --- a/contrib/miniselect/AUTHORS +++ /dev/null @@ -1,2 +0,0 @@ -# List of authors for copyright purposes, in no particular order -Danila Kutenin diff --git a/contrib/miniselect/CMakeLists.txt b/contrib/miniselect/CMakeLists.txt deleted file mode 100644 index 09e92031784..00000000000 --- a/contrib/miniselect/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -cmake_minimum_required(VERSION 3.7) -project(miniselect) - -option(MINISELECT_TESTING "Building the tests." OFF) -option(MINISELECT_SANITIZE "Building the library with sanitizers." OFF) -option(MINISELECT_BUILD_LIBCXX "Building the library with libcxx." OFF) -option(MINISELECT_ENABLE_FUZZING "Building the library with fuzzing." OFF) - -include_directories(include) - -if (MINISELECT_TESTING) - enable_testing() - set(CMAKE_CXX_STANDARD 17) - if (NOT CMAKE_BUILD_TYPE) - message(STATUS "No build type selected, default to Release") - set(CMAKE_BUILD_TYPE "Release") - endif() - if (MINISELECT_SANITIZE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all") - endif() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Wextra -Wpedantic -Wno-gnu-zero-variadic-macro-arguments") - - if (MINISELECT_BUILD_LIBCXX AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - message(STATUS "Using libcxx as a default standard C++ library") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") - endif() - - add_subdirectory(benchmark) - include_directories(testing) - include_directories(benches) - - add_executable(benchmark_sort benches/benchmark_sort.cpp) - target_link_libraries(benchmark_sort benchmark::benchmark gtest) - add_executable(benchmark_select benches/benchmark_select.cpp) - target_link_libraries(benchmark_select benchmark::benchmark gtest) - - set(TEST_SOURCES testing/test_select.cpp) - add_executable(test_select ${TEST_SOURCES}) - target_link_libraries(test_select gtest gmock gtest_main) - add_test(NAME test_select COMMAND test_select) - - set(TEST_SOURCES testing/test_sort.cpp) - add_executable(test_sort ${TEST_SOURCES}) - target_link_libraries(test_sort gtest gmock gtest_main) - add_test(NAME test_sort COMMAND test_sort) -endif() - -if(MINISELECT_ENABLE_FUZZING) - add_subdirectory(benchmark) - include_directories(testing) - add_subdirectory(fuzz) -endif() diff --git a/contrib/miniselect/CONTRIBUTORS b/contrib/miniselect/CONTRIBUTORS deleted file mode 100644 index 75d47387e67..00000000000 --- a/contrib/miniselect/CONTRIBUTORS +++ /dev/null @@ -1 +0,0 @@ -# contributors (in no particular order) diff --git a/contrib/miniselect/LICENSE_1_0.txt b/contrib/miniselect/LICENSE_1_0.txt deleted file mode 100644 index 36b7cd93cdf..00000000000 --- a/contrib/miniselect/LICENSE_1_0.txt +++ /dev/null @@ -1,23 +0,0 @@ -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/contrib/miniselect/README.md b/contrib/miniselect/README.md deleted file mode 100644 index cbe576ddba8..00000000000 --- a/contrib/miniselect/README.md +++ /dev/null @@ -1,272 +0,0 @@ -[![Build Status](https://travis-ci.com/danlark1/miniselect.svg?branch=main)](https://travis-ci.com/danlark1/miniselect) -[![License](https://img.shields.io/badge/License-Boost%201.0-lightblue.svg)](https://www.boost.org/LICENSE_1_0.txt) - -miniselect : Generic selection and partial ordering algorithms -============================================================== - -`miniselect` is a C++ header-only library that contains various generic selection -and partial sorting algorithms with the ease of use, testing, advice on usage and -benchmarking. - -Sorting is everywhere and there are many outstanding sorting algorithms that -compete in speed, comparison count and cache friendliness. However selection -algorithms are always a bit outside of the competition scope, however they are -pretty important, for example, in databases ORDER BY LIMIT N is used extremely -often which can benefit from more optimal selection and partial sorting -algorithms. This library tries to solve this problem with Modern C++. - -* **Easy:** First-class, easy to use dependency and carefully documented APIs and algorithm properties. -* **Fast:** We do care about speed of the algorithms and provide reasonable implementations. -* **Standard compliant:** We provide C++11 compatible APIs that are compliant to the standard [`std::nth_element`](https://en.cppreference.com/w/cpp/algorithm/nth_element) and [`std::partial_sort`](https://en.cppreference.com/w/cpp/algorithm/partial_sort) functions including custom comparators and order guarantees. Just replace the names of the functions in your project and it should work! -* **Well tested:** We test all algorithms with a unified framework, under sanitizers and fuzzing. -* **Benchmarked:** We gather benchmarks for all implementations to better understand good and bad spots. - -Table of Contents ------------------ - -* [Quick Start](#quick-start) -* [Testing](#testing) -* [Documentation](#documentation) -* [Performance results](#performance-results) -* [Real-world usage](#real-world-usage) -* [Contributing](#contributing) -* [Motivation](#motivation) -* [License](#license) - -Quick Start ------------ - -You can either include this project as a cmake dependency and then use the -headers that are provided in the [include](./include) folder or just pass the -[include](./include) folder to your compiler. - -```cpp -#include -#include - -#include "miniselect/median_of_ninthers.h" - -int main() { - std::vector v = {1, 8, 4, 3, 2, 9, 0, 7, 6, 5}; - miniselect::median_of_ninthers_select(v.begin(), v.begin() + 5, v.end()); - for (const int i : v) { - std::cout << i << ' '; - } - return 0; -} -// Compile it `clang++/g++ -I$DIRECTORY/miniselect/include/ example.cpp -std=c++11 -O3 -o example -// Possible output: 0 1 4 3 2 5 8 7 6 9 -``` - -Examples can be found in [examples](./examples). - -We support all compilers starting from GCC 7 and Clang 6. We are also planning -to support Windows, for now it is best effort but no issues are known so far. - -More on which algorithms are available, see [documentation](#documentation). - -Testing -------- - -To test and benchmark, we use [Google benchmark](https://github.com/google/benchmark) library. -Simply do in the root directory: - -```console -# Check out the library. -$ git clone https://github.com/google/benchmark.git -# Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory. -$ git clone https://github.com/google/googletest.git benchmark/googletest -$ mkdir build && cd build -$ cmake -DMINISELECT_TESTING=on .. -$ make -j -$ ctest -j4 --output-on-failure -``` - -It will create two tests and two benchmarks `test_sort`, `test_select`, -`benchmark_sort`, `benchmark_select`. Use them to validate or contribute. You -can also use `ctest` - -Documentation -------------- - -There are several selection algorithms available, further ![\large n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+n) is the number -of elements in the array, ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) is the selection element that is needed to be found (all algorithms are deterministic and not stable unless otherwise is specified): - - -| Name | Average | Best Case | Worst Case | Comparisons | Memory | -|------------------------- |--------------------------------------------------------------------------------------------------------- |--------------------------------------------------------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |--------------------------------------------------------------------------------------------------------------------------------- | -| [pdqselect](./include/miniselect/pdqselect.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5Clog+n%29) | At least ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n). Random data ![\large 2.5n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2.5n) | ![\large O(1)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%281%29) | -| [Floyd-Rivest](./include/miniselect/floyd_rivest_select.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n^2 )](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5E2+%29) | Avg: ![\large n + \min(k, n - k) + O(\sqrt{n \log n})](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+n+%2B+%5Cmin%28k%2C+n+-+k%29+%2B+O%28%5Csqrt%7Bn+%5Clog+n%7D%29) | ![\large O(\log \log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+%5Clog+n%29) | -| [Median Of Medians](./include/miniselect/median_of_medians.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | Between ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n) and ![\large 22n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+22n). Random data ![\large 2.5n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2.5n) | ![\large O(\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+n%29) | -| [Median Of Ninthers](./include/miniselect/median_of_ninthers.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | Between ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n) and ![\large 12n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+12.5n). Random data ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n) | ![\large O(\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+n%29) | -| [Median Of 3 Random](./include/miniselect/median_of_3_random.h) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n^2 )](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5E2+%29) | At least ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n). Random data ![\large 3n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+3n) | ![\large O(\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+n%29) | -| [libstdc++ (introselect)](https://github.com/gcc-mirror/gcc/blob/e0af865ab9d9d5b6b3ac7fdde26cf9bbf635b6b4/libstdc%2B%2B-v3/include/bits/stl_algo.h#L4748) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5Clog+n%29) | At least ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n). Random data ![\large 3n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+3n) | ![\large O(1)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%281%29) | -| [libc++ (median of 3)](https://github.com/llvm/llvm-project/blob/3ed89b51da38f081fedb57727076262abb81d149/libcxx/include/algorithm#L5159) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n^2 )](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5E2+%29) | At least ![\large 2n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+2n). Random data ![\large 3n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+3n) | ![\large O(1)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%281%29) | - -For sorting the situation is similar except every line adds ![\large O(k\log k)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28k%5Clog+k%29) comparisons and pdqselect is using ![\large O(\log n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28%5Clog+n%29) memory with one more general exception called partial sorting in C++ standard library. - -| Name | Average | Best Case | Worst Case | Comparisons | Memory | -|-------------------|-----------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------- | -| [std::partial_sort](https://github.com/llvm/llvm-project/blob/3ed89b51da38f081fedb57727076262abb81d149/libcxx/include/algorithm#L5074) | ![\large O(n\log k)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5Clog+k%29) | ![\large O(n)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%29) | ![\large O(n\log k)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%28n%5Clog+k%29) | ![\large n\log k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+n%5Clog+k) on average, for some data patterns might be better | ![\large O(1)](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+O%281%29) | - -## API - -All functions end either in `select`, either in `partial_sort` and -their behavior is exactly the same as for -[`std::nth_element`](https://en.cppreference.com/w/cpp/algorithm/nth_element) -and [`std::partial_sort`](https://en.cppreference.com/w/cpp/algorithm/partial_sort) -respectively, i.e. they accept 3 arguments as `first`, `middle`, `end` iterators -and an optional comparator. Several notes: - -* You should not throw exceptions from `Compare` function. Standard library -also does not specify the behavior in that matter. -* We don't support ParallelSTL for now. -* C++20 constexpr specifiers might be added but currently we don't have them -because of some floating point math in several algorithms. -* All functions are in the `miniselect` namespace. See the example for that. - -- pdqselect - - This algorithm is based on [`pdqsort`](https://github.com/orlp/pdqsort) which is acknowledged as one of the fastest generic sort algorithms. - - **Location:** [`miniselect/pdqselect.h`](./include/miniselect/pdqselect.h). - - **Functions:** `pdqselect`, `pdqselect_branchless`, `pdqpartial_sort`, `pdqpartial_sort_branchless`. Branchless version uses branchless partition algorithm provided by [`pdqsort`](https://github.com/orlp/pdqsort). Use it if your comparison function is branchless, it might give performance for very big ranges. - - **Performance advice:** Use it when you need to sort a big chunk so that ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) is close to ![\large n](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+n). - -

- -- Floyd-Rivest - - This algorithm is based on [Floyd-Rivest algorithm](https://en.wikipedia.org/wiki/Floyd%E2%80%93Rivest_algorithm). - - **Location:** [`miniselect/floyd_rivest_select.h`](./include/miniselect/floyd_rivest_select.h). - - **Functions:** `floyd_rivest_select`, `floyd_rivest_partial_sort`. - - **Performance advice:** Given that this algorithm performs as one of the best on average case in terms of comparisons and speed, we highly advise to - at least try this in your project. Especially it is good for small ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) or types that are expensive to compare (for example, strings). But even for median the benchmarks show it outperforms others. It is not easy for this algorithm to build a reasonable worst case but one of examples when this algorithm does not perform well is when there are lots of similar values of linear size (random01 dataset showed some moderate penalties). - -We present here two gifs, for median and for ![\large k = n / 10](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k+%3D+n+%2F+10) order statistic. - -

- - -

- -- Median Of Medians - - This algorithm is based on [Median of Medians](https://en.wikipedia.org/wiki/Median_of_medians) algorithm, one of the first deterministic linear time worst case median algorithm - - **Location:** [`miniselect/median_of_medians.h`](./include/miniselect/median_of_medians.h). - - **Functions:** `median_of_medians_select`, `median_of_medians_partial_sort`. - - **Performance advice:** This algorithm does not show advantages over others, implemented for historical reasons and for bechmarking. - -

- -- Median Of Ninthers - - This algorithm is based on [Fast Deterministic Selection](https://erdani.com/research/sea2017.pdf) paper by Andrei Alexandrescu, one of the latest and fastest deterministic linear time worst case median algorithms - - **Location:** [`miniselect/median_of_ninthers.h`](./include/miniselect/median_of_ninthers.h). - - **Functions:** `median_of_ninthers_select`, `median_of_ninthers_partial_sort`. - - **Performance advice:** Use this algorithm if you absolutely need linear time worst case scenario for selection algorithm. This algorithm shows some strengths over other deterministic [`PICK`](https://en.wikipedia.org/wiki/Median_of_medians) algorithms and has lower constanst than MedianOfMedians. - -

- -- Median Of 3 Random - - This algorithm is based on QuickSelect with the random median of 3 pivot choice algorithm (it chooses random 3 elements in the range and takes the middle value). It is a rando - - **Location:** [`miniselect/median_of_3_random.h`](./include/miniselect/median_of_3_random.h). - - **Functions:** `median_of_3_random_select`, `median_of_3_random_partial_sort`. - - **Performance advice:** This is a randomized algorithm and also it did not show any strengths against Median Of Ninthers. - -

- -- Introselect - - This algorithm is based on [Introselect](https://en.wikipedia.org/wiki/Introselect) algorithm, it is used in libstdc++ in `std::nth_element`, however instead of falling back to MedianOfMedians it is using HeapSelect which adds logarithm to its worst complexity. - - **Location:** ``. - - **Functions:** `std::nth_element`. - - **Performance advice:** This algorithm is used in standard library and is not recommended to use if you are looking for performance. - -

- -- Median Of 3 - - This algorithm is based on QuickSelect with median of 3 pivot choice algorithm (the middle value between begin, mid and end values), it is used in libc++ in `std::nth_element`. - - **Location:** ``. - - **Functions:** `std::nth_element`. - - **Performance advice:** This algorithm is used in standard library and is not recommended to use if you are looking for performance. - -

- -- `std::partial_sort` - - This algorithm has [heap-based solutions](https://en.wikipedia.org/wiki/Partial_sorting) both in libc++ and libstdc++, from the first ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) elements the max heap is built, then one by one the elements are trying to be pushed to that heap with HeapSort in the end. - - **Location:** ``. - - **Functions:** `std::partial_sort`. - - **Performance advice:** This algorithm is very good for random data and small ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) and might outperform all selection+sort algorithms. However, for descending data it starts to significantly degrade and is not recommended for use if you have such patterns in real data. - -

- -## Other algorithms to come - -* Kiwiel modification of FloydRivest algorithm which is described in [On Floyd and Rivest’s SELECT algorithm](https://core.ac.uk/download/pdf/82672439.pdf) with ternary and quintary pivots. -* Combination of FloydRivest and pdqsort pivot strategies, currently all experiments did not show any boost. - -Performance results -------------------- - -We use 10 datasets and 8 algorithms with 10000000 elements to find median and -other ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) on `Intel(R) Core(TM) i5-4200H CPU @ 2.80GHz` for `std::vector`, -for median the benchmarks are the following: - -![median](benches/plots/result_10000000_5000000.png) - -![median](benches/plots/result_comparisons_10000000_5000000.png) - -For smaller ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k), -for example, 1000, the results are the following - -![k equals 1000](benches/plots/result_10000000_1000.png) - -![k equals 1000](benches/plots/result_comparisons_10000000_1000.png) - -Other benchmarks can be found [here](https://drive.google.com/drive/folders/1DHEaeXgZuX6AJ9eByeZ8iQVQv0ueP8XM). - -The benchmarks for number of swaps will be later. - -Real-world usage ----------------- - -- [Yandex ClickHouse](https://github.com/yandex/ClickHouse) - -If you are planning to use miniselect in your product, please work from one of -our releases and if you wish, you can write the acknowledgment in this section -for visibility. - -Contributing ------------- - -Patches are welcome with new algorithms! You should add the selection algorithm -together with the partial sorting algorithm in [include](./include), add -tests in [testing](./testing) and ideally run benchmarks to see how it performs. -If you also have some data cases to test against, we would be more than happy -to merge them. - -Motivation ----------- - -Firstly the author was interested if any research had been done for small ![\large k](https://render.githubusercontent.com/render/math?math=%5Cdisplaystyle+%5Clarge+k) -in selection algorithms and was struggling to find working implementations to -compare different approaches from standard library and quickselect algorithms. -After that it turned out that the problem is much more interesting than it looks -like and after reading The Art of Computer Programming from Donald Knuth about -minimum comparison sorting and selection algorithms the author decided to look -through all non-popular algorithms and try them out. - -The author have not found any decent library for selection algorithms and little -research is published in open source, so that they decided to merge all that -implementations and compare them with possible merging of different ideas -into a decent one algorithm for most needs. For a big story of adventures see -the author's blog post TODO. - -License -------- - -The code is made available under the [Boost License 1.0](https://boost.org/LICENSE_1_0.txt). - -Third-Party Libraries Used and Adjusted ---------------------------------------- - -| Library | License | -|---------------------|--------------------------------------------------------------------------------------------------| -| pdqsort | [MIT](https://github.com/orlp/pdqsort/blob/47a46767d76fc852284eaa083e4b7034ee6e2559/license.txt) | -| MedianOfNinthers | [Boost License 1.0](https://github.com/andralex/MedianOfNinthers/blob/master/LICENSE_1_0.txt) | - diff --git a/contrib/miniselect/benches/bench_common.h b/contrib/miniselect/benches/bench_common.h deleted file mode 100644 index b49b55dac9d..00000000000 --- a/contrib/miniselect/benches/bench_common.h +++ /dev/null @@ -1,170 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#pragma once - -#include -#include -#include - -namespace miniselect { -namespace datagens { - -struct Random { - static std::vector Gen(size_t size) { - std::random_device rnd_device; - std::mt19937_64 mersenne_engine{rnd_device()}; - std::vector v; - v.reserve(size); - for (size_t i = 0; i < size; ++i) { - v.push_back(i); - } - std::shuffle(v.begin(), v.end(), mersenne_engine); - return v; - } -}; - -struct Shuffled16 { - static std::vector Gen(size_t size) { - std::random_device rnd_device; - std::mt19937_64 mersenne_engine{rnd_device()}; - std::vector v; - v.reserve(size); - for (size_t i = 0; i < size; ++i) { - v.push_back(i % 16); - } - std::shuffle(v.begin(), v.end(), mersenne_engine); - return v; - } -}; - -struct Random01 { - static std::vector Gen(size_t size) { - std::random_device rnd_device; - std::mt19937_64 mersenne_engine{rnd_device()}; - std::vector v; - v.reserve(size); - for (size_t i = 0; i < size; ++i) { - v.push_back(i % 2); - } - std::shuffle(v.begin(), v.end(), mersenne_engine); - return v; - } -}; - -struct Ascending { - static std::vector Gen(size_t size) { - std::vector v; - v.reserve(size); - for (size_t i = 0; i < size; ++i) { - v.push_back(i); - } - return v; - } -}; - -struct Descending { - static std::vector Gen(size_t size) { - std::vector v; - v.reserve(size); - for (int i = size - 1; i >= 0; --i) { - v.push_back(i); - } - return v; - } -}; - -struct PipeOrgan { - static std::vector Gen(size_t size) { - std::vector v; - v.reserve(size); - for (size_t i = 0; i < size / 2; ++i) { - v.push_back(i); - } - for (size_t i = size / 2; i < size; ++i) { - v.push_back(size - i); - } - return v; - } -}; - -struct PushFront { - static std::vector Gen(size_t size) { - std::vector v; - v.reserve(size); - for (size_t i = 1; i < size; ++i) { - v.push_back(i); - } - v.push_back(0); - return v; - } -}; - -struct PushMiddle { - static std::vector Gen(size_t size) { - std::vector v; - v.reserve(size); - for (size_t i = 0; i < size; ++i) { - if (i != size / 2) { - v.push_back(i); - } - } - v.push_back(size / 2); - return v; - } -}; - -struct Median3Killer { - static std::vector Gen(size_t size) { - size_t k = size / 2; - std::vector v; - v.reserve(size); - for (size_t i = 1; i < k + 1; ++i) { - if (i & 1) { - v.push_back(i); - } else { - v.push_back(k + i - 1); - } - } - for (size_t i = 1; i < k + 1; ++i) { - v.push_back(2 * i); - } - return v; - } -}; - -#define BENCH_IMPL(BENCH, GEN, IMPL) \ - BENCHMARK_TEMPLATE(BENCH, GEN, IMPL) \ - ->Unit(benchmark::kMicrosecond) \ - ->Arg(kSize - 10) \ - ->Arg(kSize / 2) \ - ->Arg(10000) \ - ->Arg(1000) \ - ->Arg(100) \ - ->Arg(10) \ - ->Arg(1) - -#define BENCH_GENS(BENCH, IMPL) \ - BENCH_IMPL(BENCH, datagens::Random, IMPL); \ - BENCH_IMPL(BENCH, datagens::Shuffled16, IMPL); \ - BENCH_IMPL(BENCH, datagens::Random01, IMPL); \ - BENCH_IMPL(BENCH, datagens::Ascending, IMPL); \ - BENCH_IMPL(BENCH, datagens::Descending, IMPL); \ - BENCH_IMPL(BENCH, datagens::PipeOrgan, IMPL); \ - BENCH_IMPL(BENCH, datagens::PushMiddle, IMPL); \ - BENCH_IMPL(BENCH, datagens::PushFront, IMPL); \ - BENCH_IMPL(BENCH, datagens::Median3Killer, IMPL) - -#define BENCH(NAME) \ - BENCH_GENS(NAME, algorithms::FloydRivest); \ - BENCH_GENS(NAME, algorithms::MedianOfNinthers); \ - BENCH_GENS(NAME, algorithms::MedianOfMedians); \ - BENCH_GENS(NAME, algorithms::MedianOf3Random); \ - BENCH_GENS(NAME, algorithms::PDQ); \ - BENCH_GENS(NAME, algorithms::PDQBranchless); \ - BENCH_GENS(NAME, algorithms::STD) - -} // namespace datagens -} // namespace miniselect diff --git a/contrib/miniselect/benches/benchmark_select.cpp b/contrib/miniselect/benches/benchmark_select.cpp deleted file mode 100644 index 2a9b238c90a..00000000000 --- a/contrib/miniselect/benches/benchmark_select.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#include - -#include -#include -#include -#include -#include -#include - -#include "bench_common.h" -#include "test_common.h" - -namespace miniselect { -namespace { - -static constexpr size_t kSize = 65536; - -template -static void BM_sel(benchmark::State& state) { - auto vec = DataGen::Gen(kSize); - const size_t arg = state.range(0); - size_t cnt = 0; - size_t cmp = 0; - for (auto _ : state) { - Impl::Select(vec.begin(), vec.begin() + arg, vec.end(), - [&cmp](const auto& left, const auto& right) { - cmp++; - return left < right; - }); - ++cnt; - benchmark::DoNotOptimize(vec[arg]); - } - state.counters["Comparisons"] = 1.0 * cmp / cnt; -} - -BENCH(BM_sel); - -} // namespace -} // namespace miniselect - -BENCHMARK_MAIN(); diff --git a/contrib/miniselect/benches/benchmark_sort.cpp b/contrib/miniselect/benches/benchmark_sort.cpp deleted file mode 100644 index 8b3bbd1a77f..00000000000 --- a/contrib/miniselect/benches/benchmark_sort.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#include - -#include -#include -#include -#include -#include -#include - -#include "bench_common.h" -#include "test_common.h" - -namespace miniselect { -namespace { - -static constexpr size_t kSize = 65536; - -template -static void BM_sort(benchmark::State& state) { - auto vec = DataGen::Gen(kSize); - const size_t arg = state.range(0); - size_t cnt = 0; - size_t cmp = 0; - for (auto _ : state) { - Impl::Sort(vec.begin(), vec.begin() + arg, vec.end(), - [&cmp](const auto& left, const auto& right) { - cmp++; - return left < right; - }); - ++cnt; - benchmark::DoNotOptimize(vec[arg]); - } - state.counters["Comparisons"] = 1.0 * cmp / cnt; -} - -BENCH(BM_sort); - -} // namespace -} // namespace miniselect - -BENCHMARK_MAIN(); diff --git a/contrib/miniselect/examples/example.cpp b/contrib/miniselect/examples/example.cpp deleted file mode 100644 index 183e81ae1b6..00000000000 --- a/contrib/miniselect/examples/example.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include -#include - -#include "miniselect/median_of_ninthers.h" - -int main() { - std::vector v = {1, 8, 4, 3, 2, 9, 0, 7, 6, 5}; - miniselect::median_of_ninthers_select(v.begin(), v.begin() + 5, v.end()); - for (const int i : v) { - std::cout << i << ' '; - } - return 0; -} - -// Compile it `clang++/g++ -I$DIRECTORY/miniselect/include/ example.cpp -std=c++11 -O3 -o example - -// Possible output: 0 1 4 3 2 5 8 7 6 9 -// ^ on the right place diff --git a/contrib/miniselect/fuzz/CMakeLists.txt b/contrib/miniselect/fuzz/CMakeLists.txt deleted file mode 100644 index 38473bd78ad..00000000000 --- a/contrib/miniselect/fuzz/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -cmake_minimum_required(VERSION 3.7) - -project(fuzz) - -option(ENABLE_FUZZING "enable building the fuzzers" ON) -set(CMAKE_CXX_STANDARD 17) - -if(ENABLE_FUZZING) - set(MINISELECT_FUZZ_LDFLAGS "" CACHE STRING "LDFLAGS for the fuzz targets") - - add_library(miniselect-fuzzer INTERFACE) - target_link_libraries(miniselect-fuzzer INTERFACE gtest) - target_link_libraries(miniselect-fuzzer INTERFACE ${MINISELECT_FUZZ_LDFLAGS}) - - if(MINISELECT_FUZZ_LINKMAIN) - target_sources(simdjson-fuzzer INTERFACE $/main.cpp) - endif() - - # Define the fuzzers - add_custom_target(all_fuzzers) - - set(fuzzernames) - function(implement_fuzzer name) - add_executable(${name} ${name}.cpp) - target_link_libraries(${name} PRIVATE miniselect-fuzzer) - add_dependencies(all_fuzzers ${name}) - set(fuzzernames ${fuzzernames} ${name} PARENT_SCOPE) - endfunction() - - implement_fuzzer(fuzz_select) - implement_fuzzer(fuzz_string_select) - implement_fuzzer(fuzz_sort) - implement_fuzzer(fuzz_string_sort) - - # to be able to get a list of all fuzzers from within a script - add_custom_target(print_all_fuzzernames - COMMAND ${CMAKE_COMMAND} -E echo ${fuzzernames}) -endif() diff --git a/contrib/miniselect/fuzz/build_like_oss_fuzz.sh b/contrib/miniselect/fuzz/build_like_oss_fuzz.sh deleted file mode 100755 index 547348133a9..00000000000 --- a/contrib/miniselect/fuzz/build_like_oss_fuzz.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -# -# This script emulates how oss fuzz invokes the build -# process, handy for trouble shooting cmake issues and possibly -# recreating testcases. For proper debugging of the oss fuzz -# build, follow the procedure at https://google.github.io/oss-fuzz/getting-started/new-project-guide/#testing-locally - -set -eu - -ossfuzz=$(readlink -f $(dirname $0))/ossfuzz.sh - -mkdir -p ossfuzz-out -export OUT=$(pwd)/ossfuzz-out -export CC=clang -export CXX="clang++" -export CFLAGS="-fsanitize=fuzzer-no-link" -export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -O1" -export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" - -$ossfuzz - -echo "look at the results in $OUT" diff --git a/contrib/miniselect/fuzz/fuzz_select.cpp b/contrib/miniselect/fuzz/fuzz_select.cpp deleted file mode 100644 index f70980bd0d9..00000000000 --- a/contrib/miniselect/fuzz/fuzz_select.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include -#include -#include -#include - -#include "test_common.h" - -template -void ChooseImplementation(uint8_t byte, std::vector& working, - Iter partition_iter, const ::testing::Types&) { - static_assert(sizeof...(T) < 256); - int i = 0; - constexpr size_t size = sizeof...(T); - ( - [&]() { - if (byte % size == i++) { - T::Select(working.begin(), partition_iter, working.end()); - } - }(), - ...); -} - -// Use the first element as a position into the data -extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, - std::size_t size) { - if (size <= 3) return 0; - uint8_t impl = data[0]; - uint16_t partition_point = 0; - memcpy(&partition_point, data + 1, 2); - partition_point %= (size - 3); - std::vector working(data + 3, data + size); - auto canonical = working; - const auto partition_iter = working.begin() + partition_point; - ChooseImplementation(impl, working, partition_iter, - miniselect::algorithms::All{}); - - if (partition_iter != working.end()) { - const auto& nth = *partition_iter; - bool is_error = false; - if (!std::all_of(working.begin(), partition_iter, - [&](const auto& v) { return v <= nth; })) { - is_error = true; - } - if (!std::all_of(partition_iter, working.end(), - [&](const auto& v) { return v >= nth; })) { - is_error = true; - } - if (is_error) { - std::cerr << "FAILED!\nCanonical: "; - for (const auto& s : canonical) { - std::cerr << static_cast(s) << ' '; - } - std::cerr << std::endl; - std::cerr << "Got: "; - for (const auto& s : working) { - std::cerr << static_cast(s) << ' '; - } - std::cerr << std::endl; - std::cerr << "partition_iter = " << partition_iter - working.begin() - << std::endl; - std::abort(); - } - } - - return 0; -} diff --git a/contrib/miniselect/fuzz/fuzz_sort.cpp b/contrib/miniselect/fuzz/fuzz_sort.cpp deleted file mode 100644 index ba0a2b6ca3e..00000000000 --- a/contrib/miniselect/fuzz/fuzz_sort.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include -#include -#include - -#include "test_common.h" - -template -void ChooseImplementation(uint8_t byte, std::vector& working, - Iter partition_iter, const ::testing::Types&) { - static_assert(sizeof...(T) < 256); - int i = 0; - constexpr size_t size = sizeof...(T); - ( - [&]() { - if (byte % size == i++) { - T::Sort(working.begin(), partition_iter, working.end()); - } - }(), - ...); -} - -// Use the first element as a position into the data -extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, - std::size_t size) { - if (size <= 3) return 0; - uint8_t impl = data[0]; - uint16_t partition_point = 0; - memcpy(&partition_point, data + 1, 2); - partition_point %= (size - 3); - std::vector working(data + 3, data + size); - auto canonical = working; - const auto partition_iter = working.begin() + partition_point; - ChooseImplementation(impl, working, partition_iter, - miniselect::algorithms::All{}); - - bool is_error = false; - if (partition_iter != working.end()) { - const auto& nth = *std::min_element(partition_iter, working.end()); - if (!std::all_of(working.begin(), partition_iter, - [&](const auto& v) { return v <= nth; })) { - is_error = true; - } - if (!std::all_of(partition_iter, working.end(), - [&](const auto& v) { return v >= nth; })) { - is_error = true; - } - } - if (!std::is_sorted(working.begin(), partition_iter)) { - is_error = true; - } - if (is_error) { - std::cerr << "FAILED!\nCanonical: "; - for (const auto& s : canonical) { - std::cerr << static_cast(s) << ' '; - } - std::cerr << std::endl; - std::cerr << "Got: "; - for (const auto& s : working) { - std::cerr << static_cast(s) << ' '; - } - std::cerr << std::endl; - std::cerr << "partition_iter = " << partition_iter - working.begin() - << std::endl; - std::abort(); - } - - return 0; -} diff --git a/contrib/miniselect/fuzz/fuzz_string_select.cpp b/contrib/miniselect/fuzz/fuzz_string_select.cpp deleted file mode 100644 index cd24b376d86..00000000000 --- a/contrib/miniselect/fuzz/fuzz_string_select.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include -#include -#include -#include - -#include "test_common.h" - -template -void ChooseImplementation(uint8_t byte, std::vector& working, - Iter partition_iter, const ::testing::Types&) { - static_assert(sizeof...(T) < 256); - int i = 0; - constexpr size_t size = sizeof...(T); - ( - [&]() { - if (byte % size == i++) { - T::Select(working.begin(), partition_iter, working.end()); - } - }(), - ...); -} - -// Use the first element as a position into the data -extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, - std::size_t size) { - if (size <= 3) return 0; - uint8_t impl = data[0]; - uint16_t partition_point = 0; - memcpy(&partition_point, data + 1, 2); - partition_point %= (size - 3); - std::vector working; - for (auto i = data + 3; i < data + size; ++i) { - std::string s(1, *i); - working.push_back(s); - } - auto canonical = working; - const auto partition_iter = working.begin() + partition_point; - ChooseImplementation(impl, working, partition_iter, - miniselect::algorithms::All{}); - // nth may be the end iterator, in this case nth_element has no effect. - if (partition_iter != working.end()) { - const auto& nth = *partition_iter; - bool is_error = false; - if (!std::all_of(working.begin(), partition_iter, - [&](const auto& v) { return v <= nth; })) { - is_error = true; - } - if (!std::all_of(partition_iter, working.end(), - [&](const auto& v) { return v >= nth; })) { - is_error = true; - } - if (is_error) { - std::cerr << "FAILED!\nCanonical: "; - for (const auto& s : canonical) { - std::cerr << s << ' '; - } - std::cerr << std::endl; - std::cerr << "Got: "; - for (const auto& s : working) { - std::cerr << s << ' '; - } - std::cerr << std::endl; - std::cerr << "partition_iter = " << partition_iter - working.begin() - << std::endl; - std::abort(); - } - } - - return 0; -} diff --git a/contrib/miniselect/fuzz/fuzz_string_sort.cpp b/contrib/miniselect/fuzz/fuzz_string_sort.cpp deleted file mode 100644 index a797e0d7e22..00000000000 --- a/contrib/miniselect/fuzz/fuzz_string_sort.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include -#include -#include -#include - -#include "test_common.h" - -template -void ChooseImplementation(uint8_t byte, std::vector& working, - Iter partition_iter, const ::testing::Types&) { - static_assert(sizeof...(T) < 256); - int i = 0; - constexpr size_t size = sizeof...(T); - ( - [&]() { - if (byte % size == i++) { - T::Sort(working.begin(), partition_iter, working.end()); - } - }(), - ...); -} - -// Use the first element as a position into the data -extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, - std::size_t size) { - if (size <= 3) return 0; - uint8_t impl = data[0]; - uint16_t partition_point = 0; - memcpy(&partition_point, data + 1, 2); - partition_point %= (size - 3); - std::vector working; - for (auto i = data + 3; i < data + size; ++i) { - std::string s(1, *i); - working.push_back(s); - } - auto canonical = working; - const auto partition_iter = working.begin() + partition_point; - ChooseImplementation(impl, working, partition_iter, - miniselect::algorithms::All{}); - // nth may be the end iterator, in this case nth_element has no effect. - bool is_error = false; - if (partition_iter != working.end()) { - const auto& nth = *std::min_element(partition_iter, working.end()); - if (!std::all_of(working.begin(), partition_iter, - [&](const auto& v) { return v <= nth; })) { - is_error = true; - } - if (!std::all_of(partition_iter, working.end(), - [&](const auto& v) { return v >= nth; })) { - is_error = true; - } - } - if (!std::is_sorted(working.begin(), partition_iter)) { - is_error = true; - } - if (is_error) { - std::cerr << "FAILED!\nCanonical: "; - for (const auto& s : canonical) { - std::cerr << s << ' '; - } - std::cerr << std::endl; - std::cerr << "Got: "; - for (const auto& s : working) { - std::cerr << s << ' '; - } - std::cerr << std::endl; - std::cerr << "partition_iter = " << partition_iter - working.begin() - << std::endl; - std::abort(); - } - - return 0; -} diff --git a/contrib/miniselect/fuzz/main.cpp b/contrib/miniselect/fuzz/main.cpp deleted file mode 100644 index e3377035f33..00000000000 --- a/contrib/miniselect/fuzz/main.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include -#include -#include - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, std::size_t Size); - -int main(int argc, char* argv[]) { - for (int i = 1; i < argc; ++i) { - std::ifstream in(argv[i]); - assert(in); - in.seekg(0, std::ios_base::end); - const auto pos = in.tellg(); - assert(pos >= 0); - in.seekg(0, std::ios_base::beg); - std::vector buf(static_cast(pos)); - in.read(buf.data(), static_cast(buf.size())); - assert(in.gcount() == pos); - LLVMFuzzerTestOneInput(reinterpret_cast(buf.data()), - buf.size()); - } -} diff --git a/contrib/miniselect/fuzz/ossfuzz.sh b/contrib/miniselect/fuzz/ossfuzz.sh deleted file mode 100755 index 83f37f54eee..00000000000 --- a/contrib/miniselect/fuzz/ossfuzz.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# -# entry point for oss-fuzz, so that fuzzers -# and build invocation can be changed without having -# to modify the oss-fuzz repo. -# -# invoke it from the git root. - -# make sure to exit on problems -set -eux - -mkdir -p build -cd build - -cmake .. \ --GNinja \ --DCMAKE_BUILD_TYPE=Debug \ --DENABLE_FUZZING=On \ --DMINISELECT_FUZZ_LINKMAIN=off \ --DMINISELECT_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE - -cmake --build . --target all_fuzzers - diff --git a/contrib/miniselect/include/miniselect/floyd_rivest_select.h b/contrib/miniselect/include/miniselect/floyd_rivest_select.h deleted file mode 100644 index e7d5f80f572..00000000000 --- a/contrib/miniselect/include/miniselect/floyd_rivest_select.h +++ /dev/null @@ -1,120 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -namespace miniselect { -namespace floyd_rivest_detail { - -template -struct CompareRefType { - // Pass the comparator by lvalue reference. Or in debug mode, using a - // debugging wrapper that stores a reference. - using type = typename std::add_lvalue_reference::type; -}; - -template -inline void floyd_rivest_select_loop(Iter begin, Diff left, Diff right, Diff k, - Compare comp) { - while (right > left) { - Diff size = right - left; - if (size > 600) { - Diff n = right - left + 1; - Diff i = k - left + 1; - double z = log(n); - double s = 0.5 * exp(2 * z / 3); - double sd = 0.5 * sqrt(z * s * (n - s) / n); - if (i < n / 2) { - sd *= -1.0; - } - Diff newLeft = std::max(left, (Diff)(k - i * s / n + sd)); - Diff newRight = std::min(right, (Diff)(k + (n - i) * s / n + sd)); - floyd_rivest_select_loop(begin, newLeft, newRight, k, - comp); - } - Diff i = left; - Diff j = right; - std::swap(begin[left], begin[k]); - const bool to_swap = comp(begin[left], begin[right]); - if (to_swap) { - std::swap(begin[left], begin[right]); - } - // Make sure that non copyable types compile. - const auto& t = to_swap ? begin[left] : begin[right]; - while (i < j) { - std::swap(begin[i], begin[j]); - i++; - j--; - while (comp(begin[i], t)) { - i++; - } - while (comp(t, begin[j])) { - j--; - } - } - - if (to_swap) { - std::swap(begin[left], begin[j]); - } else { - j++; - std::swap(begin[right], begin[j]); - } - - if (j <= k) { - left = j + 1; - } - if (k <= j) { - right = j - 1; - } - } -} - -} // namespace floyd_rivest_detail - -template -inline void floyd_rivest_partial_sort(Iter begin, Iter mid, Iter end, - Compare comp) { - if (begin == end) return; - if (begin == mid) return; - using CompType = typename floyd_rivest_detail::CompareRefType::type; - - floyd_rivest_detail::floyd_rivest_select_loop< - Iter, CompType, typename std::iterator_traits::difference_type>( - begin, 0, end - begin - 1, mid - begin - 1, comp); - // std::sort proved to be better than other sorts because of pivoting. - std::sort(begin, mid, comp); -} - -template -inline void floyd_rivest_partial_sort(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - floyd_rivest_partial_sort(begin, mid, end, std::less()); -} - -template -inline void floyd_rivest_select(Iter begin, Iter mid, Iter end, Compare comp) { - if (mid == end) return; - using CompType = typename floyd_rivest_detail::CompareRefType::type; - - floyd_rivest_detail::floyd_rivest_select_loop< - Iter, CompType, typename std::iterator_traits::difference_type>( - begin, 0, end - begin - 1, mid - begin, comp); -} - -template -inline void floyd_rivest_select(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - floyd_rivest_select(begin, mid, end, std::less()); -} - -} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/median_of_3_random.h b/contrib/miniselect/include/miniselect/median_of_3_random.h deleted file mode 100644 index 0f7b62fd61c..00000000000 --- a/contrib/miniselect/include/miniselect/median_of_3_random.h +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#pragma once - -#include -#include -#include -#include -#include - -#include "private/median_common.h" - -namespace miniselect { -namespace median_of_3_random_detail { - -template -static inline Iter partition(Iter r, Iter end, Compare&& comp) { - typedef typename std::iterator_traits::difference_type T; - const T len = end - r; - assert(len >= 3); - static std::mt19937_64 gen(1); - std::uniform_int_distribution dis(0, len - 1); - T x = dis(gen); - T y = dis(gen); - T z = dis(gen); - return median_common_detail::pivotPartition( - r, median_common_detail::medianIndex(r, x, y, z, comp), len, comp); -} - -} // namespace median_of_3_random_detail - -template -inline void median_of_3_random_select(Iter begin, Iter mid, Iter end, - Compare comp) { - if (mid == end) return; - using CompType = typename floyd_rivest_detail::CompareRefType::type; - - median_common_detail::quickselect< - Iter, CompType, &median_of_3_random_detail::partition>( - begin, mid, end, comp); -} - -template -inline void median_of_3_random_select(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - median_of_3_random_select(begin, mid, end, std::less()); -} - -template -inline void median_of_3_random_sort(Iter begin, Iter mid, Iter end, - Compare comp) { - if (begin == mid) return; - using CompType = typename floyd_rivest_detail::CompareRefType::type; - median_common_detail::quickselect< - Iter, CompType, &median_of_3_random_detail::partition>( - begin, mid - 1, end, comp); - std::sort(begin, mid, comp); -} - -template -inline void median_of_3_random_sort(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - median_of_3_random_sort(begin, mid, end, std::less()); -} - -} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/median_of_medians.h b/contrib/miniselect/include/miniselect/median_of_medians.h deleted file mode 100644 index 922401b12d0..00000000000 --- a/contrib/miniselect/include/miniselect/median_of_medians.h +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#pragma once - -#include -#include -#include -#include - -#include "private/median_common.h" - -namespace miniselect { -namespace median_of_medians_detail { - -template -static inline Iter partition(Iter r, Iter end, Compare&& comp) { - using CompType = typename median_common_detail::CompareRefType::type; - const size_t len = end - r; - if (len < 5) { - return median_common_detail::pivotPartition(r, len / 2, len, comp); - } - size_t j = 0; - for (size_t i = 4; i < len; i += 5, ++j) { - median_common_detail::partition5(r, i - 4, i - 3, i, i - 2, i - 1, comp); - std::swap(r[i], r[j]); - } - median_common_detail::quickselect(r, r + j / 2, - r + j, comp); - return median_common_detail::pivotPartition(r, j / 2, len, comp); -} - -} // namespace median_of_medians_detail - -template -inline void median_of_medians_select(Iter begin, Iter mid, Iter end, - Compare comp) { - if (mid == end) return; - using CompType = typename median_common_detail::CompareRefType::type; - - median_common_detail::quickselect< - Iter, CompType, &median_of_medians_detail::partition>( - begin, mid, end, comp); -} - -template -inline void median_of_medians_select(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - median_of_medians_select(begin, mid, end, std::less()); -} - -template -inline void median_of_medians_sort(Iter begin, Iter mid, Iter end, - Compare comp) { - if (begin == mid) return; - using CompType = typename median_common_detail::CompareRefType::type; - median_common_detail::quickselect< - Iter, CompType, &median_of_medians_detail::partition>( - begin, mid - 1, end, comp); - std::sort(begin, mid, comp); -} - -template -inline void median_of_medians_sort(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - median_of_medians_sort(begin, mid, end, std::less()); -} - -} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/median_of_ninthers.h b/contrib/miniselect/include/miniselect/median_of_ninthers.h deleted file mode 100644 index 099786cf518..00000000000 --- a/contrib/miniselect/include/miniselect/median_of_ninthers.h +++ /dev/null @@ -1,190 +0,0 @@ -/* Copyright Andrei Alexandrescu, 2016-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -// Adjusted from Alexandrescu paper to support arbitrary comparators. -#pragma once - -#include -#include -#include -#include -#include - -#include "private/median_common.h" - -namespace miniselect { -namespace median_of_ninthers_detail { - -template -void adaptiveQuickselect(Iter r, size_t n, size_t length, Compare&& comp); - -/** -Median of minima -*/ -template -size_t medianOfMinima(Iter const r, const size_t n, const size_t length, - Compare&& comp) { - assert(length >= 2); - assert(n * 4 <= length); - assert(n > 0); - const size_t subset = n * 2, computeMinOver = (length - subset) / subset; - assert(computeMinOver > 0); - for (size_t i = 0, j = subset; i < subset; ++i) { - const auto limit = j + computeMinOver; - size_t minIndex = j; - while (++j < limit) - if (comp(r[j], r[minIndex])) minIndex = j; - if (comp(r[minIndex], r[i])) std::swap(r[i], r[minIndex]); - assert(j < length || i + 1 == subset); - } - adaptiveQuickselect(r, n, subset, comp); - return median_common_detail::expandPartition(r, 0, n, subset, length, comp); -} - -/** -Median of maxima -*/ -template -size_t medianOfMaxima(Iter const r, const size_t n, const size_t length, - Compare&& comp) { - assert(length >= 2); - assert(n * 4 >= length * 3 && n < length); - const size_t subset = (length - n) * 2, subsetStart = length - subset, - computeMaxOver = subsetStart / subset; - assert(computeMaxOver > 0); - for (size_t i = subsetStart, j = i - subset * computeMaxOver; i < length; - ++i) { - const auto limit = j + computeMaxOver; - size_t maxIndex = j; - while (++j < limit) - if (comp(r[maxIndex], r[j])) maxIndex = j; - if (comp(r[i], r[maxIndex])) std::swap(r[i], r[maxIndex]); - assert(j != 0 || i + 1 == length); - } - adaptiveQuickselect(r + subsetStart, length - n, subset, comp); - return median_common_detail::expandPartition(r, subsetStart, n, length, - length, comp); -} - -/** -Partitions r[0 .. length] using a pivot of its own choosing. Attempts to pick a -pivot that approximates the median. Returns the position of the pivot. -*/ -template -size_t medianOfNinthers(Iter const r, const size_t length, Compare&& comp) { - assert(length >= 12); - const auto frac = length <= 1024 - ? length / 12 - : length <= 128 * 1024 ? length / 64 : length / 1024; - auto pivot = frac / 2; - const auto lo = length / 2 - pivot, hi = lo + frac; - assert(lo >= frac * 4); - assert(length - hi >= frac * 4); - assert(lo / 2 >= pivot); - const auto gap = (length - 9 * frac) / 4; - auto a = lo - 4 * frac - gap, b = hi + gap; - for (size_t i = lo; i < hi; ++i, a += 3, b += 3) { - median_common_detail::ninther(r, a, i - frac, b, a + 1, i, b + 1, a + 2, - i + frac, b + 2, comp); - } - - adaptiveQuickselect(r + lo, pivot, frac, comp); - return median_common_detail::expandPartition(r, lo, lo + pivot, hi, length, - comp); -} - -/** -Quickselect driver for medianOfNinthers, medianOfMinima, and medianOfMaxima. -Dispathes to each depending on the relationship between n (the sought order -statistics) and length. -*/ -template -void adaptiveQuickselect(Iter r, size_t n, size_t length, Compare&& comp) { - assert(n < length); - for (;;) { - // Decide strategy for partitioning - if (n == 0) { - // That would be the max - auto pivot = n; - for (++n; n < length; ++n) - if (comp(r[n], r[pivot])) pivot = n; - std::swap(r[0], r[pivot]); - return; - } - if (n + 1 == length) { - // That would be the min - auto pivot = 0; - for (n = 1; n < length; ++n) - if (comp(r[pivot], r[n])) pivot = n; - std::swap(r[pivot], r[length - 1]); - return; - } - assert(n < length); - size_t pivot; - if (length <= 16) - pivot = median_common_detail::pivotPartition(r, n, length, comp) - r; - else if (n * 6 <= length) - pivot = medianOfMinima(r, n, length, comp); - else if (n * 6 >= length * 5) - pivot = medianOfMaxima(r, n, length, comp); - else - pivot = medianOfNinthers(r, length, comp); - - // See how the pivot fares - if (pivot == n) { - return; - } - if (pivot > n) { - length = pivot; - } else { - ++pivot; - r += pivot; - length -= pivot; - n -= pivot; - } - } -} - -} // namespace median_of_ninthers_detail - -template -inline void median_of_ninthers_select(Iter begin, Iter mid, Iter end, - Compare comp) { - if (mid == end) return; - using CompType = typename median_common_detail::CompareRefType::type; - - median_of_ninthers_detail::adaptiveQuickselect( - begin, mid - begin, end - begin, comp); -} - -template -inline void median_of_ninthers_select(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - median_of_ninthers_select(begin, mid, end, std::less()); -} - -template -inline void median_of_ninthers_sort(Iter begin, Iter mid, Iter end, - Compare comp) { - if (begin == mid) return; - using CompType = typename median_common_detail::CompareRefType::type; - - median_of_ninthers_detail::adaptiveQuickselect( - begin, mid - begin - 1, end - begin, comp); - std::sort(begin, mid, comp); -} - -template -inline void median_of_ninthers_sort(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - median_of_ninthers_sort(begin, mid, end, std::less()); -} - -} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/pdqselect.h b/contrib/miniselect/include/miniselect/pdqselect.h deleted file mode 100644 index 0a22d059103..00000000000 --- a/contrib/miniselect/include/miniselect/pdqselect.h +++ /dev/null @@ -1,935 +0,0 @@ -/* - pdqsort.h - Pattern-defeating quicksort. - - Copyright (c) 2015 Orson Peters - - This software is provided 'as-is', without any express or implied warranty. - In no event will the authors be held liable for any damages arising from the - use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software in a - product, an acknowledgment in the product documentation would be appreciated - but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. -*/ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -// Adjusted by Danila Kutenin to support pdqselect and pdqpartial_sort. - -#ifndef PDQSORT_H -#define PDQSORT_H - -#include -#include -#include -#include -#include - -#if __cplusplus >= 201103L -#include -#include -#define PDQSORT_PREFER_MOVE(x) std::move(x) -#else -#define PDQSORT_PREFER_MOVE(x) (x) -#endif - -namespace miniselect { -namespace pdqsort_detail { - -template -struct CompareRefType { - // Pass the comparator by lvalue reference. Or in debug mode, using a - // debugging wrapper that stores a reference. - using type = typename std::add_lvalue_reference::type; -}; - -enum { - // Partitions below this size are sorted using insertion sort. - insertion_sort_threshold = 24, - - // Partitions above this size use Tukey's ninther to select the pivot. - ninther_threshold = 128, - - // When we detect an already sorted partition, attempt an insertion sort that - // allows this - // amount of element moves before giving up. - partial_insertion_sort_limit = 8, - - // Must be multiple of 8 due to loop unrolling, and < 256 to fit in unsigned - // char. - block_size = 64, - - // Cacheline size, assumes power of two. - cacheline_size = 64 - -}; - -#if __cplusplus >= 201103L -template -struct is_default_compare : std::false_type {}; -template -struct is_default_compare> : std::true_type {}; -template -struct is_default_compare> : std::true_type {}; -#endif - -// Returns floor(log2(n)), assumes n > 0. -template -inline int log2(T n) { - int log = 0; - while (n >>= 1) ++log; - return log; -} - -// Sorts [begin, end) using insertion sort with the given comparison function. -template -inline void insertion_sort(Iter begin, Iter end, Compare& comp) { - typedef typename std::iterator_traits::value_type T; - if (begin == end) return; - - for (Iter cur = begin + 1; cur != end; ++cur) { - Iter sift = cur; - Iter sift_1 = cur - 1; - - // Compare first so we can avoid 2 moves for an element already positioned - // correctly. - if (comp(*sift, *sift_1)) { - T tmp = PDQSORT_PREFER_MOVE(*sift); - - do { - *sift-- = PDQSORT_PREFER_MOVE(*sift_1); - } while (sift != begin && comp(tmp, *--sift_1)); - - *sift = PDQSORT_PREFER_MOVE(tmp); - } - } -} - -// Sorts [begin, end) using insertion sort with the given comparison function. -// Assumes -// *(begin - 1) is an element smaller than or equal to any element in [begin, -// end). -template -inline void unguarded_insertion_sort(Iter begin, Iter end, Compare& comp) { - typedef typename std::iterator_traits::value_type T; - if (begin == end) return; - - for (Iter cur = begin + 1; cur != end; ++cur) { - Iter sift = cur; - Iter sift_1 = cur - 1; - - // Compare first so we can avoid 2 moves for an element already positioned - // correctly. - if (comp(*sift, *sift_1)) { - T tmp = PDQSORT_PREFER_MOVE(*sift); - - do { - *sift-- = PDQSORT_PREFER_MOVE(*sift_1); - } while (comp(tmp, *--sift_1)); - - *sift = PDQSORT_PREFER_MOVE(tmp); - } - } -} - -// Attempts to use insertion sort on [begin, end). Will return false if more -// than partial_insertion_sort_limit elements were moved, and abort sorting. -// Otherwise it will successfully sort and return true. -template -inline bool partial_insertion_sort(Iter begin, Iter end, Compare& comp) { - typedef typename std::iterator_traits::value_type T; - if (begin == end) return true; - - std::size_t limit = 0; - for (Iter cur = begin + 1; cur != end; ++cur) { - Iter sift = cur; - Iter sift_1 = cur - 1; - - // Compare first so we can avoid 2 moves for an element already positioned - // correctly. - if (comp(*sift, *sift_1)) { - T tmp = PDQSORT_PREFER_MOVE(*sift); - - do { - *sift-- = PDQSORT_PREFER_MOVE(*sift_1); - } while (sift != begin && comp(tmp, *--sift_1)); - - *sift = PDQSORT_PREFER_MOVE(tmp); - limit += cur - sift; - } - - if (limit > partial_insertion_sort_limit) return false; - } - - return true; -} - -template -inline void sort2(Iter a, Iter b, Compare& comp) { - if (comp(*b, *a)) std::iter_swap(a, b); -} - -// Sorts the elements *a, *b and *c using comparison function comp. -template -inline void sort3(Iter a, Iter b, Iter c, Compare& comp) { - sort2(a, b, comp); - sort2(b, c, comp); - sort2(a, b, comp); -} - -template -inline T* align_cacheline(T* p) { -#if defined(UINTPTR_MAX) && __cplusplus >= 201103L - std::uintptr_t ip = reinterpret_cast(p); -#else - std::size_t ip = reinterpret_cast(p); -#endif - ip = (ip + cacheline_size - 1) & -cacheline_size; - return reinterpret_cast(ip); -} - -template -inline void swap_offsets(Iter first, Iter last, unsigned char* offsets_l, - unsigned char* offsets_r, int num, bool use_swaps) { - typedef typename std::iterator_traits::value_type T; - if (use_swaps) { - // This case is needed for the descending distribution, where we need - // to have proper swapping for pdqsort to remain O(n). - for (int i = 0; i < num; ++i) { - std::iter_swap(first + offsets_l[i], last - offsets_r[i]); - } - } else if (num > 0) { - Iter l = first + offsets_l[0]; - Iter r = last - offsets_r[0]; - T tmp(PDQSORT_PREFER_MOVE(*l)); - *l = PDQSORT_PREFER_MOVE(*r); - for (int i = 1; i < num; ++i) { - l = first + offsets_l[i]; - *r = PDQSORT_PREFER_MOVE(*l); - r = last - offsets_r[i]; - *l = PDQSORT_PREFER_MOVE(*r); - } - *r = PDQSORT_PREFER_MOVE(tmp); - } -} - -// Partitions [begin, end) around pivot *begin using comparison function comp. -// Elements equal to the pivot are put in the right-hand partition. Returns the -// position of the pivot after partitioning and whether the passed sequence -// already was correctly partitioned. Assumes the pivot is a median of at least -// 3 elements and that [begin, end) is at least insertion_sort_threshold long. -// Uses branchless partitioning. -template -inline std::pair partition_right_branchless(Iter begin, Iter end, - Compare& comp) { - typedef typename std::iterator_traits::value_type T; - - // Move pivot into local for speed. - T pivot(PDQSORT_PREFER_MOVE(*begin)); - Iter first = begin; - Iter last = end; - - // Find the first element greater than or equal than the pivot (the median of - // 3 guarantees this exists). - while (comp(*++first, pivot)) - ; - - // Find the first element strictly smaller than the pivot. We have to guard - // this search if there was no element before *first. - if (first - 1 == begin) - while (first < last && !comp(*--last, pivot)) - ; - else - while (!comp(*--last, pivot)) - ; - - // If the first pair of elements that should be swapped to partition are the - // same element, the passed in sequence already was correctly partitioned. - bool already_partitioned = first >= last; - if (!already_partitioned) { - std::iter_swap(first, last); - ++first; - } - - // The following branchless partitioning is derived from "BlockQuicksort: How - // Branch Mispredictions don’t affect Quicksort" by Stefan Edelkamp and Armin - // Weiss. - unsigned char offsets_l_storage[block_size + cacheline_size]; - unsigned char offsets_r_storage[block_size + cacheline_size]; - unsigned char* offsets_l = align_cacheline(offsets_l_storage); - unsigned char* offsets_r = align_cacheline(offsets_r_storage); - int num_l, num_r, start_l, start_r; - num_l = num_r = start_l = start_r = 0; - - while (last - first > 2 * block_size) { - // Fill up offset blocks with elements that are on the wrong side. - if (num_l == 0) { - start_l = 0; - Iter it = first; - for (unsigned char i = 0; i < block_size;) { - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - } - } - if (num_r == 0) { - start_r = 0; - Iter it = last; - for (unsigned char i = 0; i < block_size;) { - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - } - } - - // Swap elements and update block sizes and first/last boundaries. - int num = std::min(num_l, num_r); - swap_offsets(first, last, offsets_l + start_l, offsets_r + start_r, num, - num_l == num_r); - num_l -= num; - num_r -= num; - start_l += num; - start_r += num; - if (num_l == 0) first += block_size; - if (num_r == 0) last -= block_size; - } - - int l_size = 0, r_size = 0; - int unknown_left = (int)(last - first) - ((num_r || num_l) ? block_size : 0); - if (num_r) { - // Handle leftover block by assigning the unknown elements to the other - // block. - l_size = unknown_left; - r_size = block_size; - } else if (num_l) { - l_size = block_size; - r_size = unknown_left; - } else { - // No leftover block, split the unknown elements in two blocks. - l_size = unknown_left / 2; - r_size = unknown_left - l_size; - } - - // Fill offset buffers if needed. - if (unknown_left && !num_l) { - start_l = 0; - Iter it = first; - for (unsigned char i = 0; i < l_size;) { - offsets_l[num_l] = i++; - num_l += !comp(*it, pivot); - ++it; - } - } - if (unknown_left && !num_r) { - start_r = 0; - Iter it = last; - for (unsigned char i = 0; i < r_size;) { - offsets_r[num_r] = ++i; - num_r += comp(*--it, pivot); - } - } - - int num = std::min(num_l, num_r); - swap_offsets(first, last, offsets_l + start_l, offsets_r + start_r, num, - num_l == num_r); - num_l -= num; - num_r -= num; - start_l += num; - start_r += num; - if (num_l == 0) first += l_size; - if (num_r == 0) last -= r_size; - - // We have now fully identified [first, last)'s proper position. Swap the last - // elements. - if (num_l) { - offsets_l += start_l; - while (num_l--) std::iter_swap(first + offsets_l[num_l], --last); - first = last; - } - if (num_r) { - offsets_r += start_r; - while (num_r--) std::iter_swap(last - offsets_r[num_r], first), ++first; - last = first; - } - - // Put the pivot in the right place. - Iter pivot_pos = first - 1; - *begin = PDQSORT_PREFER_MOVE(*pivot_pos); - *pivot_pos = PDQSORT_PREFER_MOVE(pivot); - - return std::make_pair(pivot_pos, already_partitioned); -} - -// Partitions [begin, end) around pivot *begin using comparison function comp. -// Elements equal to the pivot are put in the right-hand partition. Returns the -// position of the pivot after partitioning and whether the passed sequence -// already was correctly partitioned. Assumes the pivot is a median of at least -// 3 elements and that [begin, end) is at least insertion_sort_threshold long. -template -inline std::pair partition_right(Iter begin, Iter end, - Compare& comp) { - typedef typename std::iterator_traits::value_type T; - - // Move pivot into local for speed. - T pivot(PDQSORT_PREFER_MOVE(*begin)); - - Iter first = begin; - Iter last = end; - - // Find the first element greater than or equal than the pivot (the median of - // 3 guarantees this exists). - while (comp(*++first, pivot)) - ; - - // Find the first element strictly smaller than the pivot. We have to guard - // this search if there was no element before *first. - if (first - 1 == begin) - while (first < last && !comp(*--last, pivot)) - ; - else - while (!comp(*--last, pivot)) - ; - - // If the first pair of elements that should be swapped to partition are the - // same element, the passed in sequence already was correctly partitioned. - bool already_partitioned = first >= last; - - // Keep swapping pairs of elements that are on the wrong side of the pivot. - // Previously swapped pairs guard the searches, which is why the first - // iteration is special-cased above. - while (first < last) { - std::iter_swap(first, last); - while (comp(*++first, pivot)) - ; - while (!comp(*--last, pivot)) - ; - } - - // Put the pivot in the right place. - Iter pivot_pos = first - 1; - *begin = PDQSORT_PREFER_MOVE(*pivot_pos); - *pivot_pos = PDQSORT_PREFER_MOVE(pivot); - - return std::make_pair(pivot_pos, already_partitioned); -} - -// Similar function to the one above, except elements equal to the pivot are put -// to the left of the pivot and it doesn't check or return if the passed -// sequence already was partitioned. Since this is rarely used (the many equal -// case), and in that case pdqsort already has O(n) performance, no block -// quicksort is applied here for simplicity. -template -inline Iter partition_left(Iter begin, Iter end, Compare& comp) { - typedef typename std::iterator_traits::value_type T; - - T pivot(PDQSORT_PREFER_MOVE(*begin)); - Iter first = begin; - Iter last = end; - - while (comp(pivot, *--last)) - ; - - if (last + 1 == end) - while (first < last && !comp(pivot, *++first)) - ; - else - while (!comp(pivot, *++first)) - ; - - while (first < last) { - std::iter_swap(first, last); - while (comp(pivot, *--last)) - ; - while (!comp(pivot, *++first)) - ; - } - - Iter pivot_pos = last; - *begin = PDQSORT_PREFER_MOVE(*pivot_pos); - *pivot_pos = PDQSORT_PREFER_MOVE(pivot); - - return pivot_pos; -} - -template -inline void pdqsort_loop(Iter begin, Iter end, Compare& comp, int bad_allowed, - bool leftmost = true) { - typedef typename std::iterator_traits::difference_type diff_t; - - // Use a while loop for tail recursion elimination. - while (true) { - diff_t size = end - begin; - - // Insertion sort is faster for small arrays. - if (size < insertion_sort_threshold) { - if (leftmost) - insertion_sort(begin, end, comp); - else - unguarded_insertion_sort(begin, end, comp); - return; - } - - // Choose pivot as median of 3 or pseudomedian of 9. - diff_t s2 = size / 2; - if (size > ninther_threshold) { - sort3(begin, begin + s2, end - 1, comp); - sort3(begin + 1, begin + (s2 - 1), end - 2, comp); - sort3(begin + 2, begin + (s2 + 1), end - 3, comp); - sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); - std::iter_swap(begin, begin + s2); - } else - sort3(begin + s2, begin, end - 1, comp); - - // If *(begin - 1) is the end of the right partition of a previous partition - // operation there is no element in [begin, end) that is smaller than - // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change - // strategy, putting equal elements in the left partition, greater elements - // in the right partition. We do not have to recurse on the left partition, - // since it's sorted (all equal). - if (!leftmost && !comp(*(begin - 1), *begin)) { - begin = partition_left(begin, end, comp) + 1; - continue; - } - - // Partition and get results. - std::pair part_result = - Branchless ? partition_right_branchless(begin, end, comp) - : partition_right(begin, end, comp); - Iter pivot_pos = part_result.first; - bool already_partitioned = part_result.second; - - // Check for a highly unbalanced partition. - diff_t l_size = pivot_pos - begin; - diff_t r_size = end - (pivot_pos + 1); - bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; - - // If we got a highly unbalanced partition we shuffle elements to break many - // patterns. - if (highly_unbalanced) { - // If we had too many bad partitions, switch to heapsort to guarantee O(n - // log n). - if (--bad_allowed == 0) { - std::make_heap(begin, end, comp); - std::sort_heap(begin, end, comp); - return; - } - - if (l_size >= insertion_sort_threshold) { - std::iter_swap(begin, begin + l_size / 4); - std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); - - if (l_size > ninther_threshold) { - std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); - std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); - std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); - std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); - } - } - - if (r_size >= insertion_sort_threshold) { - std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); - std::iter_swap(end - 1, end - r_size / 4); - - if (r_size > ninther_threshold) { - std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); - std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); - std::iter_swap(end - 2, end - (1 + r_size / 4)); - std::iter_swap(end - 3, end - (2 + r_size / 4)); - } - } - } else { - // If we were decently balanced and we tried to sort an already - // partitioned sequence try to use insertion sort. - if (already_partitioned && - partial_insertion_sort(begin, pivot_pos, comp) && - partial_insertion_sort(pivot_pos + 1, end, comp)) - return; - } - - // Sort the left partition first using recursion and do tail recursion - // elimination for the right-hand partition. - pdqsort_loop(begin, pivot_pos, comp, bad_allowed, - leftmost); - begin = pivot_pos + 1; - leftmost = false; - } -} - -template -inline void pdqpartial_sort_loop(Iter begin, Iter mid, Iter end, Compare& comp, - int bad_allowed, bool leftmost = true) { - typedef typename std::iterator_traits::difference_type diff_t; - - // Use a while loop for tail recursion elimination. - while (true) { - diff_t size = end - begin; - - // Insertion sort is faster for small arrays. - if (size < insertion_sort_threshold) { - if (leftmost) - insertion_sort(begin, end, comp); - else - unguarded_insertion_sort(begin, end, comp); - return; - } - - // Choose pivot as median of 3 or pseudomedian of 9. - diff_t s2 = size / 2; - if (size > ninther_threshold) { - sort3(begin, begin + s2, end - 1, comp); - sort3(begin + 1, begin + (s2 - 1), end - 2, comp); - sort3(begin + 2, begin + (s2 + 1), end - 3, comp); - sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); - std::iter_swap(begin, begin + s2); - } else - sort3(begin + s2, begin, end - 1, comp); - - // If *(begin - 1) is the end of the right partition of a previous partition - // operation there is no element in [begin, end) that is smaller than - // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change - // strategy, putting equal elements in the left partition, greater elements - // in the right partition. We do not have to recurse on the left partition, - // since it's sorted (all equal). - if (!leftmost && !comp(*(begin - 1), *begin)) { - begin = partition_left(begin, end, comp) + 1; - continue; - } - - // Partition and get results. - std::pair part_result = - Branchless ? partition_right_branchless(begin, end, comp) - : partition_right(begin, end, comp); - Iter pivot_pos = part_result.first; - bool already_partitioned = part_result.second; - - // Check for a highly unbalanced partition. - diff_t l_size = pivot_pos - begin; - diff_t r_size = end - (pivot_pos + 1); - bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; - - // If we got a highly unbalanced partition we shuffle elements to break many - // patterns. - if (highly_unbalanced) { - // If we had too many bad partitions, switch to heapsort to guarantee O(n - // log n). - if (--bad_allowed == 0) { - std::make_heap(begin, end, comp); - std::sort_heap(begin, end, comp); - return; - } - - if (l_size >= insertion_sort_threshold) { - std::iter_swap(begin, begin + l_size / 4); - std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); - - if (l_size > ninther_threshold) { - std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); - std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); - std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); - std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); - } - } - - if (r_size >= insertion_sort_threshold) { - std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); - std::iter_swap(end - 1, end - r_size / 4); - - if (r_size > ninther_threshold) { - std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); - std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); - std::iter_swap(end - 2, end - (1 + r_size / 4)); - std::iter_swap(end - 3, end - (2 + r_size / 4)); - } - } - } else { - // If we were decently balanced and we tried to sort an already - // partitioned sequence try to use insertion sort. - if (already_partitioned && - partial_insertion_sort(begin, pivot_pos, comp) && - partial_insertion_sort(pivot_pos + 1, end, comp)) - return; - } - - // Sort the left partition first using recursion and do tail recursion - // elimination for the right-hand partition. - if (pivot_pos < mid) { - pdqsort_loop(begin, pivot_pos, comp, - bad_allowed, leftmost); - begin = pivot_pos + 1; - leftmost = false; - } else { - end = pivot_pos; - } - } -} - -template -inline void pdqselect_loop(Iter begin, Iter mid, Iter end, Compare& comp, - int bad_allowed, bool leftmost = true) { - typedef typename std::iterator_traits::difference_type diff_t; - - // Use a while loop for tail recursion elimination. - while (true) { - diff_t size = end - begin; - - // Insertion sort is faster for small arrays. - if (size < insertion_sort_threshold) { - if (leftmost) - insertion_sort(begin, end, comp); - else - unguarded_insertion_sort(begin, end, comp); - return; - } - - // Choose pivot as median of 3 or pseudomedian of 9. - diff_t s2 = size / 2; - if (size > ninther_threshold) { - sort3(begin, begin + s2, end - 1, comp); - sort3(begin + 1, begin + (s2 - 1), end - 2, comp); - sort3(begin + 2, begin + (s2 + 1), end - 3, comp); - sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); - std::iter_swap(begin, begin + s2); - } else - sort3(begin + s2, begin, end - 1, comp); - - // If *(begin - 1) is the end of the right partition of a previous partition - // operation there is no element in [begin, end) that is smaller than - // *(begin - 1). Then if our pivot compares equal to *(begin - 1) we change - // strategy, putting equal elements in the left partition, greater elements - // in the right partition. We do not have to recurse on the left partition, - // since it's sorted (all equal). - if (!leftmost && !comp(*(begin - 1), *begin)) { - begin = partition_left(begin, end, comp) + 1; - continue; - } - - // Partition and get results. - std::pair part_result = - Branchless ? partition_right_branchless(begin, end, comp) - : partition_right(begin, end, comp); - Iter pivot_pos = part_result.first; - bool already_partitioned = part_result.second; - - // Check for a highly unbalanced partition. - diff_t l_size = pivot_pos - begin; - diff_t r_size = end - (pivot_pos + 1); - bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; - - // If we got a highly unbalanced partition we shuffle elements to break many - // patterns. - if (highly_unbalanced) { - // If we had too many bad partitions, switch to heapsort to guarantee O(n - // log n). - if (--bad_allowed == 0) { - std::nth_element(begin, mid, end, comp); - return; - } - - if (l_size >= insertion_sort_threshold) { - std::iter_swap(begin, begin + l_size / 4); - std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); - - if (l_size > ninther_threshold) { - std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); - std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); - std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); - std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); - } - } - - if (r_size >= insertion_sort_threshold) { - std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); - std::iter_swap(end - 1, end - r_size / 4); - - if (r_size > ninther_threshold) { - std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); - std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); - std::iter_swap(end - 2, end - (1 + r_size / 4)); - std::iter_swap(end - 3, end - (2 + r_size / 4)); - } - } - } else { - // If we were decently balanced and we tried to sort an already - // partitioned sequence try to use insertion sort. - if (already_partitioned && - partial_insertion_sort(begin, pivot_pos, comp) && - partial_insertion_sort(pivot_pos + 1, end, comp)) - return; - } - // Sort the left partition first using recursion and do tail recursion - // elimination for the right-hand partition. - if (pivot_pos < mid) { - begin = pivot_pos + 1; - leftmost = false; - } else { - end = pivot_pos; - } - } -} -} // namespace pdqsort_detail - -template -inline void pdqsort(Iter begin, Iter end, Compare comp) { - if (begin == end) return; - -#if __cplusplus >= 201103L - pdqsort_detail::pdqsort_loop< - Iter, Compare, - pdqsort_detail::is_default_compare< - typename std::decay::type>::value && - std::is_arithmetic< - typename std::iterator_traits::value_type>::value>( - begin, end, comp, pdqsort_detail::log2(end - begin)); -#else - pdqsort_detail::pdqsort_loop( - begin, end, comp, pdqsort_detail::log2(end - begin)); -#endif -} - -template -inline void pdqsort(Iter begin, Iter end) { - typedef typename std::iterator_traits::value_type T; - pdqsort(begin, end, std::less()); -} - -template -inline void pdqsort_branchless(Iter begin, Iter end, Compare comp) { - if (begin == end) return; - pdqsort_detail::pdqsort_loop( - begin, end, comp, pdqsort_detail::log2(end - begin)); -} - -template -inline void pdqsort_branchless(Iter begin, Iter end) { - typedef typename std::iterator_traits::value_type T; - pdqsort_branchless(begin, end, std::less()); -} - -template -inline void pdqpartial_sort(Iter begin, Iter mid, Iter end, Compare comp) { - if (begin == end) return; - -#if __cplusplus >= 201103L - pdqsort_detail::pdqpartial_sort_loop< - Iter, Compare, - pdqsort_detail::is_default_compare< - typename std::decay::type>::value && - std::is_arithmetic< - typename std::iterator_traits::value_type>::value>( - begin, mid, end, comp, pdqsort_detail::log2(end - begin)); -#else - pdqsort_detail::pdqpartial_sort_loop( - begin, end, comp, pdqsort_detail::log2(end - begin)); -#endif -} - -template -inline void pdqpartial_sort(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - pdqpartial_sort(begin, mid, end, std::less()); -} - -template -inline void pdqpartial_sort_branchless(Iter begin, Iter mid, Iter end, - Compare comp) { - if (begin == end) return; - pdqsort_detail::pdqpartial_sort_loop( - begin, mid, end, comp, pdqsort_detail::log2(end - begin)); -} - -template -inline void pdqpartial_sort_branchless(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - pdqpartial_sort_branchless(begin, mid, end, std::less()); -} - -template -inline void pdqselect(Iter begin, Iter mid, Iter end, Compare comp) { - if (mid == end) return; - using CompType = typename median_common_detail::CompareRefType::type; - -#if __cplusplus >= 201103L - pdqsort_detail::pdqselect_loop< - Iter, CompType, - pdqsort_detail::is_default_compare< - typename std::decay::type>::value && - std::is_arithmetic< - typename std::iterator_traits::value_type>::value>( - begin, mid, end, comp, pdqsort_detail::log2(end - begin)); -#else - pdqsort_detail::pdqselect_loop( - begin, end, comp, pdqsort_detail::log2(end - begin)); -#endif -} - -template -inline void pdqselect(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - pdqselect(begin, mid, end, std::less()); -} - -template -inline void pdqselect_branchless(Iter begin, Iter mid, Iter end, Compare comp) { - if (mid == end) return; - using CompType = typename median_common_detail::CompareRefType::type; - pdqsort_detail::pdqselect_loop( - begin, mid, end, comp, pdqsort_detail::log2(end - begin)); -} - -template -inline void pdqselect_branchless(Iter begin, Iter mid, Iter end) { - typedef typename std::iterator_traits::value_type T; - pdqselect_branchless(begin, mid, end, std::less()); -} - -#undef PDQSORT_PREFER_MOVE - -#endif - -} // namespace miniselect diff --git a/contrib/miniselect/include/miniselect/private/median_common.h b/contrib/miniselect/include/miniselect/private/median_common.h deleted file mode 100644 index 30cb1323bbf..00000000000 --- a/contrib/miniselect/include/miniselect/private/median_common.h +++ /dev/null @@ -1,437 +0,0 @@ -/* Copyright Andrei Alexandrescu, 2016-, - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#pragma once - -#include -#include -#include - -namespace miniselect { -namespace median_common_detail { - -template -struct CompareRefType { - // Pass the comparator by lvalue reference. Or in debug mode, using a - // debugging wrapper that stores a reference. - using type = typename std::add_lvalue_reference::type; -}; -/** -Swaps the median of r[a], r[b], and r[c] into r[b]. -*/ -template -void median3(Iter r, size_t a, size_t b, size_t c, Compare&& comp) { - if (comp(r[b], r[a])) // b < a - { - if (comp(r[b], r[c])) // b < a, b < c - { - if (comp(r[c], r[a])) // b < c < a - std::swap(r[b], r[c]); - else // b < a <= c - std::swap(r[b], r[a]); - } - } else if (comp(r[c], r[b])) // a <= b, c < b - { - if (comp(r[c], r[a])) // c < a <= b - std::swap(r[b], r[a]); - else // a <= c < b - std::swap(r[b], r[c]); - } -} - -/** -Sorts in place r[a], r[b], and r[c]. -*/ -template -void sort3(Iter r, size_t a, size_t b, size_t c, Compare&& comp) { - typedef typename std::iterator_traits::value_type T; - if (comp(r[b], r[a])) // b < a - { - if (comp(r[c], r[b])) // c < b < a - { - std::swap(r[a], r[c]); // a < b < c - } else // b < a, b <= c - { - T t = std::move(r[a]); - r[a] = std::move(r[b]); - if (comp(r[c], t)) // b <= c < a - { - r[b] = std::move(r[c]); - r[c] = std::move(t); - } else // b < a <= c - { - r[b] = std::move(t); - } - } - } else if (comp(r[c], r[b])) // a <= b, c < b - { - T t = std::move(r[c]); - r[c] = std::move(r[b]); - if (comp(t, r[a])) // c < a < b - { - r[b] = std::move(r[a]); - r[a] = std::move(t); - } else // a <= c < b - { - r[b] = std::move(t); - } - } - - assert(!comp(r[b], r[a]) && !comp(r[c], r[b])); -} - -/** -If leanRight == false, swaps the lower median of r[a]...r[d] into r[b] and -the minimum into r[a]. If leanRight == true, swaps the upper median of -r[a]...r[d] into r[c] and the minimum into r[d]. -*/ -template -void partition4(Iter r, size_t a, size_t b, size_t c, size_t d, - Compare&& comp) { - assert(a != b && a != c && a != d && b != c && b != d && c != d); - /* static */ if (leanRight) { - // In the median of 5 algorithm, consider r[e] infinite - if (comp(r[c], r[a])) { - std::swap(r[a], r[c]); - } // a <= c - if (comp(r[d], r[b])) { - std::swap(r[b], r[d]); - } // a <= c, b <= d - if (comp(r[d], r[c])) { - std::swap(r[c], r[d]); // a <= d, b <= c < d - std::swap(r[a], r[b]); // b <= d, a <= c < d - } // a <= c <= d, b <= d - if (comp(r[c], r[b])) { // a <= c <= d, c < b <= d - std::swap(r[b], r[c]); // a <= b <= c <= d - } // a <= b <= c <= d - } else { - // In the median of 5 algorithm consider r[a] infinitely small, then - // change b->a. c->b, d->c, e->d - if (comp(r[c], r[a])) { - std::swap(r[a], r[c]); - } - if (comp(r[c], r[b])) { - std::swap(r[b], r[c]); - } - if (comp(r[d], r[a])) { - std::swap(r[a], r[d]); - } - if (comp(r[d], r[b])) { - std::swap(r[b], r[d]); - } else { - if (comp(r[b], r[a])) { - std::swap(r[a], r[b]); - } - } - } -} - -/** -Places the median of r[a]...r[e] in r[c] and partitions the other elements -around it. -*/ -template -void partition5(Iter r, size_t a, size_t b, size_t c, size_t d, size_t e, - Compare&& comp) { - assert(a != b && a != c && a != d && a != e && b != c && b != d && b != e && - c != d && c != e && d != e); - if (comp(r[c], r[a])) { - std::swap(r[a], r[c]); - } - if (comp(r[d], r[b])) { - std::swap(r[b], r[d]); - } - if (comp(r[d], r[c])) { - std::swap(r[c], r[d]); - std::swap(r[a], r[b]); - } - if (comp(r[e], r[b])) { - std::swap(r[b], r[e]); - } - if (comp(r[e], r[c])) { - std::swap(r[c], r[e]); - if (comp(r[c], r[a])) { - std::swap(r[a], r[c]); - } - } else { - if (comp(r[c], r[b])) { - std::swap(r[b], r[c]); - } - } -} - -/** -Implements Hoare partition. -*/ -template -Iter pivotPartition(Iter r, size_t k, size_t length, Compare&& comp) { - assert(k < length); - std::swap(*r, r[k]); - size_t lo = 1, hi = length - 1; - for (;; ++lo, --hi) { - for (;; ++lo) { - if (lo > hi) goto loop_done; - if (!comp(r[lo], *r)) break; - } - // found the left bound: r[lo] >= r[0] - assert(lo <= hi); - for (; comp(*r, r[hi]); --hi) { - } - if (lo >= hi) break; - // found the right bound: r[hi] <= r[0], swap & make progress - std::swap(r[lo], r[hi]); - } -loop_done: - --lo; - std::swap(r[lo], *r); - return r + lo; -} - -/** -Implements the quickselect algorithm, parameterized with a partition function. -*/ -template -void quickselect(Iter r, Iter mid, Iter end, Compare&& comp) { - if (r == end || mid >= end) return; - assert(r <= mid && mid < end); - for (;;) switch (end - r) { - case 1: - return; - case 2: - if (comp(r[1], *r)) std::swap(*r, r[1]); - return; - case 3: - sort3(r, 0, 1, 2, comp); - return; - case 4: - switch (mid - r) { - case 0: - goto select_min; - case 1: - partition4(r, 0, 1, 2, 3, comp); - break; - case 2: - partition4(r, 0, 1, 2, 3, comp); - break; - case 3: - goto select_max; - default: - assert(false); - } - return; - default: - assert(end - r > 4); - if (r == mid) { - select_min: - auto pivot = r; - for (++mid; mid < end; ++mid) - if (comp(*mid, *pivot)) pivot = mid; - std::swap(*r, *pivot); - return; - } - if (mid + 1 == end) { - select_max: - auto pivot = r; - for (mid = r + 1; mid < end; ++mid) - if (comp(*pivot, *mid)) pivot = mid; - std::swap(*pivot, end[-1]); - return; - } - auto pivot = partition(r, end, comp); - if (pivot == mid) return; - if (mid < pivot) { - end = pivot; - } else { - r = pivot + 1; - } - } -} - -/** -Returns the index of the median of r[a], r[b], and r[c] without writing -anything. -*/ -template -size_t medianIndex(const Iter r, size_t a, size_t b, size_t c, Compare&& comp) { - if (r[a] > r[c]) std::swap(a, c); - if (r[b] > r[c]) return c; - if (comp(r[b], r[a])) return a; - return b; -} - -/** -Returns the index of the median of r[a], r[b], r[c], and r[d] without writing -anything. If leanRight is true, computes the upper median. Otherwise, conputes -the lower median. -*/ -template -static size_t medianIndex(Iter r, size_t a, size_t b, size_t c, size_t d, - Compare&& comp) { - if (comp(r[d], r[c])) std::swap(c, d); - assert(r[c] <= r[d]); - /* static */ if (leanRight) { - if (comp(r[c], r[a])) { - assert(comp(r[c], r[a]) && !comp(r[d], r[c])); // so r[c]) is out - return medianIndex(r, a, b, d, comp); - } - } else { - if (!comp(r[d], r[a])) { - return medianIndex(r, a, b, c, comp); - } - } - // Could return medianIndex(r, b, c, d) but we already know r[c] <= r[d] - if (!comp(r[c], r[b])) return c; - if (comp(r[d], r[b])) return d; - return b; -} - -/** -Tukey's Ninther: compute the median of r[_1], r[_2], r[_3], then the median of -r[_4], r[_5], r[_6], then the median of r[_7], r[_8], r[_9], and then swap the -median of those three medians into r[_5]. -*/ -template -void ninther(Iter r, size_t _1, size_t _2, size_t _3, size_t _4, size_t _5, - size_t _6, size_t _7, size_t _8, size_t _9, Compare&& comp) { - _2 = medianIndex(r, _1, _2, _3, comp); - _8 = medianIndex(r, _7, _8, _9, comp); - if (comp(r[_8], r[_2])) std::swap(_2, _8); - if (comp(r[_6], r[_4])) std::swap(_4, _6); - // Here we know that r[_2] and r[_8] are the other two medians and that - // r[_2] <= r[_8]. We also know that r[_4] <= r[_6] - if (comp(r[_5], r[_4])) { - // r[_4] is the median of r[_4], r[_5], r[_6] - } else if (comp(r[_6], r[_5])) { - // r[_6] is the median of r[_4], r[_5], r[_6] - _4 = _6; - } else { - // Here we know r[_5] is the median of r[_4], r[_5], r[_6] - if (comp(r[_5], r[_2])) return std::swap(r[_5], r[_2]); - if (comp(r[_8], r[_5])) return std::swap(r[_5], r[_8]); - // This is the only path that returns with no swap - return; - } - // Here we know r[_4] is the median of r[_4], r[_5], r[_6] - if (comp(r[_4], r[_2])) - _4 = _2; - else if (comp(r[_8], r[_4])) - _4 = _8; - std::swap(r[_5], r[_4]); -} - -/** -Input assumptions: -(a) hi <= rite -(c) the range r[0 .. hi] contains elements no smaller than r[0] -Output guarantee: same as Hoare partition using r[0] as pivot. Returns the new -position of the pivot. -*/ -template -size_t expandPartitionRight(Iter r, size_t hi, size_t rite, Compare&& comp) { - size_t pivot = 0; - assert(pivot <= hi); - assert(hi <= rite); - // First loop: spend r[pivot .. hi] - for (; pivot < hi; --rite) { - if (rite == hi) goto done; - if (!comp(r[rite], r[0])) continue; - ++pivot; - std::swap(r[rite], r[pivot]); - } - // Second loop: make left and pivot meet - for (; rite > pivot; --rite) { - if (!comp(r[rite], r[0])) continue; - while (rite > pivot) { - ++pivot; - if (comp(r[0], r[pivot])) { - std::swap(r[rite], r[pivot]); - break; - } - } - } - -done: - std::swap(r[0], r[pivot]); - return pivot; -} - -/** -Input assumptions: -(a) lo > 0, lo <= pivot -(b) the range r[lo .. pivot] already contains elements no greater than r[pivot] -Output guarantee: Same as Hoare partition around r[pivot]. Returns the new -position of the pivot. -*/ -template -size_t expandPartitionLeft(Iter r, size_t lo, size_t pivot, Compare&& comp) { - assert(lo > 0 && lo <= pivot); - size_t left = 0; - const auto oldPivot = pivot; - for (; lo < pivot; ++left) { - if (left == lo) goto done; - if (!comp(r[oldPivot], r[left])) continue; - --pivot; - std::swap(r[left], r[pivot]); - } - // Second loop: make left and pivot meet - for (;; ++left) { - if (left == pivot) break; - if (!comp(r[oldPivot], r[left])) continue; - for (;;) { - if (left == pivot) goto done; - --pivot; - if (comp(r[pivot], r[oldPivot])) { - std::swap(r[left], r[pivot]); - break; - } - } - } - -done: - std::swap(r[oldPivot], r[pivot]); - return pivot; -} - -/** -Input assumptions: -(a) lo <= pivot, pivot < hi, hi <= length -(b) the range r[lo .. pivot] already contains elements no greater than -r[pivot] -(c) the range r[pivot .. hi] already contains elements no smaller than -r[pivot] -Output guarantee: Same as Hoare partition around r[pivot], returning the new -position of the pivot. -*/ -template -size_t expandPartition(Iter r, size_t lo, size_t pivot, size_t hi, - size_t length, Compare&& comp) { - assert(lo <= pivot && pivot < hi && hi <= length); - --hi; - --length; - size_t left = 0; - for (;; ++left, --length) { - for (;; ++left) { - if (left == lo) - return pivot + expandPartitionRight(r + pivot, hi - pivot, - length - pivot, comp); - if (comp(r[pivot], r[left])) break; - } - for (;; --length) { - if (length == hi) - return left + - expandPartitionLeft(r + left, lo - left, pivot - left, comp); - if (!comp(r[pivot], r[length])) break; - } - std::swap(r[left], r[length]); - } -} - -} // namespace median_common_detail -} // namespace miniselect diff --git a/contrib/miniselect/testing/test_common.h b/contrib/miniselect/testing/test_common.h deleted file mode 100644 index df0c179c840..00000000000 --- a/contrib/miniselect/testing/test_common.h +++ /dev/null @@ -1,180 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#pragma once - -#include - -#include - -#include "miniselect/floyd_rivest_select.h" -#include "miniselect/median_of_3_random.h" -#include "miniselect/median_of_medians.h" -#include "miniselect/median_of_ninthers.h" -#include "miniselect/pdqselect.h" - -namespace miniselect { -namespace algorithms { - -struct STD { - template - static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { - std::partial_sort(begin, mid, end, std::move(comp)); - } - - template - static void Sort(Iter begin, Iter mid, Iter end) { - std::partial_sort(begin, mid, end); - } - - template - static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { - std::nth_element(begin, mid, end, std::move(comp)); - } - - template - static void Select(Iter begin, Iter mid, Iter end) { - std::nth_element(begin, mid, end); - } -}; - -struct PDQ { - template - static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { - pdqpartial_sort(begin, mid, end, std::move(comp)); - } - - template - static void Sort(Iter begin, Iter mid, Iter end) { - pdqpartial_sort(begin, mid, end); - } - - template - static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { - pdqselect(begin, mid, end, std::move(comp)); - } - - template - static void Select(Iter begin, Iter mid, Iter end) { - pdqselect(begin, mid, end); - } -}; - -struct PDQBranchless { - template - static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { - pdqpartial_sort_branchless(begin, mid, end, std::move(comp)); - } - - template - static void Sort(Iter begin, Iter mid, Iter end) { - pdqpartial_sort_branchless(begin, mid, end); - } - - template - static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { - pdqselect_branchless(begin, mid, end, std::move(comp)); - } - - template - static void Select(Iter begin, Iter mid, Iter end) { - pdqselect_branchless(begin, mid, end); - } -}; - -struct FloydRivest { - template - static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { - floyd_rivest_partial_sort(begin, mid, end, std::move(comp)); - } - - template - static void Sort(Iter begin, Iter mid, Iter end) { - floyd_rivest_partial_sort(begin, mid, end); - } - - template - static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { - floyd_rivest_select(begin, mid, end, std::move(comp)); - } - - template - static void Select(Iter begin, Iter mid, Iter end) { - floyd_rivest_select(begin, mid, end); - } -}; - -struct MedianOfNinthers { - template - static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { - median_of_ninthers_sort(begin, mid, end, std::move(comp)); - } - - template - static void Sort(Iter begin, Iter mid, Iter end) { - median_of_ninthers_sort(begin, mid, end); - } - - template - static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { - median_of_ninthers_select(begin, mid, end, std::move(comp)); - } - - template - static void Select(Iter begin, Iter mid, Iter end) { - median_of_ninthers_select(begin, mid, end); - } -}; - -struct MedianOfMedians { - template - static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { - median_of_medians_sort(begin, mid, end, std::move(comp)); - } - - template - static void Sort(Iter begin, Iter mid, Iter end) { - median_of_medians_sort(begin, mid, end); - } - - template - static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { - median_of_medians_select(begin, mid, end, std::move(comp)); - } - - template - static void Select(Iter begin, Iter mid, Iter end) { - median_of_medians_select(begin, mid, end); - } -}; - -struct MedianOf3Random { - template - static void Sort(Iter begin, Iter mid, Iter end, Compare&& comp) { - median_of_3_random_sort(begin, mid, end, std::move(comp)); - } - - template - static void Sort(Iter begin, Iter mid, Iter end) { - median_of_3_random_sort(begin, mid, end); - } - - template - static void Select(Iter begin, Iter mid, Iter end, Compare&& comp) { - median_of_3_random_select(begin, mid, end, std::move(comp)); - } - - template - static void Select(Iter begin, Iter mid, Iter end) { - median_of_3_random_select(begin, mid, end); - } -}; - -using All = - ::testing::Types; - -} // namespace algorithms -} // namespace miniselect diff --git a/contrib/miniselect/testing/test_select.cpp b/contrib/miniselect/testing/test_select.cpp deleted file mode 100644 index 9b8e9dce970..00000000000 --- a/contrib/miniselect/testing/test_select.cpp +++ /dev/null @@ -1,231 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#include -#include - -#include -#include -#include -#include -#include - -#include "test_common.h" - -using ::testing::Eq; - -namespace miniselect { -namespace { - -struct IndirectLess { - // Non const comparator with deleted copy. - template - bool operator()(const P &x, const P &y) const { - return *x < *y; - } - IndirectLess(const IndirectLess &) = default; - IndirectLess &operator=(const IndirectLess &) = default; - IndirectLess(IndirectLess &&) = default; - IndirectLess &operator=(IndirectLess &&) = default; -}; - -template -class SelectTest : public ::testing::Test { - public: - using Base = Selector; - - static void TestSelects(size_t N, size_t M) { - ASSERT_NE(N, 0); - ASSERT_GT(N, M); - SCOPED_TRACE(N); - SCOPED_TRACE(M); - std::vector array(N); - for (size_t i = 0; i < N; ++i) { - array[i] = i; - } - auto array_smaller = array; - std::mt19937_64 mersenne_engine; - std::shuffle(array.begin(), array.end(), mersenne_engine); - Selector::Select(array.begin(), array.begin() + M, array.end(), - std::greater()); - EXPECT_EQ(array[M], N - M - 1); - for (size_t i = 0; i < M; ++i) { - EXPECT_GE(array[i], array[M]); - } - for (size_t i = M; i < N; ++i) { - EXPECT_LE(array[i], array[M]); - } - std::shuffle(array_smaller.begin(), array_smaller.end(), mersenne_engine); - Selector::Select(array_smaller.begin(), array_smaller.begin() + M, - array_smaller.end()); - EXPECT_EQ(array_smaller[M], M); - for (size_t i = 0; i < M; ++i) { - EXPECT_LE(array_smaller[i], array_smaller[M]); - } - for (size_t i = M; i < N; ++i) { - EXPECT_GE(array_smaller[i], array_smaller[M]); - } - } - - static void TestSelects(size_t N) { - TestSelects(N, 0); - TestSelects(N, 1); - TestSelects(N, 2); - TestSelects(N, 3); - TestSelects(N, N / 2 - 1); - TestSelects(N, N / 2); - TestSelects(N, N / 2 + 1); - TestSelects(N, N - 2); - TestSelects(N, N - 1); - } - - static void TestManySelects() { - TestSelects(10); - TestSelects(256); - TestSelects(257); - TestSelects(499); - TestSelects(500); - TestSelects(997); - TestSelects(1000); - TestSelects(1000 * 100); - TestSelects(1009); - TestSelects(1009 * 109); - } - - static void TestCustomComparators() { - std::vector> v(1000); - for (int i = 0; static_cast(i) < v.size(); ++i) { - v[i] = std::make_unique(i); - } - Selector::Select(v.begin(), v.begin() + v.size() / 2, v.end(), - IndirectLess{}); - EXPECT_EQ(*v[v.size() / 2], v.size() / 2); - for (size_t i = 0; i < v.size() / 2; ++i) { - ASSERT_NE(v[i], nullptr); - EXPECT_LE(*v[i], v.size() / 2); - } - for (size_t i = v.size() / 2; i < v.size(); ++i) { - ASSERT_NE(v[i], nullptr); - EXPECT_GE(*v[i], v.size() / 2); - } - } - - static void TestRepeat(size_t N, size_t M) { - ASSERT_NE(N, 0); - ASSERT_GT(N, M); - SCOPED_TRACE(N); - SCOPED_TRACE(M); - std::mt19937_64 mersenne_engine(10); - std::vector array(N); - for (size_t i = 0; i < M; ++i) { - array[i] = false; - } - for (size_t i = M; i < N; ++i) { - array[i] = true; - } - std::shuffle(array.begin(), array.end(), mersenne_engine); - Selector::Select(array.begin(), array.begin() + M, array.end()); - EXPECT_EQ(array[M], true); - for (size_t i = 0; i < M; ++i) { - EXPECT_EQ(array[i], false); - } - for (size_t i = M; i < N; ++i) { - EXPECT_EQ(array[i], true); - } - std::shuffle(array.begin(), array.end(), mersenne_engine); - Selector::Select(array.begin(), array.begin() + M / 2, array.end()); - EXPECT_EQ(array[M / 2], false); - for (size_t i = 0; i < M / 2; ++i) { - EXPECT_EQ(array[i], false); - } - std::shuffle(array.begin(), array.end(), mersenne_engine); - Selector::Select(array.begin(), array.begin() + M - 1, array.end()); - EXPECT_EQ(array[M - 1], false); - for (size_t i = 0; i < M - 1; ++i) { - EXPECT_EQ(array[i], false); - } - } - - static void TestRepeats(size_t N) { - TestRepeat(N, 1); - TestRepeat(N, 2); - TestRepeat(N, 3); - TestRepeat(N, N / 2 - 1); - TestRepeat(N, N / 2); - TestRepeat(N, N / 2 + 1); - TestRepeat(N, N - 2); - TestRepeat(N, N - 1); - } - - static void TestManyRepeats() { - TestRepeats(10); - TestRepeats(100); - TestRepeats(257); - TestRepeats(1000); - TestRepeats(100000); - } -}; - -TYPED_TEST_SUITE(SelectTest, algorithms::All); - -TYPED_TEST(SelectTest, TestSmall) { - std::vector v = {"ab", "aaa", "ab"}; - TypeParam::Select(v.begin(), v.begin() + 1, v.end()); - EXPECT_THAT(v, Eq(std::vector{"aaa", "ab", "ab"})); - v = {"aba"}; - TypeParam::Select(v.begin(), v.begin(), v.end()); - EXPECT_THAT(v, Eq(std::vector{"aba"})); - v.clear(); - TypeParam::Select(v.begin(), v.end(), v.end()); - EXPECT_TRUE(v.empty()); -} - -TYPED_TEST(SelectTest, TestAnotherSmall) { - std::vector v = {"ab", "ab", "aaa"}; - TypeParam::Select(v.begin(), v.begin() + 1, v.end()); - EXPECT_THAT(v, Eq(std::vector{"aaa", "ab", "ab"})); -} - -TYPED_TEST(SelectTest, TestEmptySmall) { - std::vector v = {"", ""}; - TypeParam::Select(v.begin(), v.begin() + 1, v.end()); - EXPECT_THAT(v, Eq(std::vector{"", ""})); -} - -TYPED_TEST(SelectTest, TestBasic) { TestFixture::TestManySelects(); } - -TYPED_TEST(SelectTest, TestComparators) { - TestFixture::TestCustomComparators(); -} - -TYPED_TEST(SelectTest, TestRepeats) { TestFixture::TestManyRepeats(); } - -TYPED_TEST(SelectTest, TestLast) { - std::vector array(100); - for (size_t i = 0; i < 100; ++i) { - array[i] = i; - } - auto array_smaller = array; - std::mt19937_64 mersenne_engine; - std::shuffle(array.begin(), array.end(), mersenne_engine); - auto copy_array = array; - // Should be no effect. - size_t cmp = 0; - TypeParam::Select(array.begin(), array.end(), array.end(), - [&cmp](const auto &lhs, const auto &rhs) { - ++cmp; - return lhs < rhs; - }); - EXPECT_EQ(cmp, 0); - EXPECT_EQ(copy_array, array); -} - -} // namespace -} // namespace miniselect - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/contrib/miniselect/testing/test_sort.cpp b/contrib/miniselect/testing/test_sort.cpp deleted file mode 100644 index 19c6ff036fe..00000000000 --- a/contrib/miniselect/testing/test_sort.cpp +++ /dev/null @@ -1,161 +0,0 @@ -/* Copyright Danila Kutenin, 2020-. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * https://boost.org/LICENSE_1_0.txt) - */ -#include -#include - -#include -#include -#include -#include -#include - -#include "test_common.h" - -using ::testing::Eq; - -namespace miniselect { -namespace { - -struct IndirectLess { - // Non const comparator with deleted copy. - template - bool operator()(const P &x, const P &y) const { - return *x < *y; - } - IndirectLess(const IndirectLess &) = default; - IndirectLess &operator=(const IndirectLess &) = default; - IndirectLess(IndirectLess &&) = default; - IndirectLess &operator=(IndirectLess &&) = default; -}; - -template -class PartialSortTest : public ::testing::Test { - public: - static void TestSorts(size_t N, size_t M) { - ASSERT_NE(N, 0); - ASSERT_GE(N, M); - SCOPED_TRACE(N); - SCOPED_TRACE(M); - std::vector array(N); - for (size_t i = 0; i < N; ++i) { - array[i] = i; - } - auto array_smaller = array; - std::mt19937_64 mersenne_engine; - std::shuffle(array.begin(), array.end(), mersenne_engine); - Sorter::Sort(array.begin(), array.begin() + M, array.end(), - std::greater()); - for (size_t i = 0; i < M; ++i) { - EXPECT_EQ(array[i], N - i - 1); - } - std::shuffle(array_smaller.begin(), array_smaller.end(), mersenne_engine); - Sorter::Sort(array_smaller.begin(), array_smaller.begin() + M, - array_smaller.end()); - for (size_t i = 0; i < M; ++i) { - EXPECT_EQ(array_smaller[i], i); - } - } - - static void TestSorts(size_t N) { - TestSorts(N, 0); - TestSorts(N, 1); - TestSorts(N, 2); - TestSorts(N, 3); - TestSorts(N, N / 2 - 1); - TestSorts(N, N / 2); - TestSorts(N, N / 2 + 1); - TestSorts(N, N - 2); - TestSorts(N, N - 1); - TestSorts(N, N); - } - - static void TestManySorts() { - TestSorts(10); - TestSorts(256); - TestSorts(257); - TestSorts(499); - TestSorts(500); - TestSorts(997); - TestSorts(1000); - TestSorts(1000 * 100); - TestSorts(1009); - TestSorts(1009 * 109); - } - - static void TestCustomComparators() { - std::vector> v(1000); - for (int i = 0; static_cast(i) < v.size(); ++i) { - v[i] = std::make_unique(i); - } - Sorter::Sort(v.begin(), v.begin() + v.size() / 2, v.end(), IndirectLess{}); - for (int i = 0; static_cast(i) < v.size() / 2; ++i) { - ASSERT_NE(v[i], nullptr); - EXPECT_EQ(*v[i], i); - } - } -}; - -TYPED_TEST_SUITE(PartialSortTest, algorithms::All); - -TYPED_TEST(PartialSortTest, TestSmall) { - std::vector v = {"ab", "aaa", "ab"}; - TypeParam::Sort(v.begin(), v.begin() + 1, v.end()); - EXPECT_THAT(v, Eq(std::vector{"aaa", "ab", "ab"})); - v = {"aba"}; - TypeParam::Sort(v.begin(), v.begin(), v.end()); - EXPECT_THAT(v, Eq(std::vector{"aba"})); - v.clear(); - TypeParam::Sort(v.begin(), v.end(), v.end()); - EXPECT_TRUE(v.empty()); -} - -TYPED_TEST(PartialSortTest, TestAnotherSmall) { - std::vector v = {"ab", "ab", "aaa"}; - TypeParam::Sort(v.begin(), v.begin() + 1, v.end()); - EXPECT_THAT(v, Eq(std::vector{"aaa", "ab", "ab"})); -} - -TYPED_TEST(PartialSortTest, TestEmptySmall) { - std::vector v = {"", ""}; - TypeParam::Sort(v.begin(), v.begin() + 1, v.end()); - EXPECT_THAT(v, Eq(std::vector{"", ""})); -} - -TYPED_TEST(PartialSortTest, TestBasic) { TestFixture::TestManySorts(); } - -TYPED_TEST(PartialSortTest, TestComparators) { - TestFixture::TestCustomComparators(); -} - -// The standard says that the order of other elements is unspecified even if -// nothing should be sorted so it fails for libcxx and PDQ which is Ok. Saving -// this test for a reference. -TYPED_TEST(PartialSortTest, DISABLED_TestEmpty) { - std::vector array(100); - for (size_t i = 0; i < 100; ++i) { - array[i] = i; - } - std::mt19937_64 mersenne_engine; - std::shuffle(array.begin(), array.end(), mersenne_engine); - size_t cmp = 0; - auto copy_array = array; - // Should be no effect. - TypeParam::Sort(array.begin(), array.begin(), array.end(), - [&cmp](const auto &lhs, const auto &rhs) { - ++cmp; - return lhs < rhs; - }); - EXPECT_EQ(cmp, 0); - EXPECT_EQ(copy_array, array); -} - -} // namespace -} // namespace miniselect - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/src/AggregateFunctions/QuantileExact.h b/src/AggregateFunctions/QuantileExact.h index 3f5a0907126..e3d1ae5d8b8 100644 --- a/src/AggregateFunctions/QuantileExact.h +++ b/src/AggregateFunctions/QuantileExact.h @@ -8,6 +8,7 @@ #include #include +#include namespace DB { @@ -87,7 +88,7 @@ struct QuantileExact : QuantileExactBase> { size_t n = level < 1 ? level * array.size() : (array.size() - 1); - std::nth_element(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm. + miniselect::floyd_rivest_partial_select(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm. return array[n]; } @@ -107,7 +108,7 @@ struct QuantileExact : QuantileExactBase> size_t n = level < 1 ? level * array.size() : (array.size() - 1); - std::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n, array.end()); result[indices[i]] = array[n]; prev_n = n; @@ -144,7 +145,7 @@ struct QuantileExactExclusive : public QuantileExact else if (n < 1) return static_cast(array[0]); - std::nth_element(array.begin(), array.begin() + n - 1, array.end()); + miniselect::floyd_rivest_select(array.begin(), array.begin() + n - 1, array.end()); auto nth_element = std::min_element(array.begin() + n, array.end()); return static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -173,7 +174,7 @@ struct QuantileExactExclusive : public QuantileExact result[indices[i]] = static_cast(array[0]); else { - std::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); + miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n - 1, array.end()); auto nth_element = std::min_element(array.begin() + n, array.end()); result[indices[i]] = static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -209,7 +210,7 @@ struct QuantileExactInclusive : public QuantileExact else if (n < 1) return static_cast(array[0]); - std::nth_element(array.begin(), array.begin() + n - 1, array.end()); + miniselect::floyd_rivest_select(array.begin(), array.begin() + n - 1, array.end()); auto nth_element = std::min_element(array.begin() + n, array.end()); return static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -236,7 +237,7 @@ struct QuantileExactInclusive : public QuantileExact result[indices[i]] = static_cast(array[0]); else { - std::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); + miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n - 1, array.end()); auto nth_element = std::min_element(array.begin() + n, array.end()); result[indices[i]] = static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); diff --git a/src/AggregateFunctions/QuantileTiming.h b/src/AggregateFunctions/QuantileTiming.h index 2ab8c866615..28bcde5c140 100644 --- a/src/AggregateFunctions/QuantileTiming.h +++ b/src/AggregateFunctions/QuantileTiming.h @@ -7,6 +7,7 @@ #include #include +#include namespace DB { @@ -179,7 +180,7 @@ namespace detail /// Sorting an array will not be considered a violation of constancy. auto & array = elems; - std::nth_element(array.begin(), array.begin() + n, array.end()); + miniselect::floyd_rivest_select(array.begin(), array.begin() + n, array.end()); quantile = array[n]; } @@ -200,7 +201,7 @@ namespace detail ? level * elems.size() : (elems.size() - 1); - std::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n, array.end()); result[level_index] = array[n]; prev_n = n; diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index b27506c1cfb..4285259a4f4 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -163,10 +163,10 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum { const auto& [first, last] = equal_ranges[i]; if (reverse) - miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, + std::sort(res.begin() + first, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else - miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, + std::sort(res.begin() + first, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); auto new_first = first; From a4e000cc9f0877451864016ef1607a78c6a44de7 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 00:55:17 +0300 Subject: [PATCH 074/114] Fix name --- src/AggregateFunctions/QuantileExact.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/QuantileExact.h b/src/AggregateFunctions/QuantileExact.h index e3d1ae5d8b8..de909c27565 100644 --- a/src/AggregateFunctions/QuantileExact.h +++ b/src/AggregateFunctions/QuantileExact.h @@ -88,7 +88,7 @@ struct QuantileExact : QuantileExactBase> { size_t n = level < 1 ? level * array.size() : (array.size() - 1); - miniselect::floyd_rivest_partial_select(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm. + miniselect::floyd_rivest_select(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm. return array[n]; } From 5d138f3475a78a1eccf4b1b92c68344b5120e344 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 01:31:24 +0300 Subject: [PATCH 075/114] Fix submodules for fast test --- docker/test/fasttest/run.sh | 2 +- utils/check-style/check-include | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index eb737948aa6..36aa8baf252 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -127,7 +127,7 @@ function clone_submodules ( cd "$FASTTEST_SOURCE" -SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring) +SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring contrib/miniselect) git submodule sync git submodule update --init --recursive "${SUBMODULES_TO_UPDATE[@]}" diff --git a/utils/check-style/check-include b/utils/check-style/check-include index 35f94d6e706..fc88f1b1b9f 100755 --- a/utils/check-style/check-include +++ b/utils/check-style/check-include @@ -19,6 +19,7 @@ inc="-I. \ -I./contrib/double-conversion \ -I./contrib/cityhash102/include \ -I./contrib/croaring \ +-I./contrib/miniselect \ -I./contrib/murmurhash/include \ -I./contrib/zookeeper/src/c/include \ -I./contrib/zookeeper/src/c/generated \ From 811c3e5cd18a19200edcf720b70754fc0536c07b Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 01:32:50 +0300 Subject: [PATCH 076/114] Fix submodules --- utils/check-style/check-include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check-style/check-include b/utils/check-style/check-include index fc88f1b1b9f..b4f105ed0cb 100755 --- a/utils/check-style/check-include +++ b/utils/check-style/check-include @@ -19,7 +19,7 @@ inc="-I. \ -I./contrib/double-conversion \ -I./contrib/cityhash102/include \ -I./contrib/croaring \ --I./contrib/miniselect \ +-I./contrib/miniselect/include \ -I./contrib/murmurhash/include \ -I./contrib/zookeeper/src/c/include \ -I./contrib/zookeeper/src/c/generated \ From 258d0549cdbdc029984f841ca020e806181c453c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 10 Nov 2020 01:34:19 +0300 Subject: [PATCH 077/114] Update adding_test_queries.md --- docs/en/development/adding_test_queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/adding_test_queries.md b/docs/en/development/adding_test_queries.md index 710f6467ce7..a9f47d53024 100644 --- a/docs/en/development/adding_test_queries.md +++ b/docs/en/development/adding_test_queries.md @@ -25,7 +25,7 @@ Why/when you should add a test case into ClickHouse code: I assume you run some Linux machine (you can use docker / virtual machines on other OS) and any modern browser / internet connection, and you have some basic Linux & SQL skills. -Any highly specialized knowledge are not needed (so you don't need to know C++ or know something about how ClickHouse CI works). +Any highly specialized knowledge is not needed (so you don't need to know C++ or know something about how ClickHouse CI works). #### Preparation From a000a6b41a78d46294fbd9a68fc35100e8536f3e Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 10 Nov 2020 01:37:00 +0300 Subject: [PATCH 078/114] Update adding_test_queries.md --- docs/en/development/adding_test_queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/adding_test_queries.md b/docs/en/development/adding_test_queries.md index a9f47d53024..4770d48ebd4 100644 --- a/docs/en/development/adding_test_queries.md +++ b/docs/en/development/adding_test_queries.md @@ -123,7 +123,7 @@ clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee te - use `number` / `numbers_mt` / `zeros` / `zeros_mt` and similar for queries / to initialize data when appliable - clean up the created objects after test and before the test (DROP IF EXISTS) - in case of some dirty state - prefer sync mode of operations (mutations, merges, etc.) -- use other SQL files in the `0_stateless` folder as a reference +- use other SQL files in the `0_stateless` folder as an example - ensure the feature / feature combination you want to tests is not covered yet with existsing tests #### Commit / push / create PR. From 26bc0a07440fa04bb54268f4e485904ad5adf8ef Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2020 06:21:51 +0300 Subject: [PATCH 079/114] Update version_date.tsv after release 20.6.9.1 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index a6ca642b985..0e55de496b7 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -10,6 +10,7 @@ v20.8.2.3-stable 2020-09-08 v20.7.4.11-stable 2020-10-09 v20.7.3.7-stable 2020-09-18 v20.7.2.30-stable 2020-08-31 +v20.6.9.1-stable 2020-11-10 v20.6.8.5-stable 2020-10-12 v20.6.7.4-stable 2020-09-18 v20.6.6.7-stable 2020-09-11 From ca51cf9235316ffee17af7cfec5b7e2a611bc3d5 Mon Sep 17 00:00:00 2001 From: fenglv Date: Tue, 10 Nov 2020 07:20:50 +0000 Subject: [PATCH 080/114] Make member function const to prevent -readability-make-member-function-const --- src/Interpreters/TreeRewriter.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 8d3cb123955..51609cbaa23 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -63,7 +63,7 @@ struct CustomizeFunctionsData const String & customized_func_name; - void visit(ASTFunction & func, ASTPtr &) + void visit(ASTFunction & func, ASTPtr &) const { if (Poco::toLower(func.name) == func_name) { @@ -97,7 +97,7 @@ struct CustomizeFunctionsSuffixData const String & customized_func_suffix; - void visit(ASTFunction & func, ASTPtr &) + void visit(ASTFunction & func, ASTPtr &) const { if (endsWith(Poco::toLower(func.name), func_suffix)) { @@ -118,7 +118,7 @@ struct CustomizeAggregateFunctionsSuffixData const String & customized_func_suffix; - void visit(ASTFunction & func, ASTPtr &) + void visit(ASTFunction & func, ASTPtr &) const { const auto & instance = AggregateFunctionFactory::instance(); if (instance.isAggregateFunctionName(func.name) && !endsWith(func.name, customized_func_suffix)) From c0ae61e3061968267d7f8341e5af3823189acfdf Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 10 Nov 2020 10:33:31 +0300 Subject: [PATCH 081/114] Fix stress test script --- docker/test/stress/stress | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 874dca751f3..458f78fcdb4 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -17,13 +17,6 @@ def get_skip_list_cmd(path): return '' -def run_perf_test(cmd, xmls_path, output_folder): - output_path = os.path.join(output_folder, "perf_stress_run.txt") - f = open(output_path, 'w') - p = Popen("{} --skip-tags=long --recursive --input-files {}".format(cmd, xmls_path), shell=True, stdout=f, stderr=f) - return p - - def get_options(i): options = "" if 0 < i: @@ -75,8 +68,6 @@ if __name__ == "__main__": args = parser.parse_args() func_pipes = [] - perf_process = None - perf_process = run_perf_test(args.perf_test_cmd, args.perf_test_xml_path, args.output_folder) func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit) logging.info("Will wait functests to finish") From 435f410aaa393667f6d107e0054db705d679c740 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 10:35:27 +0300 Subject: [PATCH 082/114] Restart C From aa8e6db786fd7007e71e0071cde9fe9018987045 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 3 Nov 2020 15:52:31 +0800 Subject: [PATCH 083/114] reload auxiliary zookeepers configuration --- programs/server/Server.cpp | 2 + src/Interpreters/Context.cpp | 42 +++++++-- src/Interpreters/Context.h | 3 + .../__init__.py | 0 .../configs/config.xml | 31 +++++++ .../configs/users.xml | 23 +++++ .../configs/zookeeper.xml | 17 ++++ .../test_reload_auxiliary_zookeepers/test.py | 89 +++++++++++++++++++ 8 files changed, 200 insertions(+), 7 deletions(-) create mode 100644 tests/integration/test_reload_auxiliary_zookeepers/__init__.py create mode 100644 tests/integration/test_reload_auxiliary_zookeepers/configs/config.xml create mode 100644 tests/integration/test_reload_auxiliary_zookeepers/configs/users.xml create mode 100644 tests/integration/test_reload_auxiliary_zookeepers/configs/zookeeper.xml create mode 100644 tests/integration/test_reload_auxiliary_zookeepers/test.py diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ed18793a537..a481bc020d8 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -568,6 +568,8 @@ int Server::main(const std::vector & /*args*/) if (config->has("zookeeper")) global_context->reloadZooKeeperIfChanged(config); + global_context->reloadAuxiliaryZooKeepersConfigIfChanged(config); + global_context->updateStorageConfiguration(*config); }, /* already_loaded = */ true); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index a22eacd9a49..73604f93d12 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -302,9 +302,11 @@ struct ContextShared mutable std::mutex zookeeper_mutex; mutable zkutil::ZooKeeperPtr zookeeper; /// Client for ZooKeeper. + ConfigurationPtr zookeeper_config; /// Stores zookeeper configs mutable std::mutex auxiliary_zookeepers_mutex; mutable std::map auxiliary_zookeepers; /// Map for auxiliary ZooKeeper clients. + ConfigurationPtr auxiliary_zookeepers_config; /// Stores auxiliary zookeepers configs String interserver_io_host; /// The host name by which this server is available for other servers. UInt16 interserver_io_port = 0; /// and port. @@ -364,8 +366,7 @@ struct ContextShared /// Initialized on demand (on distributed storages initialization) since Settings should be initialized std::unique_ptr clusters; ConfigurationPtr clusters_config; /// Stores updated configs - ConfigurationPtr zookeeper_config; /// Stores zookeeper configs - mutable std::mutex clusters_mutex; /// Guards clusters and clusters_config + mutable std::mutex clusters_mutex; /// Guards clusters and clusters_config #if USE_EMBEDDED_COMPILER std::shared_ptr compiled_expression_cache; @@ -1498,10 +1499,16 @@ zkutil::ZooKeeperPtr Context::getAuxiliaryZooKeeper(const String & name) const auto zookeeper = shared->auxiliary_zookeepers.find(name); if (zookeeper == shared->auxiliary_zookeepers.end()) { - if (!getConfigRef().has("auxiliary_zookeepers." + name)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown auxiliary ZooKeeper name '{}'. If it's required it can be added to the section in config.xml", name); + const auto & config = shared->auxiliary_zookeepers_config ? *shared->auxiliary_zookeepers_config : getConfigRef(); + if (!config.has("auxiliary_zookeepers." + name)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unknown auxiliary ZooKeeper name '{}'. If it's required it can be added to the section in " + "config.xml", + name); - zookeeper->second = std::make_shared(getConfigRef(), "auxiliary_zookeepers." + name); + zookeeper + = shared->auxiliary_zookeepers.emplace(name, std::make_shared(config, "auxiliary_zookeepers." + name)).first; } else if (zookeeper->second->expired()) zookeeper->second = zookeeper->second->startNewSession(); @@ -1515,17 +1522,38 @@ void Context::resetZooKeeper() const shared->zookeeper.reset(); } +static void reloadZooKeeperIfChangedImpl(const ConfigurationPtr & config, const std::string & config_name, zkutil::ZooKeeperPtr & zk) +{ + if (!zk || zk->configChanged(*config, config_name)) + zk = std::make_shared(*config, config_name); +} + void Context::reloadZooKeeperIfChanged(const ConfigurationPtr & config) const { std::lock_guard lock(shared->zookeeper_mutex); shared->zookeeper_config = config; + reloadZooKeeperIfChangedImpl(config, "zookeeper", shared->zookeeper); +} - if (!shared->zookeeper || shared->zookeeper->configChanged(*config, "zookeeper")) +void Context::reloadAuxiliaryZooKeepersConfigIfChanged(const ConfigurationPtr & config) +{ + std::lock_guard lock(shared->auxiliary_zookeepers_mutex); + + shared->auxiliary_zookeepers_config = config; + + for (auto it = shared->auxiliary_zookeepers.begin(); it != shared->auxiliary_zookeepers.end();) { - shared->zookeeper = std::make_shared(*config, "zookeeper"); + if (!config->has("auxiliary_zookeepers." + it->first)) + it = shared->auxiliary_zookeepers.erase(it); + else + { + reloadZooKeeperIfChangedImpl(config, "auxiliary_zookeepers." + it->first, it->second); + ++it; + } } } + bool Context::hasZooKeeper() const { return getConfigRef().has("zookeeper"); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index ec42880309f..a2b61b29514 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -487,6 +487,9 @@ public: std::shared_ptr getZooKeeper() const; /// Same as above but return a zookeeper connection from auxiliary_zookeepers configuration entry. std::shared_ptr getAuxiliaryZooKeeper(const String & name) const; + + /// Set auxiliary zookeepers configuration at server starting or configuration reloading. + void reloadAuxiliaryZooKeepersConfigIfChanged(const ConfigurationPtr & config); /// Has ready or expired ZooKeeper bool hasZooKeeper() const; /// Reset current zookeeper session. Do not create a new one. diff --git a/tests/integration/test_reload_auxiliary_zookeepers/__init__.py b/tests/integration/test_reload_auxiliary_zookeepers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_reload_auxiliary_zookeepers/configs/config.xml b/tests/integration/test_reload_auxiliary_zookeepers/configs/config.xml new file mode 100644 index 00000000000..b5e5495c096 --- /dev/null +++ b/tests/integration/test_reload_auxiliary_zookeepers/configs/config.xml @@ -0,0 +1,31 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + + 1 + 1 + diff --git a/tests/integration/test_reload_auxiliary_zookeepers/configs/users.xml b/tests/integration/test_reload_auxiliary_zookeepers/configs/users.xml new file mode 100644 index 00000000000..6061af8e33d --- /dev/null +++ b/tests/integration/test_reload_auxiliary_zookeepers/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/tests/integration/test_reload_auxiliary_zookeepers/configs/zookeeper.xml b/tests/integration/test_reload_auxiliary_zookeepers/configs/zookeeper.xml new file mode 100644 index 00000000000..8157cc1d30b --- /dev/null +++ b/tests/integration/test_reload_auxiliary_zookeepers/configs/zookeeper.xml @@ -0,0 +1,17 @@ + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + zoo3 + 2181 + + 2000 + + diff --git a/tests/integration/test_reload_auxiliary_zookeepers/test.py b/tests/integration/test_reload_auxiliary_zookeepers/test.py new file mode 100644 index 00000000000..92c66c890fc --- /dev/null +++ b/tests/integration/test_reload_auxiliary_zookeepers/test.py @@ -0,0 +1,89 @@ +import time +import pytest +import os + +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml") +node = cluster.add_instance("node", with_zookeeper=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_reload_auxiliary_zookeepers(start_cluster): + + node.query( + "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" + ) + node.query("INSERT INTO simple VALUES ('2020-08-27', 1)") + + node.query( + "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" + ) + + # Add an auxiliary zookeeper + new_config = """ + + + zoo1 + 2181 + + + zoo2 + 2181 + + + zoo3 + 2181 + + 2000 + + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + +""" + node.replace_config("/etc/clickhouse-server/conf.d/zookeeper.xml", new_config) + + # Hopefully it has finished the configuration reload + time.sleep(2) + + node.query( + "ALTER TABLE simple2 FETCH PARTITION '2020-08-27' FROM 'zookeeper2:/clickhouse/tables/0/simple';" + ) + node.query("ALTER TABLE simple2 ATTACH PARTITION '2020-08-27';") + assert node.query("SELECT id FROM simple2").strip() == "1" + + new_config = """ + + + zoo2 + 2181 + + 2000 + +""" + node.replace_config("/etc/clickhouse-server/conf.d/zookeeper.xml", new_config) + time.sleep(2) + with pytest.raises(QueryRuntimeException): + node.query( + "ALTER TABLE simple2 FETCH PARTITION '2020-08-27' FROM 'zookeeper2:/clickhouse/tables/0/simple';" + ) + assert node.query("SELECT id FROM simple2").strip() == "1" From 59c0a739c4a3ff4d157f64b5837f57c7aa448895 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 Nov 2020 12:18:51 +0300 Subject: [PATCH 084/114] Add test to skiplist. --- src/Storages/IStorage.cpp | 1 - tests/queries/0_stateless/arcadia_skip_list.txt | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index ddd5b6727dc..3b04c6cd632 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -80,7 +80,6 @@ TableExclusiveLockHolder IStorage::lockExclusively(const String & query_id, cons return result; } - Pipe IStorage::read( const Names & /*column_names*/, const StorageMetadataPtr & /*metadata_snapshot*/, diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 4c4f7b29d66..6420eadfc09 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -159,9 +159,10 @@ 01533_collate_in_nullable 01542_collate_in_array 01543_collate_in_tuple +01545_url_file_format_settings 01546_log_queries_min_query_duration_ms 01547_query_log_current_database 01548_query_log_query_execution_ms 01552_dict_fixedstring 01555_system_distribution_queue_mask -01557_max_parallel_replicas_no_sample.sql +01557_max_parallel_replicas_no_sample.sql \ No newline at end of file From 363c1e05c0776b723970e1d15c0f6bdb6002b89e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 Nov 2020 12:35:05 +0300 Subject: [PATCH 085/114] Try fix tests. --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index f8220002b38..d9b201a0390 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1344,6 +1344,9 @@ QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( pipes.emplace_back(std::move(source_processor)); } + if (pipes.empty()) + continue; + auto pipe = Pipe::unitePipes(std::move(pipes)); /// Drop temporary columns, added by 'sorting_key_expr' From 059357d51eaff84d0790abadc65d938ee45202e1 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Tue, 10 Nov 2020 13:23:46 +0300 Subject: [PATCH 086/114] `ALTER UPDATE/DELETE ... IN PARTITION` with partition pruning in `ReplicatedMergeTree` (#13403) Co-authored-by: Alexander Kazakov --- src/Interpreters/MutationsInterpreter.cpp | 100 +++++++++++--- src/Interpreters/MutationsInterpreter.h | 14 +- src/Parsers/ASTAlterQuery.cpp | 24 +++- src/Parsers/ASTAlterQuery.h | 2 +- src/Parsers/ExpressionListParsers.cpp | 14 ++ src/Parsers/ExpressionListParsers.h | 9 +- src/Parsers/ParserAlterQuery.cpp | 19 ++- src/Parsers/ParserAlterQuery.h | 10 +- src/Storages/MergeTree/BoolMask.h | 12 +- .../MergeTree/EphemeralLockInZooKeeper.cpp | 14 +- .../MergeTree/EphemeralLockInZooKeeper.h | 69 +++++++++- .../ReplicatedMergeTreeMutationEntry.h | 5 +- .../MergeTree/StorageFromMergeTreeDataPart.h | 10 ++ src/Storages/MutationCommands.cpp | 7 +- src/Storages/MutationCommands.h | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 123 +++++++++++++----- src/Storages/StorageReplicatedMergeTree.h | 15 ++- .../__init__.py | 0 .../configs/cluster.xml | 16 +++ .../configs/logs_config.xml | 17 +++ .../test.py | 98 ++++++++++++++ 21 files changed, 493 insertions(+), 89 deletions(-) create mode 100644 tests/integration/test_mutations_in_partitions_of_merge_tree/__init__.py create mode 100644 tests/integration/test_mutations_in_partitions_of_merge_tree/configs/cluster.xml create mode 100644 tests/integration/test_mutations_in_partitions_of_merge_tree/configs/logs_config.xml create mode 100644 tests/integration/test_mutations_in_partitions_of_merge_tree/test.py diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 3e7ebfec139..66cce64cff7 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ namespace DB namespace ErrorCodes { + extern const int NOT_IMPLEMENTED; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int UNKNOWN_MUTATION_COMMAND; @@ -92,6 +94,7 @@ std::optional findFirstNonDeterministicFunctionName(const MutationComman if (finder_data.nondeterministic_function_name) return finder_data.nondeterministic_function_name; + /// Currently UPDATE and DELETE both always have predicates so we can use fallthrough [[fallthrough]]; } @@ -110,7 +113,7 @@ std::optional findFirstNonDeterministicFunctionName(const MutationComman return {}; } -ASTPtr prepareQueryAffectedAST(const std::vector & commands) +ASTPtr prepareQueryAffectedAST(const std::vector & commands, const StoragePtr & storage, const Context & context) { /// Execute `SELECT count() FROM storage WHERE predicate1 OR predicate2 OR ...` query. /// The result can differ from the number of affected rows (e.g. if there is an UPDATE command that @@ -125,20 +128,23 @@ ASTPtr prepareQueryAffectedAST(const std::vector & commands) count_func->arguments = std::make_shared(); select->select()->children.push_back(count_func); - if (commands.size() == 1) - select->setExpression(ASTSelectQuery::Expression::WHERE, commands[0].predicate->clone()); - else + ASTs conditions; + for (const MutationCommand & command : commands) { - auto coalesced_predicates = std::make_shared(); - coalesced_predicates->name = "or"; - coalesced_predicates->arguments = std::make_shared(); - coalesced_predicates->children.push_back(coalesced_predicates->arguments); - - for (const MutationCommand & command : commands) - coalesced_predicates->arguments->children.push_back(command.predicate->clone()); + if (ASTPtr condition = getPartitionAndPredicateExpressionForMutationCommand(command, storage, context)) + conditions.push_back(std::move(condition)); + } + if (conditions.size() > 1) + { + auto coalesced_predicates = makeASTFunction("or"); + coalesced_predicates->arguments->children = std::move(conditions); select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(coalesced_predicates)); } + else if (conditions.size() == 1) + { + select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(conditions.front())); + } return select; } @@ -167,8 +173,9 @@ ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_ } + bool isStorageTouchedByMutations( - StoragePtr storage, + const StoragePtr & storage, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, Context context_copy) @@ -176,16 +183,33 @@ bool isStorageTouchedByMutations( if (commands.empty()) return false; + bool all_commands_can_be_skipped = true; + auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast(storage); for (const MutationCommand & command : commands) { if (!command.predicate) /// The command touches all rows. return true; + + if (command.partition && !storage_from_merge_tree_data_part) + throw Exception("ALTER UPDATE/DELETE ... IN PARTITION is not supported for non-MergeTree tables", ErrorCodes::NOT_IMPLEMENTED); + + if (command.partition && storage_from_merge_tree_data_part) + { + const String partition_id = storage_from_merge_tree_data_part->getPartitionIDFromQuery(command.partition, context_copy); + if (partition_id == storage_from_merge_tree_data_part->getPartitionId()) + all_commands_can_be_skipped = false; + } + else + all_commands_can_be_skipped = false; } + if (all_commands_can_be_skipped) + return false; + context_copy.setSetting("max_streams_to_max_threads_ratio", 1); context_copy.setSetting("max_threads", 1); - ASTPtr select_query = prepareQueryAffectedAST(commands); + ASTPtr select_query = prepareQueryAffectedAST(commands, storage, context_copy); /// Interpreter must be alive, when we use result of execute() method. /// For some reason it may copy context and and give it into ExpressionBlockInputStream @@ -202,9 +226,42 @@ bool isStorageTouchedByMutations( auto count = (*block.getByName("count()").column)[0].get(); return count != 0; - } + +ASTPtr getPartitionAndPredicateExpressionForMutationCommand( + const MutationCommand & command, + const StoragePtr & storage, + const Context & context +) +{ + ASTPtr partition_predicate_as_ast_func; + if (command.partition) + { + String partition_id; + + auto storage_merge_tree = std::dynamic_pointer_cast(storage); + auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast(storage); + if (storage_merge_tree) + partition_id = storage_merge_tree->getPartitionIDFromQuery(command.partition, context); + else if (storage_from_merge_tree_data_part) + partition_id = storage_from_merge_tree_data_part->getPartitionIDFromQuery(command.partition, context); + else + throw Exception("ALTER UPDATE/DELETE ... IN PARTITION is not supported for non-MergeTree tables", ErrorCodes::NOT_IMPLEMENTED); + + partition_predicate_as_ast_func = makeASTFunction("equals", + std::make_shared("_partition_id"), + std::make_shared(partition_id) + ); + } + + if (command.predicate && command.partition) + return makeASTFunction("and", command.predicate->clone(), std::move(partition_predicate_as_ast_func)); + else + return command.predicate ? command.predicate->clone() : partition_predicate_as_ast_func; +} + + MutationsInterpreter::MutationsInterpreter( StoragePtr storage_, const StorageMetadataPtr & metadata_snapshot_, @@ -349,7 +406,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) if (stages.empty() || !stages.back().column_to_updated.empty()) stages.emplace_back(context); - auto negated_predicate = makeASTFunction("isZeroOrNull", command.predicate->clone()); + auto negated_predicate = makeASTFunction("isZeroOrNull", getPartitionAndPredicateExpressionForMutationCommand(command)); stages.back().filters.push_back(negated_predicate); } else if (command.type == MutationCommand::UPDATE) @@ -387,7 +444,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) const auto & update_expr = kv.second; auto updated_column = makeASTFunction("CAST", makeASTFunction("if", - command.predicate->clone(), + getPartitionAndPredicateExpressionForMutationCommand(command), makeASTFunction("CAST", update_expr->clone(), type_literal), @@ -592,7 +649,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & for (const String & column : stage.output_columns) all_asts->children.push_back(std::make_shared(column)); - auto syntax_result = TreeRewriter(context).analyze(all_asts, all_columns); + auto syntax_result = TreeRewriter(context).analyze(all_asts, all_columns, storage, metadata_snapshot); if (context.hasQueryContext()) for (const auto & it : syntax_result->getScalars()) context.getQueryContext().addScalar(it.first, it.second); @@ -759,10 +816,10 @@ const Block & MutationsInterpreter::getUpdatedHeader() const size_t MutationsInterpreter::evaluateCommandsSize() { for (const MutationCommand & command : commands) - if (unlikely(!command.predicate)) /// The command touches all rows. + if (unlikely(!command.predicate && !command.partition)) /// The command touches all rows. return mutation_ast->size(); - return std::max(prepareQueryAffectedAST(commands)->size(), mutation_ast->size()); + return std::max(prepareQueryAffectedAST(commands, storage, context)->size(), mutation_ast->size()); } std::optional MutationsInterpreter::getStorageSortDescriptionIfPossible(const Block & header) const @@ -783,6 +840,11 @@ std::optional MutationsInterpreter::getStorageSortDescriptionIf return sort_description; } +ASTPtr MutationsInterpreter::getPartitionAndPredicateExpressionForMutationCommand(const MutationCommand & command) const +{ + return DB::getPartitionAndPredicateExpressionForMutationCommand(command, storage, context); +} + bool MutationsInterpreter::Stage::isAffectingAllColumns(const Names & storage_columns) const { /// is subset diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 59d9e7657c3..18658e605ad 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -20,7 +20,17 @@ using QueryPipelinePtr = std::unique_ptr; /// Return false if the data isn't going to be changed by mutations. bool isStorageTouchedByMutations( - StoragePtr storage, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, Context context_copy); + const StoragePtr & storage, + const StorageMetadataPtr & metadata_snapshot, + const std::vector & commands, + Context context_copy +); + +ASTPtr getPartitionAndPredicateExpressionForMutationCommand( + const MutationCommand & command, + const StoragePtr & storage, + const Context & context +); /// Create an input stream that will read data from storage and apply mutation commands (UPDATEs, DELETEs, MATERIALIZEs) /// to this data. @@ -59,6 +69,8 @@ private: std::optional getStorageSortDescriptionIfPossible(const Block & header) const; + ASTPtr getPartitionAndPredicateExpressionForMutationCommand(const MutationCommand & command) const; + StoragePtr storage; StorageMetadataPtr metadata_snapshot; MutationCommands commands; diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 918bd4acaa7..d07e57eefae 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -90,7 +90,7 @@ void ASTAlterCommand::formatImpl( column->formatImpl(settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } } @@ -150,7 +150,7 @@ void ASTAlterCommand::formatImpl( index->formatImpl(settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } } @@ -161,7 +161,7 @@ void ASTAlterCommand::formatImpl( index->formatImpl(settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " IN PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } } @@ -272,7 +272,15 @@ void ASTAlterCommand::formatImpl( } else if (type == ASTAlterCommand::DELETE) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE WHERE " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE" << (settings.hilite ? hilite_none : ""); + + if (partition) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(settings, state, frame); + } + + settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); predicate->formatImpl(settings, state, frame); } else if (type == ASTAlterCommand::UPDATE) @@ -280,6 +288,12 @@ void ASTAlterCommand::formatImpl( settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "UPDATE " << (settings.hilite ? hilite_none : ""); update_assignments->formatImpl(settings, state, frame); + if (partition) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(settings, state, frame); + } + settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); predicate->formatImpl(settings, state, frame); } @@ -298,7 +312,7 @@ void ASTAlterCommand::formatImpl( << (settings.hilite ? hilite_none : ""); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str<< " IN PARTITION " << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } } diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 78e0c726ddf..65657e5ecfd 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -103,7 +103,7 @@ public: */ ASTPtr constraint; - /** Used in DROP PARTITION and ATTACH PARTITION FROM queries. + /** Used in DROP PARTITION, ATTACH PARTITION FROM, UPDATE, DELETE queries. * The value or ID of the partition is stored here. */ ASTPtr partition; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index ad03d949174..0f06a0d2480 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -55,6 +55,12 @@ const char * ParserComparisonExpression::operators[] = nullptr }; +const char * ParserComparisonExpression::overlapping_operators_to_skip[] = +{ + "IN PARTITION", + nullptr +}; + const char * ParserLogicalNotExpression::operators[] = { "NOT", "not", @@ -137,6 +143,14 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node /// try to find any of the valid operators const char ** it; + Expected stub; + for (it = overlapping_operators_to_skip; *it; ++it) + if (ParserKeyword{*it}.checkWithoutMoving(pos, stub)) + break; + + if (*it) + break; + for (it = operators; *it; it += 2) if (parseOperator(pos, *it, expected)) break; diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index cf77b8b4da4..4e21eff7f0e 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -82,6 +82,7 @@ class ParserLeftAssociativeBinaryOperatorList : public IParserBase { private: Operators_t operators; + Operators_t overlapping_operators_to_skip = { (const char *[]){ nullptr } }; ParserPtr first_elem_parser; ParserPtr remaining_elem_parser; @@ -93,6 +94,11 @@ public: { } + ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_) + : operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_), first_elem_parser(std::move(first_elem_parser_)) + { + } + ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, ParserPtr && first_elem_parser_, ParserPtr && remaining_elem_parser_) : operators(operators_), first_elem_parser(std::move(first_elem_parser_)), @@ -284,7 +290,8 @@ class ParserComparisonExpression : public IParserBase { private: static const char * operators[]; - ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique()}; + static const char * overlapping_operators_to_skip[]; + ParserLeftAssociativeBinaryOperatorList operator_parser {operators, overlapping_operators_to_skip, std::make_unique()}; protected: const char * getName() const override{ return "comparison expression"; } diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index bb682455a99..7050614007e 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -79,7 +79,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_to_volume("TO VOLUME"); ParserKeyword s_to_table("TO TABLE"); - ParserKeyword s_delete_where("DELETE WHERE"); + ParserKeyword s_delete("DELETE"); ParserKeyword s_update("UPDATE"); ParserKeyword s_where("WHERE"); ParserKeyword s_to("TO"); @@ -506,8 +506,17 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->type = ASTAlterCommand::MODIFY_SAMPLE_BY; } - else if (s_delete_where.ignore(pos, expected)) + else if (s_delete.ignore(pos, expected)) { + if (s_in_partition.ignore(pos, expected)) + { + if (!parser_partition.parse(pos, command->partition, expected)) + return false; + } + + if (!s_where.ignore(pos, expected)) + return false; + if (!parser_exp_elem.parse(pos, command->predicate, expected)) return false; @@ -518,6 +527,12 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected if (!parser_assignment_list.parse(pos, command->update_assignments, expected)) return false; + if (s_in_partition.ignore(pos, expected)) + { + if (!parser_partition.parse(pos, command->partition, expected)) + return false; + } + if (!s_where.ignore(pos, expected)) return false; diff --git a/src/Parsers/ParserAlterQuery.h b/src/Parsers/ParserAlterQuery.h index a0981c77ca6..514ef876430 100644 --- a/src/Parsers/ParserAlterQuery.h +++ b/src/Parsers/ParserAlterQuery.h @@ -10,7 +10,7 @@ namespace DB * ALTER TABLE [db.]name [ON CLUSTER cluster] * [ADD COLUMN [IF NOT EXISTS] col_name type [AFTER col_after],] * [DROP COLUMN [IF EXISTS] col_to_drop, ...] - * [CLEAR COLUMN [IF EXISTS] col_to_clear [IN PARTITION partition],] + * [CLEAR COLUMN [IF EXISTS] col_to_clear[ IN PARTITION partition],] * [MODIFY COLUMN [IF EXISTS] col_to_modify type, ...] * [RENAME COLUMN [IF EXISTS] col_name TO col_name] * [MODIFY PRIMARY KEY (a, b, c...)] @@ -19,8 +19,12 @@ namespace DB * [DROP|DETACH|ATTACH PARTITION|PART partition, ...] * [FETCH PARTITION partition FROM ...] * [FREEZE [PARTITION] [WITH NAME name]] - * [DELETE WHERE ...] - * [UPDATE col_name = expr, ... WHERE ...] + * [DELETE[ IN PARTITION partition] WHERE ...] + * [UPDATE col_name = expr, ...[ IN PARTITION partition] WHERE ...] + * [ADD INDEX [IF NOT EXISTS] index_name [AFTER index_name]] + * [DROP INDEX [IF EXISTS] index_name] + * [CLEAR INDEX [IF EXISTS] index_name IN PARTITION partition] + * [MATERIALIZE INDEX [IF EXISTS] index_name [IN PARTITION partition]] * ALTER LIVE VIEW [db.name] * [REFRESH] */ diff --git a/src/Storages/MergeTree/BoolMask.h b/src/Storages/MergeTree/BoolMask.h index 3538c581137..c26a0ed6c58 100644 --- a/src/Storages/MergeTree/BoolMask.h +++ b/src/Storages/MergeTree/BoolMask.h @@ -9,17 +9,17 @@ struct BoolMask BoolMask() {} BoolMask(bool can_be_true_, bool can_be_false_) : can_be_true(can_be_true_), can_be_false(can_be_false_) {} - BoolMask operator &(const BoolMask & m) + BoolMask operator &(const BoolMask & m) const { - return BoolMask(can_be_true && m.can_be_true, can_be_false || m.can_be_false); + return {can_be_true && m.can_be_true, can_be_false || m.can_be_false}; } - BoolMask operator |(const BoolMask & m) + BoolMask operator |(const BoolMask & m) const { - return BoolMask(can_be_true || m.can_be_true, can_be_false && m.can_be_false); + return {can_be_true || m.can_be_true, can_be_false && m.can_be_false}; } - BoolMask operator !() + BoolMask operator !() const { - return BoolMask(can_be_false, can_be_true); + return {can_be_false, can_be_true}; } /// If mask is (true, true), then it can no longer change under operation |. diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp index 6b00215fd26..1f194092f5f 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB @@ -71,13 +72,13 @@ EphemeralLockInZooKeeper::~EphemeralLockInZooKeeper() EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( const String & block_numbers_path, const String & path_prefix, const String & temp_path, zkutil::ZooKeeper & zookeeper_) - : zookeeper(zookeeper_) + : zookeeper(&zookeeper_) { std::vector holders; while (true) { Coordination::Stat partitions_stat; - Strings partitions = zookeeper.getChildren(block_numbers_path, &partitions_stat); + Strings partitions = zookeeper->getChildren(block_numbers_path, &partitions_stat); if (holders.size() < partitions.size()) { @@ -85,7 +86,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( for (size_t i = 0; i < partitions.size() - holders.size(); ++i) { String path = temp_path + "/abandonable_lock-"; - holder_futures.push_back(zookeeper.asyncCreate(path, {}, zkutil::CreateMode::EphemeralSequential)); + holder_futures.push_back(zookeeper->asyncCreate(path, {}, zkutil::CreateMode::EphemeralSequential)); } for (auto & future : holder_futures) { @@ -104,7 +105,7 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( lock_ops.push_back(zkutil::makeCheckRequest(block_numbers_path, partitions_stat.version)); Coordination::Responses lock_responses; - Coordination::Error rc = zookeeper.tryMulti(lock_ops, lock_responses); + Coordination::Error rc = zookeeper->tryMulti(lock_ops, lock_responses); if (rc == Coordination::Error::ZBADVERSION) { LOG_TRACE(&Poco::Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); @@ -131,13 +132,16 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( void EphemeralLocksInAllPartitions::unlock() { + if (!zookeeper) + return; + std::vector futures; for (const auto & lock : locks) { Coordination::Requests unlock_ops; unlock_ops.emplace_back(zkutil::makeRemoveRequest(lock.path, -1)); unlock_ops.emplace_back(zkutil::makeRemoveRequest(lock.holder_path, -1)); - futures.push_back(zookeeper.asyncMulti(unlock_ops)); + futures.push_back(zookeeper->asyncMulti(unlock_ops)); } for (auto & future : futures) diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.h b/src/Storages/MergeTree/EphemeralLockInZooKeeper.h index b85761e0b15..007768aea3a 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.h +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.h @@ -1,9 +1,14 @@ #pragma once +#include "ReplicatedMergeTreeMutationEntry.h" + #include #include #include +#include +#include + namespace DB { @@ -87,13 +92,30 @@ private: /// Acquires block number locks in all partitions. -class EphemeralLocksInAllPartitions : private boost::noncopyable +class EphemeralLocksInAllPartitions : public boost::noncopyable { public: EphemeralLocksInAllPartitions( const String & block_numbers_path, const String & path_prefix, const String & temp_path, zkutil::ZooKeeper & zookeeper_); + EphemeralLocksInAllPartitions() = default; + + EphemeralLocksInAllPartitions(EphemeralLocksInAllPartitions && rhs) noexcept + : zookeeper(rhs.zookeeper) + , locks(std::move(rhs.locks)) + { + rhs.zookeeper = nullptr; + } + + EphemeralLocksInAllPartitions & operator=(EphemeralLocksInAllPartitions && rhs) noexcept + { + zookeeper = rhs.zookeeper; + rhs.zookeeper = nullptr; + locks = std::move(rhs.locks); + return *this; + } + struct LockInfo { String path; @@ -110,8 +132,51 @@ public: ~EphemeralLocksInAllPartitions(); private: - zkutil::ZooKeeper & zookeeper; + zkutil::ZooKeeper * zookeeper = nullptr; std::vector locks; }; + +/// This class allows scoped manipulations with block numbers locked in certain partitions +/// See StorageReplicatedMergeTree::allocateBlockNumbersInAffectedPartitions and alter()/mutate() methods +class PartitionBlockNumbersHolder +{ +public: + PartitionBlockNumbersHolder(const PartitionBlockNumbersHolder &) = delete; + PartitionBlockNumbersHolder & operator=(const PartitionBlockNumbersHolder &) = delete; + + using BlockNumbersType = ReplicatedMergeTreeMutationEntry::BlockNumbersType; + + PartitionBlockNumbersHolder() = default; + PartitionBlockNumbersHolder( + BlockNumbersType block_numbers_, std::optional locked_block_numbers_holder) + : block_numbers(std::move(block_numbers_)) + , multiple_partitions_holder(std::move(locked_block_numbers_holder)) + { + } + PartitionBlockNumbersHolder( + BlockNumbersType block_numbers_, std::optional locked_block_numbers_holder) + : block_numbers(std::move(block_numbers_)) + , single_partition_holder(std::move(locked_block_numbers_holder)) + { + } + + PartitionBlockNumbersHolder & operator=(PartitionBlockNumbersHolder &&) = default; + + const BlockNumbersType & getBlockNumbers() const { return block_numbers; } + + void reset() + { + multiple_partitions_holder.reset(); + single_partition_holder.reset(); + block_numbers.clear(); + } + +private: + BlockNumbersType block_numbers; + + std::optional multiple_partitions_holder; + std::optional single_partition_holder; +}; + } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h index cfcc3dec6f2..6229b13c47f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h @@ -35,9 +35,10 @@ struct ReplicatedMergeTreeMutationEntry /// Replica which initiated mutation String source_replica; - /// Accured numbers of blocks + /// Acquired block numbers /// partition_id -> block_number - std::map block_numbers; + using BlockNumbersType = std::map; + BlockNumbersType block_numbers; /// Mutation commands which will give to MUTATE_PART entries MutationCommands commands; diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index aa24ddcf33c..4c171053d61 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -45,6 +45,16 @@ public: return part->storage.getVirtuals(); } + String getPartitionId() const + { + return part->info.partition_id; + } + + String getPartitionIDFromQuery(const ASTPtr & ast, const Context & context) const + { + return part->storage.getPartitionIDFromQuery(ast, context); + } + protected: StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(getIDFromPart(part_)) diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index 53c9b50cb9d..bb22a1d0395 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -2,11 +2,13 @@ #include #include #include -#include #include #include #include +#include +#include #include +#include #include #include #include @@ -32,6 +34,7 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, res.ast = command->ptr(); res.type = DELETE; res.predicate = command->predicate; + res.partition = command->partition; return res; } else if (command->type == ASTAlterCommand::UPDATE) @@ -40,6 +43,7 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, res.ast = command->ptr(); res.type = UPDATE; res.predicate = command->predicate; + res.partition = command->partition; for (const ASTPtr & assignment_ast : command->update_assignments->children) { const auto & assignment = assignment_ast->as(); @@ -124,6 +128,7 @@ std::shared_ptr MutationCommands::ast() const return res; } + void MutationCommands::writeText(WriteBuffer & out) const { std::stringstream commands_ss; diff --git a/src/Storages/MutationCommands.h b/src/Storages/MutationCommands.h index 6e641e42cff..0f031eb56e6 100644 --- a/src/Storages/MutationCommands.h +++ b/src/Storages/MutationCommands.h @@ -43,8 +43,10 @@ struct MutationCommand /// Columns with corresponding actions std::unordered_map column_to_update_expression; - /// For MATERIALIZE INDEX + /// For MATERIALIZE INDEX. String index_name; + + /// For MATERIALIZE INDEX, UPDATE and DELETE. ASTPtr partition; /// For reads, drops and etc. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ac55df5885c..8c48febca1b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3915,6 +3915,60 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer } +std::set StorageReplicatedMergeTree::getPartitionIdsAffectedByCommands( + const MutationCommands & commands, const Context & query_context) const +{ + std::set affected_partition_ids; + + for (const auto & command : commands) + { + if (!command.partition) + { + affected_partition_ids.clear(); + break; + } + + affected_partition_ids.insert( + getPartitionIDFromQuery(command.partition, query_context) + ); + } + + return affected_partition_ids; +} + + +PartitionBlockNumbersHolder StorageReplicatedMergeTree::allocateBlockNumbersInAffectedPartitions( + const MutationCommands & commands, const Context & query_context, const zkutil::ZooKeeperPtr & zookeeper) const +{ + const std::set mutation_affected_partition_ids = getPartitionIdsAffectedByCommands(commands, query_context); + + if (mutation_affected_partition_ids.size() == 1) + { + const auto & affected_partition_id = *mutation_affected_partition_ids.cbegin(); + auto block_number_holder = allocateBlockNumber(affected_partition_id, zookeeper); + if (!block_number_holder.has_value()) + return {}; + auto block_number = block_number_holder->getNumber(); /// Avoid possible UB due to std::move + return {{{affected_partition_id, block_number}}, std::move(block_number_holder)}; + } + else + { + /// TODO: Implement optimal block number aqcuisition algorithm in multiple (but not all) partitions + EphemeralLocksInAllPartitions lock_holder( + zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + + PartitionBlockNumbersHolder::BlockNumbersType block_numbers; + for (const auto & lock : lock_holder.getLocks()) + { + if (mutation_affected_partition_ids.empty() || mutation_affected_partition_ids.count(lock.partition_id)) + block_numbers[lock.partition_id] = lock.number; + } + + return {std::move(block_numbers), std::move(lock_holder)}; + } +} + + void StorageReplicatedMergeTree::alter( const AlterCommands & commands, const Context & query_context, TableLockHolder & table_lock_holder) { @@ -3942,7 +3996,7 @@ void StorageReplicatedMergeTree::alter( return queryToString(query); }; - auto zookeeper = getZooKeeper(); + const auto zookeeper = getZooKeeper(); std::optional alter_entry; std::optional mutation_znode; @@ -3953,10 +4007,6 @@ void StorageReplicatedMergeTree::alter( alter_entry.emplace(); mutation_znode.reset(); - /// We can safely read structure, because we guarded with alter_intention_lock - if (is_readonly) - throw Exception("Can't ALTER readonly table", ErrorCodes::TABLE_IS_READ_ONLY); - auto current_metadata = getInMemoryMetadataPtr(); StorageInMemoryMetadata future_metadata = *current_metadata; @@ -4029,27 +4079,23 @@ void StorageReplicatedMergeTree::alter( ops.emplace_back(zkutil::makeCreateRequest( zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); - std::optional lock_holder; - - /// Now we will prepare mutations record. - /// This code pretty same with mutate() function but process results slightly differently. + PartitionBlockNumbersHolder partition_block_numbers_holder; if (alter_entry->have_mutation) { - String mutations_path = zookeeper_path + "/mutations"; + const String mutations_path(zookeeper_path + "/mutations"); ReplicatedMergeTreeMutationEntry mutation_entry; - mutation_entry.source_replica = replica_name; - mutation_entry.commands = maybe_mutation_commands; mutation_entry.alter_version = new_metadata_version; + mutation_entry.source_replica = replica_name; + mutation_entry.commands = std::move(maybe_mutation_commands); + Coordination::Stat mutations_stat; zookeeper->get(mutations_path, &mutations_stat); - lock_holder.emplace( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); - - for (const auto & lock : lock_holder->getLocks()) - mutation_entry.block_numbers[lock.partition_id] = lock.number; + partition_block_numbers_holder = + allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper); + mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers(); mutation_entry.create_time = time(nullptr); ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); @@ -4060,6 +4106,11 @@ void StorageReplicatedMergeTree::alter( Coordination::Responses results; Coordination::Error rc = zookeeper->tryMulti(ops, results); + /// For the sake of constitency with mechanics of concurrent background process of assigning parts merge tasks + /// this placeholder must be held up until the moment of committing into ZK of the mutation entry + /// See ReplicatedMergeTreeMergePredicate::canMergeTwoParts() method + partition_block_numbers_holder.reset(); + if (rc == Coordination::Error::ZOK) { if (alter_entry->have_mutation) @@ -4398,7 +4449,7 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_table_data, c } -bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) +bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) const { { std::lock_guard lock(existing_nodes_cache_mutex); @@ -4420,7 +4471,7 @@ bool StorageReplicatedMergeTree::existsNodeCached(const std::string & path) std::optional StorageReplicatedMergeTree::allocateBlockNumber( - const String & partition_id, zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_block_id_path) + const String & partition_id, const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_block_id_path) const { /// Lets check for duplicates in advance, to avoid superfluous block numbers allocation Coordination::Requests deduplication_check_ops; @@ -5063,44 +5114,46 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const /// After all needed parts are mutated (i.e. all active parts have the mutation version greater than /// the version of this mutation), the mutation is considered done and can be deleted. - ReplicatedMergeTreeMutationEntry entry; - entry.source_replica = replica_name; - entry.commands = commands; + ReplicatedMergeTreeMutationEntry mutation_entry; + mutation_entry.source_replica = replica_name; + mutation_entry.commands = commands; - String mutations_path = zookeeper_path + "/mutations"; + const String mutations_path = zookeeper_path + "/mutations"; + const auto zookeeper = getZooKeeper(); /// Update the mutations_path node when creating the mutation and check its version to ensure that /// nodes for mutations are created in the same order as the corresponding block numbers. /// Should work well if the number of concurrent mutation requests is small. while (true) { - auto zookeeper = getZooKeeper(); - Coordination::Stat mutations_stat; zookeeper->get(mutations_path, &mutations_stat); - EphemeralLocksInAllPartitions block_number_locks( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + PartitionBlockNumbersHolder partition_block_numbers_holder = + allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper); - for (const auto & lock : block_number_locks.getLocks()) - entry.block_numbers[lock.partition_id] = lock.number; - - entry.create_time = time(nullptr); + mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers(); + mutation_entry.create_time = time(nullptr); + /// The following version check guarantees the linearizability property for any pair of mutations: + /// mutation with higher sequence number is guaranteed to have higher block numbers in every partition + /// (and thus will be applied strictly according to sequence numbers of mutations) Coordination::Requests requests; requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); requests.emplace_back(zkutil::makeCreateRequest( - mutations_path + "/", entry.toString(), zkutil::CreateMode::PersistentSequential)); + mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); Coordination::Responses responses; Coordination::Error rc = zookeeper->tryMulti(requests, responses); + partition_block_numbers_holder.reset(); + if (rc == Coordination::Error::ZOK) { const String & path_created = dynamic_cast(responses[1].get())->path_created; - entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); - LOG_TRACE(log, "Created mutation with ID {}", entry.znode_name); + mutation_entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); + LOG_TRACE(log, "Created mutation with ID {}", mutation_entry.znode_name); break; } else if (rc == Coordination::Error::ZBADVERSION) @@ -5112,7 +5165,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const throw Coordination::Exception("Unable to create a mutation znode", rc); } - waitMutation(entry.znode_name, query_context.getSettingsRef().mutations_sync); + waitMutation(mutation_entry.znode_name, query_context.getSettingsRef().mutations_sync); } void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t mutations_sync) const diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 5944c9ce3a8..c9f94427112 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -506,8 +506,8 @@ private: /// Creates new block number if block with such block_id does not exist std::optional allocateBlockNumber( - const String & partition_id, zkutil::ZooKeeperPtr & zookeeper, - const String & zookeeper_block_id_path = ""); + const String & partition_id, const zkutil::ZooKeeperPtr & zookeeper, + const String & zookeeper_block_id_path = "") const; /** Wait until all replicas, including this, execute the specified action from the log. * If replicas are added at the same time, it can not wait the added replica . @@ -531,9 +531,9 @@ private: bool getFakePartCoveringAllPartsInPartition(const String & partition_id, MergeTreePartInfo & part_info, bool for_replace_partition = false); /// Check for a node in ZK. If it is, remember this information, and then immediately answer true. - std::unordered_set existing_nodes_cache; - std::mutex existing_nodes_cache_mutex; - bool existsNodeCached(const std::string & path); + mutable std::unordered_set existing_nodes_cache; + mutable std::mutex existing_nodes_cache_mutex; + bool existsNodeCached(const std::string & path) const; void getClearBlocksInPartitionOps(Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num); /// Remove block IDs from `blocks/` in ZooKeeper for the given partition ID in the given block number range. @@ -565,6 +565,11 @@ private: MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const override; void startBackgroundMovesIfNeeded() override; + + std::set getPartitionIdsAffectedByCommands(const MutationCommands & commands, const Context & query_context) const; + PartitionBlockNumbersHolder allocateBlockNumbersInAffectedPartitions( + const MutationCommands & commands, const Context & query_context, const zkutil::ZooKeeperPtr & zookeeper) const; + protected: /** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table. */ diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/__init__.py b/tests/integration/test_mutations_in_partitions_of_merge_tree/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/cluster.xml b/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/cluster.xml new file mode 100644 index 00000000000..ec7c9b8e4f8 --- /dev/null +++ b/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/cluster.xml @@ -0,0 +1,16 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + \ No newline at end of file diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/logs_config.xml b/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/logs_config.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/integration/test_mutations_in_partitions_of_merge_tree/configs/logs_config.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py b/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py new file mode 100644 index 00000000000..c7858c2f74d --- /dev/null +++ b/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py @@ -0,0 +1,98 @@ +import pytest +import helpers.client +import helpers.cluster + + +cluster = helpers.cluster.ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/logs_config.xml', 'configs/cluster.xml'], + with_zookeeper=True, stay_alive=True) + +node2 = cluster.add_instance('node2', main_configs=['configs/logs_config.xml', 'configs/cluster.xml'], + with_zookeeper=True, stay_alive=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_trivial_alter_in_partition_merge_tree_without_where(started_cluster): + try: + name = "test_trivial_alter_in_partition_merge_tree_without_where" + node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query("CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format(name)) + node1.query("INSERT INTO {} VALUES (1, 2), (2, 3)".format(name)) + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} DELETE IN PARTITION 1 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +def test_trivial_alter_in_partition_merge_tree_with_where(started_cluster): + try: + name = "test_trivial_alter_in_partition_merge_tree_with_where" + node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query("CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format(name)) + node1.query("INSERT INTO {} VALUES (1, 2), (2, 3)".format(name)) + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + node1.query("ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + node1.query("ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +def test_trivial_alter_in_partition_replicated_merge_tree(started_cluster): + try: + name = "test_trivial_alter_in_partition_replicated_merge_tree" + + node1.query("DROP TABLE IF EXISTS {}".format(name)) + node2.query("DROP TABLE IF EXISTS {}".format(name)) + + for node in (node1, node2): + node.query( + "CREATE TABLE {name} (p Int64, x Int64) ENGINE=ReplicatedMergeTree('/clickhouse/{name}', '{{instance}}') ORDER BY tuple() PARTITION BY p" + .format(name=name)) + + node1.query("INSERT INTO {} VALUES (1, 2)".format(name)) + node2.query("INSERT INTO {} VALUES (2, 3)".format(name)) + + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE 1 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query("ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + node1.query("ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + node1.query("ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + for node in (node1, node2): + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + node2.query("DROP TABLE IF EXISTS {}".format(name)) From 046830967c80542535f64912fdab75ad9a6bc503 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 Nov 2020 13:26:26 +0300 Subject: [PATCH 087/114] Try fix tests. --- src/Storages/StorageMergeTree.cpp | 3 ++- src/Storages/StorageReplicatedMergeTree.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index a257173d6b4..efd933c9a8b 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -181,7 +181,8 @@ void StorageMergeTree::read( size_t max_block_size, unsigned num_streams) { - query_plan = std::move(*reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)); + if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)) + query_plan = std::move(*plan); } Pipe StorageMergeTree::read( diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 394294507b1..faeb7e69022 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3629,11 +3629,13 @@ void StorageReplicatedMergeTree::read( if (context.getSettingsRef().select_sequential_consistency) { auto max_added_blocks = getMaxAddedBlocks(); - query_plan = std::move(*reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams, &max_added_blocks)); + if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams, &max_added_blocks)) + query_plan = std::move(*plan); return; } - query_plan = std::move(*reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)); + if (auto plan = reader.read(column_names, metadata_snapshot, query_info, context, max_block_size, num_streams)) + query_plan = std::move(*plan); } Pipe StorageReplicatedMergeTree::read( From 55631e442bd4e69bfc23fa2a9584e0bf6ee07211 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 10 Nov 2020 14:43:22 +0300 Subject: [PATCH 088/114] fix bug in WriteBufferFromVector --- src/IO/WriteBufferFromVector.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/IO/WriteBufferFromVector.h b/src/IO/WriteBufferFromVector.h index 047e16643ce..54f43b6b591 100644 --- a/src/IO/WriteBufferFromVector.h +++ b/src/IO/WriteBufferFromVector.h @@ -36,8 +36,10 @@ private: throw Exception("WriteBufferFromVector is finished", ErrorCodes::CANNOT_WRITE_AFTER_END_OF_BUFFER); size_t old_size = vector.size(); + /// pos may not be equal to vector.data() + old_size, because WriteBuffer::next() can be used to flush data + size_t pos_offset = pos - reinterpret_cast(vector.data()); vector.resize(old_size * size_multiplier); - internal_buffer = Buffer(reinterpret_cast(vector.data() + old_size), reinterpret_cast(vector.data() + vector.size())); + internal_buffer = Buffer(reinterpret_cast(vector.data() + pos_offset), reinterpret_cast(vector.data() + vector.size())); working_buffer = internal_buffer; } From 07fe3a6347f7d5a83d4a374adac9a531062c5b59 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 Nov 2020 15:14:05 +0300 Subject: [PATCH 089/114] Fix build. --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index dd8f51aaa72..7f031771373 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -79,17 +79,6 @@ static Block getBlockWithPartColumn(const MergeTreeData::DataPartsVector & parts return Block{ColumnWithTypeAndName(std::move(column), std::make_shared(), "_part")}; } -/// Check if ORDER BY clause of the query has some expression. -static bool sortingDescriptionHasExpressions(const SortDescription & sort_description, const StorageMetadataPtr & metadata_snapshot) -{ - auto all_columns = metadata_snapshot->getColumns(); - for (const auto & sort_column : sort_description) - { - if (!all_columns.has(sort_column.column_name)) - return true; - } - return false; -} size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead( const MergeTreeData::DataPartsVector & parts, From b659efdb437abb91ae4a7b7544013d6439ccd46e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 Nov 2020 16:18:36 +0300 Subject: [PATCH 090/114] Fix test. --- ...1_mergetree_read_in_order_spread.reference | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference index fc10b4707a9..adbb03ae018 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference @@ -8,4 +8,23 @@ ExpressionTransform AggregatingInOrderTransform × 3 (Expression) ExpressionTransform × 3 - (ReadFromStorage) + (SettingQuotaAndLimits) + (Expression) + ExpressionTransform × 3 + (Union) + (MergingSorted) + (Expression) + ExpressionTransform + (ReadFromStorage) + MergeTree 0 → 1 + (MergingSorted) + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (ReadFromStorage) + MergeTree × 2 0 → 1 + (MergingSorted) + (Expression) + ExpressionTransform + (ReadFromStorage) + MergeTree 0 → 1 From 115807e77add0c46355530a2672991af44bcba1a Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Tue, 10 Nov 2020 17:54:53 +0300 Subject: [PATCH 091/114] DOCSUP-3043: Document the null function (ru) (#16795) * my changes to gitignore * Corrections in English docs and Russian docs added. * TOC corrections * TOC fixed * Revert "my changes to gitignore" This reverts commit 5884b1e79b0eb40d9c39b019d345d9dbc3c45640. * Update docs/en/sql-reference/table-functions/null.md * Update docs/en/sql-reference/table-functions/null.md * Update docs/ru/sql-reference/table-functions/null.md * Update docs/ru/sql-reference/table-functions/null.md Co-authored-by: Olga Revyakina Co-authored-by: BayoNet --- docs/en/sql-reference/table-functions/null.md | 8 ++-- docs/ru/sql-reference/table-functions/null.md | 43 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 docs/ru/sql-reference/table-functions/null.md diff --git a/docs/en/sql-reference/table-functions/null.md b/docs/en/sql-reference/table-functions/null.md index 6edec61add7..355a45a83e1 100644 --- a/docs/en/sql-reference/table-functions/null.md +++ b/docs/en/sql-reference/table-functions/null.md @@ -5,7 +5,7 @@ toc_title: null function # null {#null-function} -Accepts an inserted data of the specified structure and immediately drops it away. The function is used for convenience writing tests and demonstrations. +Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately droped right after the query execution. The function is used for the convenience of test writing and demonstrations. **Syntax** @@ -19,7 +19,7 @@ null('structure') **Returned value** -A table with the specified structure, which is dropped right after the query execution. +A temporary `Null`-engine table with the specified structure. **Example** @@ -36,6 +36,8 @@ INSERT INTO t SELECT * FROM numbers_mt(1000000000); DROP TABLE IF EXISTS t; ``` -See also: format **Null**. +See also: + +- [Null table engine](../../engines/table-engines/special/null.md) [Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/null/) diff --git a/docs/ru/sql-reference/table-functions/null.md b/docs/ru/sql-reference/table-functions/null.md new file mode 100644 index 00000000000..8e0173733f8 --- /dev/null +++ b/docs/ru/sql-reference/table-functions/null.md @@ -0,0 +1,43 @@ +--- +toc_priority: 53 +toc_title: null функция +--- + +# null {#null-function} + +Создает временную таблицу указанной структуры с движком [Null](../../engines/table-engines/special/null.md). В соответствии со свойствами движка, данные в таблице игнорируются, а сама таблица удаляется сразу после выполнения запроса. Функция используется для удобства написания тестов и демонстрационных примеров. + +**Синтаксис** + +``` sql +null('structure') +``` + +**Параметр** + +- `structure` — список колонок и их типов. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +Временная таблица указанной структуры с движком `Null`. + +**Пример** + +Один запрос с функцией `null`: + +``` sql +INSERT INTO function null('x UInt64') SELECT * FROM numbers_mt(1000000000); +``` +заменяет три запроса: + +```sql +CREATE TABLE t (x UInt64) ENGINE = Null; +INSERT INTO t SELECT * FROM numbers_mt(1000000000); +DROP TABLE IF EXISTS t; +``` + +См. также: + +- [Движок таблиц Null](../../engines/table-engines/special/null.md) + +[Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/null/) From 205719877945167b1615fcfa15f502310e55867d Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 17:58:05 +0300 Subject: [PATCH 092/114] Fix performance tests that became very fast --- tests/performance/push_down_limit.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 0dcd9335a52..6fc60bc5768 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,4 +1,4 @@ - select number from (select number from numbers(10000000) order by -number) limit 10 - select number from (select number from numbers_mt(100000000) order by -number) limit 10 + select number from (select number from numbers(150000000) order by -number) limit 10 + select number from (select number from numbers_mt(1500000000) order by -number) limit 10 From 8504efde772ffb7b08f56b6c52f77c2b257778ee Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 18:12:49 +0300 Subject: [PATCH 093/114] Merge --- tests/performance/push_down_limit.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 6fc60bc5768..4aa30cb9d5d 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,4 +1,4 @@ - + select number from (select number from numbers(150000000) order by -number) limit 10 select number from (select number from numbers_mt(1500000000) order by -number) limit 10 From 847fa9ca3cfa2e18233eb83da9975f41d86d8153 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 18:15:26 +0300 Subject: [PATCH 094/114] Merge --- tests/performance/push_down_limit.xml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 4aa30cb9d5d..549882bf7a7 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,4 +1,8 @@ - - select number from (select number from numbers(150000000) order by -number) limit 10 - select number from (select number from numbers_mt(1500000000) order by -number) limit 10 + + CREATE VIEW numbers_view AS SELECT number from numbers_mt(100000000) order by number desc + + select number from (select number from numbers(1500000000) order by -number) limit 10 + select number from (select number from numbers_mt(15000000000) order by -number) limit 10 + + select number from numbers_view limit 100 From 0ef6bd61880c2924e282dbe1992738b0936e0818 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 18:19:46 +0300 Subject: [PATCH 095/114] Fix tests, I cannot count number of zeros sometimes --- tests/performance/push_down_limit.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 549882bf7a7..0af6eec8b32 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,8 +1,8 @@ CREATE VIEW numbers_view AS SELECT number from numbers_mt(100000000) order by number desc - select number from (select number from numbers(1500000000) order by -number) limit 10 - select number from (select number from numbers_mt(15000000000) order by -number) limit 10 + select number from (select number from numbers(150000000) order by -number) limit 10 + select number from (select number from numbers_mt(1500000000) order by -number) limit 10 select number from numbers_view limit 100 From c0308a5d85f66932822553b4daabb9c999511689 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 18:21:39 +0300 Subject: [PATCH 096/114] Fix tests, I cannot count number of zeros sometimes --- tests/performance/push_down_limit.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 549882bf7a7..0af6eec8b32 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,8 +1,8 @@ CREATE VIEW numbers_view AS SELECT number from numbers_mt(100000000) order by number desc - select number from (select number from numbers(1500000000) order by -number) limit 10 - select number from (select number from numbers_mt(15000000000) order by -number) limit 10 + select number from (select number from numbers(150000000) order by -number) limit 10 + select number from (select number from numbers_mt(1500000000) order by -number) limit 10 select number from numbers_view limit 100 From c36e6fe37849d3b1f506476eab47e04324ac074e Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Tue, 10 Nov 2020 18:22:59 +0300 Subject: [PATCH 097/114] Fix tests finally --- tests/performance/push_down_limit.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 0af6eec8b32..6ae63b54ec6 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,7 +1,7 @@ CREATE VIEW numbers_view AS SELECT number from numbers_mt(100000000) order by number desc - select number from (select number from numbers(150000000) order by -number) limit 10 + select number from (select number from numbers(1500000000) order by -number) limit 10 select number from (select number from numbers_mt(1500000000) order by -number) limit 10 select number from numbers_view limit 100 From b94cc5c4e53c3d275d287e0b6b71e37d9421d339 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 10 Nov 2020 21:22:26 +0300 Subject: [PATCH 098/114] remove more stringstreams --- base/mysqlxx/Query.h | 2 +- programs/client/Client.cpp | 2 + programs/client/Suggest.cpp | 2 +- programs/client/TestHint.h | 2 +- programs/copier/ClusterCopier.cpp | 2 +- programs/copier/Internals.cpp | 2 +- programs/copier/TaskTableAndShard.h | 8 +- programs/local/LocalServer.cpp | 2 +- programs/server/Server.cpp | 8 +- .../AggregateFunctionRetention.h | 2 - .../AggregateFunctionTimeSeriesGroupSum.h | 2 - .../AggregateFunctionWindowFunnel.h | 2 - src/AggregateFunctions/ReservoirSampler.h | 1 - .../ReservoirSamplerDeterministic.h | 1 - src/Columns/tests/gtest_weak_hash_32.cpp | 2 +- src/Common/Config/ConfigProcessor.cpp | 6 +- src/Common/Exception.cpp | 6 +- src/Common/MemoryTracker.cpp | 30 ++---- src/Common/ShellCommand.cpp | 4 +- src/Common/StackTrace.cpp | 8 +- src/Common/StudentTTest.cpp | 2 +- src/Common/ThreadStatus.cpp | 10 +- src/Common/UInt128.h | 1 - src/Common/XDBCBridgeHelper.h | 4 - src/Common/ZooKeeper/TestKeeper.cpp | 2 +- src/Common/checkStackSize.cpp | 12 +-- src/Common/formatReadable.cpp | 2 - src/Compression/CompressedReadBufferBase.cpp | 14 ++- src/Compression/tests/compressed_buffer.cpp | 5 +- src/Core/MySQL/IMySQLReadPacket.cpp | 8 +- src/Core/MySQL/IMySQLWritePacket.cpp | 12 ++- src/Core/MySQL/MySQLReplication.cpp | 101 +++++++++--------- src/Core/MySQL/MySQLReplication.h | 22 ++-- src/Core/SortDescription.h | 5 +- src/Core/tests/mysql_protocol.cpp | 21 ++-- .../CheckConstraintsBlockOutputStream.cpp | 38 +++---- src/DataStreams/IBlockInputStream.cpp | 10 +- src/DataStreams/IBlockInputStream.h | 2 +- .../MergingSortedBlockInputStream.cpp | 2 - src/DataStreams/MongoDBBlockInputStream.cpp | 1 - .../DataTypeCustomSimpleAggregateFunction.cpp | 3 +- src/DataTypes/DataTypesDecimal.cpp | 5 +- .../MySQL/MaterializeMySQLSyncThread.cpp | 27 ++--- src/Functions/abtesting.cpp | 1 - src/IO/HTTPCommon.cpp | 2 +- src/IO/MySQLPacketPayloadReadBuffer.cpp | 1 - src/IO/S3/PocoHTTPClient.cpp | 5 +- src/IO/tests/gtest_bit_io.cpp | 2 +- src/IO/tests/hashing_read_buffer.cpp | 2 +- src/IO/tests/limit_read_buffer2.cpp | 2 +- src/IO/tests/write_buffer.cpp | 2 +- src/Interpreters/Context.cpp | 28 +++-- src/Interpreters/ExpressionActions.cpp | 13 +-- src/Interpreters/InterpreterExplainQuery.cpp | 2 - ...InterpreterShowCreateAccessEntityQuery.cpp | 1 - .../InterpreterShowCreateQuery.cpp | 2 - .../InterpreterShowTablesQuery.cpp | 25 ++--- src/Interpreters/QueryAliasesVisitor.cpp | 3 - src/Interpreters/Set.cpp | 8 +- src/Interpreters/TreeRewriter.cpp | 3 +- src/Interpreters/executeQuery.cpp | 8 +- src/Parsers/ASTCreateRowPolicyQuery.cpp | 1 - src/Parsers/ASTIdentifier.cpp | 1 - src/Parsers/ASTWithAlias.cpp | 1 - src/Parsers/DumpASTNode.h | 1 - src/Parsers/IAST.cpp | 1 - src/Parsers/IAST.h | 2 - src/Parsers/formatAST.cpp | 1 - src/Parsers/formatAST.h | 22 ++-- .../Formats/Impl/MySQLOutputFormat.cpp | 2 - .../Formats/Impl/PrettyBlockOutputFormat.cpp | 14 +-- .../Impl/PrettyCompactBlockOutputFormat.cpp | 16 ++- .../Algorithms/CollapsingSortedAlgorithm.cpp | 4 +- src/Server/ReplicasStatusHandler.cpp | 5 +- src/Server/TCPHandler.cpp | 10 +- .../DistributedBlockOutputStream.cpp | 3 +- src/Storages/Kafka/StorageKafka.cpp | 10 +- src/Storages/MergeTree/KeyCondition.h | 1 - .../MergeTree/MergeTreeDataPartChecksum.cpp | 9 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 10 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 3 +- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 9 +- src/Storages/StorageDictionary.cpp | 14 +-- src/Storages/StorageInMemoryMetadata.cpp | 4 +- src/Storages/StorageMergeTree.cpp | 28 ++--- src/Storages/StorageReplicatedMergeTree.cpp | 67 +++++------- src/Storages/StorageS3.cpp | 2 - .../transformQueryForExternalDatabase.cpp | 1 - src/TableFunctions/TableFunctionRemote.cpp | 10 +- utils/check-mysql-binlog/main.cpp | 21 ++-- utils/check-style/check-style | 2 +- .../ProtobufDelimitedMessagesSerializer.cpp | 4 +- utils/zookeeper-cli/zookeeper-cli.cpp | 2 +- 93 files changed, 323 insertions(+), 471 deletions(-) diff --git a/base/mysqlxx/Query.h b/base/mysqlxx/Query.h index d0a905e2031..1d3ab9678d5 100644 --- a/base/mysqlxx/Query.h +++ b/base/mysqlxx/Query.h @@ -77,7 +77,7 @@ public: private: Connection * conn; - std::ostringstream query_buf; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + std::ostringstream query_buf; void executeImpl(); }; diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 98c1463ab23..c7884b813cf 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1189,6 +1189,7 @@ private: fprintf(stderr, "dump after fuzz:\n"); WriteBufferFromOStream cerr_buf(std::cerr, 4096); fuzz_base->dumpTree(cerr_buf); + cerr_buf.next(); fmt::print(stderr, "IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly."); @@ -1533,6 +1534,7 @@ private: std::cout << std::endl; WriteBufferFromOStream res_buf(std::cout, 4096); formatAST(*res, res_buf); + res_buf.next(); std::cout << std::endl << std::endl; } diff --git a/programs/client/Suggest.cpp b/programs/client/Suggest.cpp index e85e7a21261..87083c2c27b 100644 --- a/programs/client/Suggest.cpp +++ b/programs/client/Suggest.cpp @@ -86,7 +86,7 @@ Suggest::Suggest() void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit) { - std::stringstream query; + std::stringstream query; // STYLE_CHECK_ALLOW_STD_STRING_STREAM query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" "SELECT name FROM system.functions" " UNION ALL " diff --git a/programs/client/TestHint.h b/programs/client/TestHint.h index 641c3e0ccf0..65666f4304c 100644 --- a/programs/client/TestHint.h +++ b/programs/client/TestHint.h @@ -93,7 +93,7 @@ private: void parse(const String & hint) { - std::stringstream ss; + std::stringstream ss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM ss << hint; String item; diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 4ee14b14119..a129dc7efcc 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -162,7 +162,7 @@ void ClusterCopier::discoverShardPartitions(const ConnectionTimeouts & timeouts, if (!missing_partitions.empty()) { - std::stringstream ss; + WriteBufferFromOwnString ss; for (const String & missing_partition : missing_partitions) ss << " " << missing_partition; diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 0f607ea5faf..ea2be469945 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -13,7 +13,7 @@ using ConfigurationPtr = Poco::AutoPtr; ConfigurationPtr getConfigurationFromXMLString(const std::string & xml_data) { - std::stringstream ss(xml_data); + std::stringstream ss(xml_data); // STYLE_CHECK_ALLOW_STD_STRING_STREAM Poco::XML::InputSource input_source{ss}; return {new Poco::Util::XMLConfiguration{&input_source}}; } diff --git a/programs/copier/TaskTableAndShard.h b/programs/copier/TaskTableAndShard.h index cc12fe556c7..4f5bfb443e6 100644 --- a/programs/copier/TaskTableAndShard.h +++ b/programs/copier/TaskTableAndShard.h @@ -394,12 +394,8 @@ inline ASTPtr TaskTable::rewriteReplicatedCreateQueryToPlain() inline String DB::TaskShard::getDescription() const { - std::stringstream ss; - ss << "N" << numberInCluster() - << " (having a replica " << getHostNameExample() - << ", pull table " + getQuotedTable(task_table.table_pull) - << " of cluster " + task_table.cluster_pull_name << ")"; - return ss.str(); + return fmt::format("N{} (having a replica {}, pull table {} of cluster {}", + numberInCluster(), getHostNameExample(), getQuotedTable(task_table.table_pull), task_table.cluster_pull_name); } inline String DB::TaskShard::getHostNameExample() const diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 9ecc2a50a42..15e71198eb1 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -422,7 +422,7 @@ static const char * minimal_default_user_xml = static ConfigurationPtr getConfigurationFromXMLString(const char * xml_data) { - std::stringstream ss{std::string{xml_data}}; + std::stringstream ss{std::string{xml_data}}; // STYLE_CHECK_ALLOW_STD_STRING_STREAM Poco::XML::InputSource input_source{ss}; return {new Poco::Util::XMLConfiguration{&input_source}}; } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ed18793a537..ca445e83a14 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -191,10 +191,10 @@ int Server::run() if (config().hasOption("help")) { Poco::Util::HelpFormatter help_formatter(Server::options()); - std::stringstream header; - header << commandName() << " [OPTION] [-- [ARG]...]\n"; - header << "positional arguments can be used to rewrite config.xml properties, for example, --http_port=8010"; - help_formatter.setHeader(header.str()); + auto header_str = fmt::format("{} [OPTION] [-- [ARG]...]\n" + "positional arguments can be used to rewrite config.xml properties, for example, --http_port=8010", + commandName()); + help_formatter.setHeader(header_str); help_formatter.format(std::cout); return 0; } diff --git a/src/AggregateFunctions/AggregateFunctionRetention.h b/src/AggregateFunctions/AggregateFunctionRetention.h index b742dcdf77f..f8a2163ccb9 100644 --- a/src/AggregateFunctions/AggregateFunctionRetention.h +++ b/src/AggregateFunctions/AggregateFunctionRetention.h @@ -1,7 +1,5 @@ #pragma once -#include -#include #include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h index b755fbf081b..28c3f53d879 100644 --- a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h +++ b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h @@ -1,10 +1,8 @@ #pragma once #include -#include #include #include -#include #include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index 3297819a9ff..2dd6ef8a9fd 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -1,7 +1,5 @@ #pragma once -#include -#include #include #include #include diff --git a/src/AggregateFunctions/ReservoirSampler.h b/src/AggregateFunctions/ReservoirSampler.h index c5d2158d59c..a321ef08648 100644 --- a/src/AggregateFunctions/ReservoirSampler.h +++ b/src/AggregateFunctions/ReservoirSampler.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index eae24c1f3e9..3b7817e9308 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Columns/tests/gtest_weak_hash_32.cpp b/src/Columns/tests/gtest_weak_hash_32.cpp index a7fd60a3b9c..a04bd94124c 100644 --- a/src/Columns/tests/gtest_weak_hash_32.cpp +++ b/src/Columns/tests/gtest_weak_hash_32.cpp @@ -71,7 +71,7 @@ void checkColumn( std::unordered_map map; size_t num_collisions = 0; - std::stringstream collisions_str; + std::stringstream collisions_str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM collisions_str.exceptions(std::ios::failbit); for (size_t i = 0; i < eq_class.size(); ++i) diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 8a6093c47c9..3e06be94b11 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,8 @@ #include #include #include +#include +#include #define PREPROCESSED_SUFFIX "-preprocessed" @@ -537,8 +538,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( if (has_zk_includes) *has_zk_includes = !contributing_zk_paths.empty(); - std::stringstream comment; - comment.exceptions(std::ios::failbit); + WriteBufferFromOwnString comment; comment << " This file was generated automatically.\n"; comment << " Do not edit it: it is likely to be discarded and generated again before it's read next time.\n"; comment << " Files used to generate this file:"; diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 820e3857bfc..dd78d0ec9fc 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -245,8 +245,7 @@ static std::string getExtraExceptionInfo(const std::exception & e) std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace /*= false*/, bool with_extra_info /*= true*/) { - std::stringstream stream; - stream.exceptions(std::ios::failbit); + WriteBufferFromOwnString stream; try { @@ -365,8 +364,7 @@ void tryLogException(std::exception_ptr e, Poco::Logger * logger, const std::str std::string getExceptionMessage(const Exception & e, bool with_stacktrace, bool check_embedded_stacktrace) { - std::stringstream stream; - stream.exceptions(std::ios::failbit); + WriteBufferFromOwnString stream; try { diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 5257f95898a..1d324a00473 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -133,17 +133,12 @@ void MemoryTracker::alloc(Int64 size) BlockerInThread untrack_lock; ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded); - std::stringstream message; - message.exceptions(std::ios::failbit); - message << "Memory tracker"; - if (const auto * description = description_ptr.load(std::memory_order_relaxed)) - message << " " << description; - message << ": fault injected. Would use " << formatReadableSizeWithBinarySuffix(will_be) - << " (attempt to allocate chunk of " << size << " bytes)" - << ", maximum: " << formatReadableSizeWithBinarySuffix(current_hard_limit); - + const auto * description = description_ptr.load(std::memory_order_relaxed); amount.fetch_sub(size, std::memory_order_relaxed); - throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); + throw DB::Exception(DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, + "Memory tracker {}: fault injected. Would use {} (attempt to allocate chunk of {} bytes), maximum: {}", + description ? description : "", formatReadableSizeWithBinarySuffix(will_be), + size, formatReadableSizeWithBinarySuffix(current_hard_limit)); } if (unlikely(current_profiler_limit && will_be > current_profiler_limit)) @@ -166,17 +161,12 @@ void MemoryTracker::alloc(Int64 size) BlockerInThread untrack_lock; ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded); - std::stringstream message; - message.exceptions(std::ios::failbit); - message << "Memory limit"; - if (const auto * description = description_ptr.load(std::memory_order_relaxed)) - message << " " << description; - message << " exceeded: would use " << formatReadableSizeWithBinarySuffix(will_be) - << " (attempt to allocate chunk of " << size << " bytes)" - << ", maximum: " << formatReadableSizeWithBinarySuffix(current_hard_limit); - + const auto * description = description_ptr.load(std::memory_order_relaxed); amount.fetch_sub(size, std::memory_order_relaxed); - throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); + throw DB::Exception(DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, + "Memory limit {} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}", + description ? description : "", formatReadableSizeWithBinarySuffix(will_be), + size, formatReadableSizeWithBinarySuffix(current_hard_limit)); } updatePeak(will_be); diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index db0928ea605..069c7774729 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -73,8 +74,7 @@ ShellCommand::~ShellCommand() void ShellCommand::logCommand(const char * filename, char * const argv[]) { - std::stringstream args; - args.exceptions(std::ios::failbit); + WriteBufferFromOwnString args; for (int i = 0; argv != nullptr && argv[i] != nullptr; ++i) { if (i > 0) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 7e9474ac3a7..9cfdba2687f 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -23,7 +23,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext_t & context) { - std::stringstream error; + std::stringstream error; // STYLE_CHECK_ALLOW_STD_STRING_STREAM error.exceptions(std::ios::failbit); switch (sig) { @@ -319,7 +319,7 @@ static void toStringEveryLineImpl( const DB::SymbolIndex & symbol_index = DB::SymbolIndex::instance(); std::unordered_map dwarfs; - std::stringstream out; + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM out.exceptions(std::ios::failbit); for (size_t i = offset; i < size; ++i) @@ -359,7 +359,7 @@ static void toStringEveryLineImpl( out.str({}); } #else - std::stringstream out; + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM out.exceptions(std::ios::failbit); for (size_t i = offset; i < size; ++i) @@ -375,7 +375,7 @@ static void toStringEveryLineImpl( static std::string toStringImpl(const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size) { - std::stringstream out; + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM out.exceptions(std::ios::failbit); toStringEveryLineImpl(frame_pointers, offset, size, [&](const std::string & str) { out << str << '\n'; }); return out.str(); diff --git a/src/Common/StudentTTest.cpp b/src/Common/StudentTTest.cpp index fe605092acc..da4913126b3 100644 --- a/src/Common/StudentTTest.cpp +++ b/src/Common/StudentTTest.cpp @@ -153,7 +153,7 @@ std::pair StudentTTest::compareAndReport(size_t confidence_le double mean_confidence_interval = table_value * t_statistic; - std::stringstream ss; + std::stringstream ss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM ss.exceptions(std::ios::failbit); if (mean_difference > mean_confidence_interval && (mean_difference - mean_confidence_interval > 0.0001)) /// difference must be more than 0.0001, to take into account connection latency. diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index f5ad28f57af..dad32293380 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -1,5 +1,3 @@ -#include - #include #include #include @@ -79,12 +77,10 @@ void ThreadStatus::assertState(const std::initializer_list & permitted_stat return; } - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << "Unexpected thread state " << getCurrentState(); if (description) - ss << ": " << description; - throw Exception(ss.str(), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected thread state {}: {}", getCurrentState(), description); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected thread state {}", getCurrentState()); } void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, diff --git a/src/Common/UInt128.h b/src/Common/UInt128.h index 735b287f90f..baa33578089 100644 --- a/src/Common/UInt128.h +++ b/src/Common/UInt128.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index c794d2fe3cd..59bae33d88d 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -307,9 +306,6 @@ struct ODBCBridgeMixin std::vector cmd_args; path.setFileName("clickhouse-odbc-bridge"); - std::stringstream command; - command.exceptions(std::ios::failbit); - #if !CLICKHOUSE_SPLIT_BINARY cmd_args.push_back("odbc-bridge"); #endif diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index f7db8a85f96..4ae26d874fb 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -218,7 +218,7 @@ std::pair TestKeeperCreateRequest::process(TestKeeper::Contai auto seq_num = it->second.seq_num; ++it->second.seq_num; - std::stringstream seq_num_str; + std::stringstream seq_num_str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM seq_num_str.exceptions(std::ios::failbit); seq_num_str << std::setw(10) << std::setfill('0') << seq_num; diff --git a/src/Common/checkStackSize.cpp b/src/Common/checkStackSize.cpp index bdc117eccac..e94abc08c6b 100644 --- a/src/Common/checkStackSize.cpp +++ b/src/Common/checkStackSize.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #if defined(__FreeBSD__) # include @@ -80,13 +79,8 @@ __attribute__((__weak__)) void checkStackSize() /// It's safe to assume that overflow in multiplying by two cannot occur. if (stack_size * 2 > max_stack_size) { - std::stringstream message; - message.exceptions(std::ios::failbit); - message << "Stack size too large" - << ". Stack address: " << stack_address - << ", frame address: " << frame_address - << ", stack size: " << stack_size - << ", maximum stack size: " << max_stack_size; - throw Exception(message.str(), ErrorCodes::TOO_DEEP_RECURSION); + throw Exception(ErrorCodes::TOO_DEEP_RECURSION, + "Stack size too large. Stack address: {}, frame address: {}, stack size: {}, maximum stack size: {}", + stack_address, frame_address, stack_size, max_stack_size); } } diff --git a/src/Common/formatReadable.cpp b/src/Common/formatReadable.cpp index 99929275521..fc5c6c19b50 100644 --- a/src/Common/formatReadable.cpp +++ b/src/Common/formatReadable.cpp @@ -1,6 +1,4 @@ #include -#include -#include #include #include diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 7a6b605d015..8b16b68a999 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace ProfileEvents @@ -42,7 +43,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c if (expected_checksum == calculated_checksum) return; - std::stringstream message; + WriteBufferFromOwnString message; /// TODO mess up of endianness in error message. message << "Checksum doesn't match: corrupted data." @@ -50,7 +51,16 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c + ". Actual: " + getHexUIntLowercase(calculated_checksum.first) + getHexUIntLowercase(calculated_checksum.second) + ". Size of compressed block: " + toString(size); - const char * message_hardware_failure = "This is most likely due to hardware failure. If you receive broken data over network and the error does not repeat every time, this can be caused by bad RAM on network interface controller or bad controller itself or bad RAM on network switches or bad CPU on network switches (look at the logs on related network switches; note that TCP checksums don't help) or bad RAM on host (look at dmesg or kern.log for enormous amount of EDAC errors, ECC-related reports, Machine Check Exceptions, mcelog; note that ECC memory can fail if the number of errors is huge) or bad CPU on host. If you read data from disk, this can be caused by disk bit rott. This exception protects ClickHouse from data corruption due to hardware failures."; + const char * message_hardware_failure = "This is most likely due to hardware failure. " + "If you receive broken data over network and the error does not repeat every time, " + "this can be caused by bad RAM on network interface controller or bad controller itself " + "or bad RAM on network switches or bad CPU on network switches " + "(look at the logs on related network switches; note that TCP checksums don't help) " + "or bad RAM on host (look at dmesg or kern.log for enormous amount of EDAC errors, " + "ECC-related reports, Machine Check Exceptions, mcelog; note that ECC memory can fail " + "if the number of errors is huge) or bad CPU on host. If you read data from disk, " + "this can be caused by disk bit rott. This exception protects ClickHouse " + "from data corruption due to hardware failures."; auto flip_bit = [](char * buf, size_t pos) { diff --git a/src/Compression/tests/compressed_buffer.cpp b/src/Compression/tests/compressed_buffer.cpp index c018fc95995..aef2cf4ab90 100644 --- a/src/Compression/tests/compressed_buffer.cpp +++ b/src/Compression/tests/compressed_buffer.cpp @@ -51,10 +51,7 @@ int main(int, char **) if (x != i) { - std::stringstream s; - s.exceptions(std::ios::failbit); - s << "Failed!, read: " << x << ", expected: " << i; - throw DB::Exception(s.str(), 0); + throw DB::Exception(0, "Failed!, read: {}, expected: {}", x, i); } } stopwatch.stop(); diff --git a/src/Core/MySQL/IMySQLReadPacket.cpp b/src/Core/MySQL/IMySQLReadPacket.cpp index 676f3986ba4..74f309d0294 100644 --- a/src/Core/MySQL/IMySQLReadPacket.cpp +++ b/src/Core/MySQL/IMySQLReadPacket.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -21,10 +20,9 @@ void IMySQLReadPacket::readPayload(ReadBuffer & in, uint8_t & sequence_id) readPayloadImpl(payload); if (!payload.eof()) { - std::stringstream tmp; - tmp.exceptions(std::ios::failbit); - tmp << "Packet payload is not fully read. Stopped after " << payload.count() << " bytes, while " << payload.available() << " bytes are in buffer."; - throw Exception(tmp.str(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Packet payload is not fully read. Stopped after {} bytes, while {} bytes are in buffer.", + payload.count(), payload.available()); } } diff --git a/src/Core/MySQL/IMySQLWritePacket.cpp b/src/Core/MySQL/IMySQLWritePacket.cpp index 3e97800177c..b5c95717a9b 100644 --- a/src/Core/MySQL/IMySQLWritePacket.cpp +++ b/src/Core/MySQL/IMySQLWritePacket.cpp @@ -1,10 +1,14 @@ #include #include -#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace MySQLProtocol { @@ -15,10 +19,8 @@ void IMySQLWritePacket::writePayload(WriteBuffer & buffer, uint8_t & sequence_id buf.next(); if (buf.remainingPayloadSize()) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << "Incomplete payload. Written " << getPayloadSize() - buf.remainingPayloadSize() << " bytes, expected " << getPayloadSize() << " bytes."; - throw Exception(ss.str(), 0); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Incomplete payload. Written {} bytes, expected {} bytes.", + getPayloadSize() - buf.remainingPayloadSize(), getPayloadSize()); } } diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index c09c4b3b034..6ff1670777a 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -35,15 +36,15 @@ namespace MySQLReplication payload.readStrict(reinterpret_cast(&flags), 2); } - void EventHeader::dump(std::ostream & out) const + void EventHeader::dump(WriteBuffer & out) const { - out << "\n=== " << to_string(this->type) << " ===" << std::endl; - out << "Timestamp: " << this->timestamp << std::endl; - out << "Event Type: " << this->type << std::endl; - out << "Server ID: " << this->server_id << std::endl; - out << "Event Size: " << this->event_size << std::endl; - out << "Log Pos: " << this->log_pos << std::endl; - out << "Flags: " << this->flags << std::endl; + out << "\n=== " << to_string(this->type) << " ===" << '\n'; + out << "Timestamp: " << this->timestamp << '\n'; + out << "Event Type: " << to_string(this->type) << '\n'; + out << "Server ID: " << this->server_id << '\n'; + out << "Event Size: " << this->event_size << '\n'; + out << "Log Pos: " << this->log_pos << '\n'; + out << "Flags: " << this->flags << '\n'; } /// https://dev.mysql.com/doc/internals/en/format-description-event.html @@ -60,13 +61,13 @@ namespace MySQLReplication readStringUntilEOF(event_type_header_length, payload); } - void FormatDescriptionEvent::dump(std::ostream & out) const + void FormatDescriptionEvent::dump(WriteBuffer & out) const { header.dump(out); - out << "Binlog Version: " << this->binlog_version << std::endl; - out << "Server Version: " << this->server_version << std::endl; - out << "Create Timestamp: " << this->create_timestamp << std::endl; - out << "Event Header Len: " << std::to_string(this->event_header_length) << std::endl; + out << "Binlog Version: " << this->binlog_version << '\n'; + out << "Server Version: " << this->server_version << '\n'; + out << "Create Timestamp: " << this->create_timestamp << '\n'; + out << "Event Header Len: " << std::to_string(this->event_header_length) << '\n'; } /// https://dev.mysql.com/doc/internals/en/rotate-event.html @@ -76,11 +77,11 @@ namespace MySQLReplication readStringUntilEOF(next_binlog, payload); } - void RotateEvent::dump(std::ostream & out) const + void RotateEvent::dump(WriteBuffer & out) const { header.dump(out); - out << "Position: " << this->position << std::endl; - out << "Next Binlog: " << this->next_binlog << std::endl; + out << "Position: " << this->position << '\n'; + out << "Next Binlog: " << this->next_binlog << '\n'; } /// https://dev.mysql.com/doc/internals/en/query-event.html @@ -116,24 +117,24 @@ namespace MySQLReplication } } - void QueryEvent::dump(std::ostream & out) const + void QueryEvent::dump(WriteBuffer & out) const { header.dump(out); - out << "Thread ID: " << this->thread_id << std::endl; - out << "Execution Time: " << this->exec_time << std::endl; - out << "Schema Len: " << std::to_string(this->schema_len) << std::endl; - out << "Error Code: " << this->error_code << std::endl; - out << "Status Len: " << this->status_len << std::endl; - out << "Schema: " << this->schema << std::endl; - out << "Query: " << this->query << std::endl; + out << "Thread ID: " << this->thread_id << '\n'; + out << "Execution Time: " << this->exec_time << '\n'; + out << "Schema Len: " << std::to_string(this->schema_len) << '\n'; + out << "Error Code: " << this->error_code << '\n'; + out << "Status Len: " << this->status_len << '\n'; + out << "Schema: " << this->schema << '\n'; + out << "Query: " << this->query << '\n'; } void XIDEvent::parseImpl(ReadBuffer & payload) { payload.readStrict(reinterpret_cast(&xid), 8); } - void XIDEvent::dump(std::ostream & out) const + void XIDEvent::dump(WriteBuffer & out) const { header.dump(out); - out << "XID: " << this->xid << std::endl; + out << "XID: " << this->xid << '\n'; } void TableMapEvent::parseImpl(ReadBuffer & payload) @@ -238,21 +239,23 @@ namespace MySQLReplication } } - void TableMapEvent::dump(std::ostream & out) const + void TableMapEvent::dump(WriteBuffer & out) const { header.dump(out); - out << "Table ID: " << this->table_id << std::endl; - out << "Flags: " << this->flags << std::endl; - out << "Schema Len: " << std::to_string(this->schema_len) << std::endl; - out << "Schema: " << this->schema << std::endl; - out << "Table Len: " << std::to_string(this->table_len) << std::endl; - out << "Table: " << this->table << std::endl; - out << "Column Count: " << this->column_count << std::endl; + out << "Table ID: " << this->table_id << '\n'; + out << "Flags: " << this->flags << '\n'; + out << "Schema Len: " << std::to_string(this->schema_len) << '\n'; + out << "Schema: " << this->schema << '\n'; + out << "Table Len: " << std::to_string(this->table_len) << '\n'; + out << "Table: " << this->table << '\n'; + out << "Column Count: " << this->column_count << '\n'; for (auto i = 0U; i < column_count; i++) { - out << "Column Type [" << i << "]: " << std::to_string(column_type[i]) << ", Meta: " << column_meta[i] << std::endl; + out << "Column Type [" << i << "]: " << std::to_string(column_type[i]) << ", Meta: " << column_meta[i] << '\n'; } - out << "Null Bitmap: " << this->null_bitmap << std::endl; + String bitmap_str; + boost::to_string(this->null_bitmap, bitmap_str); + out << "Null Bitmap: " << bitmap_str << '\n'; } void RowsEvent::parseImpl(ReadBuffer & payload) @@ -631,16 +634,16 @@ namespace MySQLReplication rows.push_back(row); } - void RowsEvent::dump(std::ostream & out) const + void RowsEvent::dump(WriteBuffer & out) const { FieldVisitorToString to_string; header.dump(out); - out << "Schema: " << this->schema << std::endl; - out << "Table: " << this->table << std::endl; + out << "Schema: " << this->schema << '\n'; + out << "Table: " << this->table << '\n'; for (auto i = 0U; i < rows.size(); i++) { - out << "Row[" << i << "]: " << applyVisitor(to_string, rows[i]) << std::endl; + out << "Row[" << i << "]: " << applyVisitor(to_string, rows[i]) << '\n'; } } @@ -663,22 +666,22 @@ namespace MySQLReplication payload.ignoreAll(); } - void GTIDEvent::dump(std::ostream & out) const + void GTIDEvent::dump(WriteBuffer & out) const { WriteBufferFromOwnString ws; writeUUIDText(gtid.uuid, ws); auto gtid_next = ws.str() + ":" + std::to_string(gtid.seq_no); header.dump(out); - out << "GTID Next: " << gtid_next << std::endl; + out << "GTID Next: " << gtid_next << '\n'; } void DryRunEvent::parseImpl(ReadBuffer & payload) { payload.ignoreAll(); } - void DryRunEvent::dump(std::ostream & out) const + void DryRunEvent::dump(WriteBuffer & out) const { header.dump(out); - out << "[DryRun Event]" << std::endl; + out << "[DryRun Event]" << '\n'; } /// Update binlog name/position/gtid based on the event type. @@ -716,12 +719,12 @@ namespace MySQLReplication gtid_sets.parse(gtid_sets_); } - void Position::dump(std::ostream & out) const + void Position::dump(WriteBuffer & out) const { - out << "\n=== Binlog Position ===" << std::endl; - out << "Binlog: " << this->binlog_name << std::endl; - out << "Position: " << this->binlog_pos << std::endl; - out << "GTIDSets: " << this->gtid_sets.toString() << std::endl; + out << "\n=== Binlog Position ===" << '\n'; + out << "Binlog: " << this->binlog_name << '\n'; + out << "Position: " << this->binlog_pos << '\n'; + out << "GTIDSets: " << this->gtid_sets.toString() << '\n'; } void MySQLFlavor::readPayloadImpl(ReadBuffer & payload) diff --git a/src/Core/MySQL/MySQLReplication.h b/src/Core/MySQL/MySQLReplication.h index 6f5b4cf0a1e..394ac729d1b 100644 --- a/src/Core/MySQL/MySQLReplication.h +++ b/src/Core/MySQL/MySQLReplication.h @@ -309,7 +309,7 @@ namespace MySQLReplication UInt16 flags; EventHeader() : timestamp(0), server_id(0), event_size(0), log_pos(0), flags(0) { } - void dump(std::ostream & out) const; + void dump(WriteBuffer & out) const; void parse(ReadBuffer & payload); }; @@ -321,7 +321,7 @@ namespace MySQLReplication EventBase(EventHeader && header_) : header(std::move(header_)) {} virtual ~EventBase() = default; - virtual void dump(std::ostream & out) const = 0; + virtual void dump(WriteBuffer & out) const = 0; virtual void parseEvent(ReadBuffer & payload) { parseImpl(payload); } virtual MySQLEventType type() const { return MYSQL_UNHANDLED_EVENT; } @@ -344,7 +344,7 @@ namespace MySQLReplication UInt8 event_header_length; String event_type_header_length; - void dump(std::ostream & out) const override; + void dump(WriteBuffer & out) const override; void parseImpl(ReadBuffer & payload) override; private: @@ -358,7 +358,7 @@ namespace MySQLReplication String next_binlog; RotateEvent(EventHeader && header_) : EventBase(std::move(header_)), position(0) {} - void dump(std::ostream & out) const override; + void dump(WriteBuffer & out) const override; protected: void parseImpl(ReadBuffer & payload) override; @@ -389,7 +389,7 @@ namespace MySQLReplication { } - void dump(std::ostream & out) const override; + void dump(WriteBuffer & out) const override; MySQLEventType type() const override { return MYSQL_QUERY_EVENT; } protected: @@ -404,7 +404,7 @@ namespace MySQLReplication protected: UInt64 xid; - void dump(std::ostream & out) const override; + void dump(WriteBuffer & out) const override; void parseImpl(ReadBuffer & payload) override; }; @@ -423,7 +423,7 @@ namespace MySQLReplication Bitmap null_bitmap; TableMapEvent(EventHeader && header_) : EventBase(std::move(header_)), table_id(0), flags(0), schema_len(0), table_len(0), column_count(0) {} - void dump(std::ostream & out) const override; + void dump(WriteBuffer & out) const override; protected: void parseImpl(ReadBuffer & payload) override; @@ -445,7 +445,7 @@ namespace MySQLReplication table = table_map->table; } - void dump(std::ostream & out) const override; + void dump(WriteBuffer & out) const override; protected: UInt64 table_id; @@ -489,7 +489,7 @@ namespace MySQLReplication GTID gtid; GTIDEvent(EventHeader && header_) : EventBase(std::move(header_)), commit_flag(0) {} - void dump(std::ostream & out) const override; + void dump(WriteBuffer & out) const override; protected: void parseImpl(ReadBuffer & payload) override; @@ -499,7 +499,7 @@ namespace MySQLReplication { public: DryRunEvent(EventHeader && header_) : EventBase(std::move(header_)) {} - void dump(std::ostream & out) const override; + void dump(WriteBuffer & out) const override; protected: void parseImpl(ReadBuffer & payload) override; @@ -515,7 +515,7 @@ namespace MySQLReplication Position() : binlog_pos(0) { } void update(BinlogEventPtr event); void update(UInt64 binlog_pos_, const String & binlog_name_, const String & gtid_sets_); - void dump(std::ostream & out) const; + void dump(WriteBuffer & out) const; }; class IFlavor : public MySQLProtocol::IMySQLReadPacket diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 98229bb73d7..79ac7ddf142 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -60,10 +60,7 @@ struct SortColumnDescription std::string dump() const { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << column_name << ":" << column_number << ":dir " << direction << "nulls " << nulls_direction; - return ss.str(); + return fmt::format("{}:{}:dir {}nulls ", column_name, column_number, direction, nulls_direction); } }; diff --git a/src/Core/tests/mysql_protocol.cpp b/src/Core/tests/mysql_protocol.cpp index 7e6aae5da23..9dc46891241 100644 --- a/src/Core/tests/mysql_protocol.cpp +++ b/src/Core/tests/mysql_protocol.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -329,6 +330,8 @@ int main(int argc, char ** argv) slave.connect(); slave.startBinlogDumpGTID(slave_id, replicate_db, gtid_sets); + WriteBufferFromOStream cerr(std::cerr); + /// Read one binlog event on by one. while (true) { @@ -337,40 +340,40 @@ int main(int argc, char ** argv) { case MYSQL_QUERY_EVENT: { auto binlog_event = std::static_pointer_cast(event); - binlog_event->dump(std::cerr); + binlog_event->dump(cerr); Position pos = slave.getPosition(); - pos.dump(std::cerr); + pos.dump(cerr); break; } case MYSQL_WRITE_ROWS_EVENT: { auto binlog_event = std::static_pointer_cast(event); - binlog_event->dump(std::cerr); + binlog_event->dump(cerr); Position pos = slave.getPosition(); - pos.dump(std::cerr); + pos.dump(cerr); break; } case MYSQL_UPDATE_ROWS_EVENT: { auto binlog_event = std::static_pointer_cast(event); - binlog_event->dump(std::cerr); + binlog_event->dump(cerr); Position pos = slave.getPosition(); - pos.dump(std::cerr); + pos.dump(cerr); break; } case MYSQL_DELETE_ROWS_EVENT: { auto binlog_event = std::static_pointer_cast(event); - binlog_event->dump(std::cerr); + binlog_event->dump(cerr); Position pos = slave.getPosition(); - pos.dump(std::cerr); + pos.dump(cerr); break; } default: if (event->header.type != MySQLReplication::EventType::HEARTBEAT_EVENT) { - event->dump(std::cerr); + event->dump(cerr); } break; } diff --git a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index d47d82689de..d4a892711ef 100644 --- a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -59,15 +59,10 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) /// Is violated. if (!value) { - std::stringstream exception_message; - exception_message.exceptions(std::ios::failbit); - - exception_message << "Constraint " << backQuote(constraint_ptr->name) - << " for table " << table_id.getNameForLogs() - << " is violated, because it is a constant expression returning 0." - << " It is most likely an error in table definition."; - - throw Exception{exception_message.str(), ErrorCodes::VIOLATED_CONSTRAINT}; + throw Exception(ErrorCodes::VIOLATED_CONSTRAINT, + "Constraint {} for table {} is violated, because it is a constant expression returning 0. " + "It is most likely an error in table definition.", + backQuote(constraint_ptr->name), table_id.getNameForLogs()); } } else @@ -87,28 +82,27 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) Names related_columns = constraint_expr->getRequiredColumns(); - std::stringstream exception_message; - exception_message.exceptions(std::ios::failbit); - - exception_message << "Constraint " << backQuote(constraint_ptr->name) - << " for table " << table_id.getNameForLogs() - << " is violated at row " << (rows_written + row_idx + 1) - << ". Expression: (" << serializeAST(*(constraint_ptr->expr), true) << ")" - << ". Column values"; - bool first = true; + String column_values_msg; + constexpr size_t approx_bytes_for_col = 32; + column_values_msg.reserve(approx_bytes_for_col * related_columns.size()); for (const auto & name : related_columns) { const IColumn & column = *block.getByName(name).column; assert(row_idx < column.size()); - exception_message << (first ? ": " : ", ") - << backQuoteIfNeed(name) << " = " << applyVisitor(FieldVisitorToString(), column[row_idx]); - + if (!first) + column_values_msg.append(", "); + column_values_msg.append(backQuoteIfNeed(name)); + column_values_msg.append(" = "); + column_values_msg.append(applyVisitor(FieldVisitorToString(), column[row_idx])); first = false; } - throw Exception{exception_message.str(), ErrorCodes::VIOLATED_CONSTRAINT}; + throw Exception(ErrorCodes::VIOLATED_CONSTRAINT, + "Constraint {} for table {} is violated at row {}. Expression: ({}). Column values: {}", + backQuote(constraint_ptr->name), table_id.getNameForLogs(), rows_written + row_idx + 1, + serializeAST(*(constraint_ptr->expr), true), column_values_msg); } } } diff --git a/src/DataStreams/IBlockInputStream.cpp b/src/DataStreams/IBlockInputStream.cpp index 23ba9ff2970..a6484c41b4f 100644 --- a/src/DataStreams/IBlockInputStream.cpp +++ b/src/DataStreams/IBlockInputStream.cpp @@ -4,7 +4,8 @@ #include #include #include -#include +#include +#include namespace ProfileEvents { @@ -359,8 +360,7 @@ Block IBlockInputStream::getExtremes() String IBlockInputStream::getTreeID() const { - std::stringstream s; - s.exceptions(std::ios::failbit); + WriteBufferFromOwnString s; s << getName(); if (!children.empty()) @@ -399,13 +399,13 @@ size_t IBlockInputStream::checkDepthImpl(size_t max_depth, size_t level) const } -void IBlockInputStream::dumpTree(std::ostream & ostr, size_t indent, size_t multiplier) const +void IBlockInputStream::dumpTree(WriteBuffer & ostr, size_t indent, size_t multiplier) const { ostr << String(indent, ' ') << getName(); if (multiplier > 1) ostr << " × " << multiplier; //ostr << ": " << getHeader().dumpStructure(); - ostr << std::endl; + ostr << '\n'; ++indent; /// If the subtree is repeated several times, then we output it once with the multiplier. diff --git a/src/DataStreams/IBlockInputStream.h b/src/DataStreams/IBlockInputStream.h index 3fbc3ce4bcd..4e314ef2980 100644 --- a/src/DataStreams/IBlockInputStream.h +++ b/src/DataStreams/IBlockInputStream.h @@ -95,7 +95,7 @@ public: virtual void readSuffix(); /// Must be called before `read()` and `readPrefix()`. - void dumpTree(std::ostream & ostr, size_t indent = 0, size_t multiplier = 1) const; + void dumpTree(WriteBuffer & ostr, size_t indent = 0, size_t multiplier = 1) const; /** Check the depth of the pipeline. * If max_depth is specified and the `depth` is greater - throw an exception. diff --git a/src/DataStreams/MergingSortedBlockInputStream.cpp b/src/DataStreams/MergingSortedBlockInputStream.cpp index 9c9213ef3cc..ee6b93dc8e4 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -1,6 +1,4 @@ #include -#include -#include #include diff --git a/src/DataStreams/MongoDBBlockInputStream.cpp b/src/DataStreams/MongoDBBlockInputStream.cpp index 25abdd909c4..5463d95151b 100644 --- a/src/DataStreams/MongoDBBlockInputStream.cpp +++ b/src/DataStreams/MongoDBBlockInputStream.cpp @@ -1,4 +1,3 @@ -#include #include #include diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp index 3fe17a4bbfc..1d063dac697 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp @@ -32,8 +32,7 @@ static const std::vector supported_functions{"any", "anyLast", "min", String DataTypeCustomSimpleAggregateFunction::getName() const { - std::stringstream stream; - stream.exceptions(std::ios::failbit); + WriteBufferFromOwnString stream; stream << "SimpleAggregateFunction(" << function->getName(); if (!parameters.empty()) diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index bd4329f6f58..6c325c5d371 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -29,10 +29,7 @@ namespace ErrorCodes template std::string DataTypeDecimal::doGetName() const { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << "Decimal(" << this->precision << ", " << this->scale << ")"; - return ss.str(); + return fmt::format("Decimal({}, {})", this->precision, this->scale); } diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp index 2a3de25c24f..7e42b2548b0 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp @@ -127,8 +127,7 @@ static String checkVariableAndGetVersion(const mysqlxx::Pool::Entry & connection } bool first = true; - std::stringstream error_message; - error_message.exceptions(std::ios::failbit); + WriteBufferFromOwnString error_message; error_message << "Illegal MySQL variables, the MaterializeMySQL engine requires "; for (const auto & [variable_name, variable_error_message] : variables_error_message) { @@ -239,8 +238,7 @@ static inline BlockOutputStreamPtr getTableOutput(const String & database_name, { const StoragePtr & storage = DatabaseCatalog::instance().getTable(StorageID(database_name, table_name), query_context); - std::stringstream insert_columns_str; - insert_columns_str.exceptions(std::ios::failbit); + WriteBufferFromOwnString insert_columns_str; const StorageInMemoryMetadata & storage_metadata = storage->getInMemoryMetadata(); const ColumnsDescription & storage_columns = storage_metadata.getColumns(); const NamesAndTypesList & insert_columns_names = insert_materialized ? storage_columns.getAllPhysical() : storage_columns.getOrdinary(); @@ -331,10 +329,9 @@ std::optional MaterializeMySQLSyncThread::prepareSynchroniz const auto & position_message = [&]() { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - position.dump(ss); - return ss.str(); + WriteBufferFromOwnString buf; + position.dump(buf); + return buf.str(); }; LOG_INFO(log, "MySQL dump database position: \n {}", position_message()); } @@ -374,10 +371,9 @@ void MaterializeMySQLSyncThread::flushBuffersData(Buffers & buffers, Materialize const auto & position_message = [&]() { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - client.getPosition().dump(ss); - return ss.str(); + WriteBufferFromOwnString buf; + client.getPosition().dump(buf); + return buf.str(); }; LOG_INFO(log, "MySQL executed position: \n {}", position_message()); } @@ -632,10 +628,9 @@ void MaterializeMySQLSyncThread::onEvent(Buffers & buffers, const BinlogEventPtr { const auto & dump_event_message = [&]() { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - receive_event->dump(ss); - return ss.str(); + WriteBufferFromOwnString buf; + receive_event->dump(buf); + return buf.str(); }; LOG_DEBUG(log, "Skip MySQL event: \n {}", dump_event_message()); diff --git a/src/Functions/abtesting.cpp b/src/Functions/abtesting.cpp index 051b1f6f0ef..9f919051ee9 100644 --- a/src/Functions/abtesting.cpp +++ b/src/Functions/abtesting.cpp @@ -3,7 +3,6 @@ #if !defined(ARCADIA_BUILD) && USE_STATS #include -#include #include #include diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 04fec145775..fa30afbd54f 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -239,7 +239,7 @@ void assertResponseIsOk(const Poco::Net::HTTPRequest & request, Poco::Net::HTTPR if (!(status == Poco::Net::HTTPResponse::HTTP_OK || (isRedirect(status) && allow_redirects))) { - std::stringstream error_message; + std::stringstream error_message; // STYLE_CHECK_ALLOW_STD_STRING_STREAM error_message.exceptions(std::ios::failbit); error_message << "Received error from remote server " << request.getURI() << ". HTTP status code: " << status << " " << response.getReason() << ", body: " << istr.rdbuf(); diff --git a/src/IO/MySQLPacketPayloadReadBuffer.cpp b/src/IO/MySQLPacketPayloadReadBuffer.cpp index f6f899e0ac7..9ca7845b2ae 100644 --- a/src/IO/MySQLPacketPayloadReadBuffer.cpp +++ b/src/IO/MySQLPacketPayloadReadBuffer.cpp @@ -1,5 +1,4 @@ #include -#include namespace DB { diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 49ccb6dc1b3..4a5b79e31ea 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -247,8 +249,7 @@ void PocoHTTPClient::makeRequestInternal( response->SetResponseCode(static_cast(status_code)); response->SetContentType(poco_response.getContentType()); - std::stringstream headers_ss; - headers_ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString headers_ss; for (const auto & [header_name, header_value] : poco_response) { response->AddHeader(header_name, header_value); diff --git a/src/IO/tests/gtest_bit_io.cpp b/src/IO/tests/gtest_bit_io.cpp index dce146eaad7..2302a6c6135 100644 --- a/src/IO/tests/gtest_bit_io.cpp +++ b/src/IO/tests/gtest_bit_io.cpp @@ -76,7 +76,7 @@ std::string dumpContents(const T& container, const size_t cols_in_row = 8) { - std::stringstream sstr; + std::stringstream sstr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM sstr.exceptions(std::ios::failbit); dumpBuffer(std::begin(container), std::end(container), &sstr, col_sep, row_sep, cols_in_row); diff --git a/src/IO/tests/hashing_read_buffer.cpp b/src/IO/tests/hashing_read_buffer.cpp index a1140160c32..67b3d61b1dc 100644 --- a/src/IO/tests/hashing_read_buffer.cpp +++ b/src/IO/tests/hashing_read_buffer.cpp @@ -22,7 +22,7 @@ static void test(size_t data_size) for (size_t read_buffer_block_size : block_sizes) { std::cout << "block size " << read_buffer_block_size << std::endl; - std::stringstream io; + std::stringstream io; // STYLE_CHECK_ALLOW_STD_STRING_STREAM io.exceptions(std::ios::failbit); DB::WriteBufferFromOStream out_impl(io); DB::HashingWriteBuffer out(out_impl); diff --git a/src/IO/tests/limit_read_buffer2.cpp b/src/IO/tests/limit_read_buffer2.cpp index 416eae0966b..d70ad832020 100644 --- a/src/IO/tests/limit_read_buffer2.cpp +++ b/src/IO/tests/limit_read_buffer2.cpp @@ -20,7 +20,7 @@ try { using namespace DB; - std::stringstream s; + std::stringstream s; // STYLE_CHECK_ALLOW_STD_STRING_STREAM s.exceptions(std::ios::failbit); { diff --git a/src/IO/tests/write_buffer.cpp b/src/IO/tests/write_buffer.cpp index c0e9150d372..09b46f810bf 100644 --- a/src/IO/tests/write_buffer.cpp +++ b/src/IO/tests/write_buffer.cpp @@ -16,7 +16,7 @@ int main(int, char **) DB::String c = "вася пе\tтя"; DB::String d = "'xyz\\"; - std::stringstream s; + std::stringstream s; // STYLE_CHECK_ALLOW_STD_STRING_STREAM s.exceptions(std::ios::failbit); { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index a22eacd9a49..75e586ec34f 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1965,21 +1965,19 @@ void Context::checkCanBeDropped(const String & database, const String & table, c String size_str = formatReadableSizeWithDecimalSuffix(size); String max_size_to_drop_str = formatReadableSizeWithDecimalSuffix(max_size_to_drop); - std::stringstream ostr; - ostr.exceptions(std::ios::failbit); - - ostr << "Table or Partition in " << backQuoteIfNeed(database) << "." << backQuoteIfNeed(table) << " was not dropped.\n" - << "Reason:\n" - << "1. Size (" << size_str << ") is greater than max_[table/partition]_size_to_drop (" << max_size_to_drop_str << ")\n" - << "2. File '" << force_file.path() << "' intended to force DROP " - << (force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist") << "\n"; - - ostr << "How to fix this:\n" - << "1. Either increase (or set to zero) max_[table/partition]_size_to_drop in server config\n" - << "2. Either create forcing file " << force_file.path() << " and make sure that ClickHouse has write permission for it.\n" - << "Example:\nsudo touch '" << force_file.path() << "' && sudo chmod 666 '" << force_file.path() << "'"; - - throw Exception(ostr.str(), ErrorCodes::TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT); + throw Exception(ErrorCodes::TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT, + "Table or Partition in {}.{} was not dropped.\nReason:\n" + "1. Size ({}) is greater than max_[table/partition]_size_to_drop ({})\n" + "2. File '{}' intended to force DROP {}\n" + "How to fix this:\n" + "1. Either increase (or set to zero) max_[table/partition]_size_to_drop in server config\n", + "2. Either create forcing file {} and make sure that ClickHouse has write permission for it.\n" + "Example:\nsudo touch '{}' && sudo chmod 666 '{}'", + backQuoteIfNeed(database), backQuoteIfNeed(table), + size_str, max_size_to_drop_str, + force_file.path(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist", + force_file.path(), + force_file.path(), force_file.path()); } diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 75872553ec3..582937c6ac1 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -453,8 +454,7 @@ void ExpressionAction::execute(Block & block, bool dry_run) const std::string ExpressionAction::toString() const { - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString ss; switch (type) { case ADD_COLUMN: @@ -550,8 +550,7 @@ void ExpressionActions::checkLimits(Block & block) const if (non_const_columns > settings.max_temporary_non_const_columns) { - std::stringstream list_of_non_const_columns; - list_of_non_const_columns.exceptions(std::ios::failbit); + WriteBufferFromOwnString list_of_non_const_columns; for (size_t i = 0, size = block.columns(); i < size; ++i) if (block.safeGetByPosition(i).column && !isColumnConst(*block.safeGetByPosition(i).column)) list_of_non_const_columns << "\n" << block.safeGetByPosition(i).name; @@ -922,8 +921,7 @@ void ExpressionActions::finalize(const Names & output_columns) std::string ExpressionActions::dumpActions() const { - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString ss; ss << "input:\n"; for (const auto & input_column : input_columns) @@ -1344,8 +1342,7 @@ void ExpressionActionsChain::finalize() std::string ExpressionActionsChain::dumpChain() const { - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString ss; for (size_t i = 0; i < steps.size(); ++i) { diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index ae8e2637cbb..b13350d7ba2 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -11,10 +11,8 @@ #include #include #include -#include #include -#include #include #include diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index a81245adfc9..3135b0cfdf2 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -26,7 +26,6 @@ #include #include #include -#include namespace DB diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 32f461863c9..907523ce94b 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -1,5 +1,3 @@ -#include - #include #include #include diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 8e67cecdd5e..861fa8c3505 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -6,8 +6,7 @@ #include #include #include -#include -#include +#include namespace DB @@ -32,8 +31,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() /// SHOW DATABASES if (query.databases) { - std::stringstream rewritten_query; - rewritten_query.exceptions(std::ios::failbit); + WriteBufferFromOwnString rewritten_query; rewritten_query << "SELECT name FROM system.databases"; if (!query.like.empty()) @@ -42,7 +40,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() << " WHERE name " << (query.not_like ? "NOT " : "") << (query.case_insensitive_like ? "ILIKE " : "LIKE ") - << std::quoted(query.like, '\''); + << DB::quote << query.like; } if (query.limit_length) @@ -54,8 +52,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() /// SHOW CLUSTER/CLUSTERS if (query.clusters) { - std::stringstream rewritten_query; - rewritten_query.exceptions(std::ios::failbit); + WriteBufferFromOwnString rewritten_query; rewritten_query << "SELECT DISTINCT cluster FROM system.clusters"; if (!query.like.empty()) @@ -64,7 +61,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() << " WHERE cluster " << (query.not_like ? "NOT " : "") << (query.case_insensitive_like ? "ILIKE " : "LIKE ") - << std::quoted(query.like, '\''); + << DB::quote << query.like; } if (query.limit_length) @@ -74,11 +71,10 @@ String InterpreterShowTablesQuery::getRewrittenQuery() } else if (query.cluster) { - std::stringstream rewritten_query; - rewritten_query.exceptions(std::ios::failbit); + WriteBufferFromOwnString rewritten_query; rewritten_query << "SELECT * FROM system.clusters"; - rewritten_query << " WHERE cluster = " << std::quoted(query.cluster_str, '\''); + rewritten_query << " WHERE cluster = " << DB::quote << query.cluster_str; return rewritten_query.str(); } @@ -89,8 +85,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() String database = context.resolveDatabase(query.from); DatabaseCatalog::instance().assertDatabaseExists(database); - std::stringstream rewritten_query; - rewritten_query.exceptions(std::ios::failbit); + WriteBufferFromOwnString rewritten_query; rewritten_query << "SELECT name FROM system."; if (query.dictionaries) @@ -107,14 +102,14 @@ String InterpreterShowTablesQuery::getRewrittenQuery() rewritten_query << "is_temporary"; } else - rewritten_query << "database = " << std::quoted(database, '\''); + rewritten_query << "database = " << DB::quote << database; if (!query.like.empty()) rewritten_query << " AND name " << (query.not_like ? "NOT " : "") << (query.case_insensitive_like ? "ILIKE " : "LIKE ") - << std::quoted(query.like, '\''); + << DB::quote << query.like; else if (query.where_expression) rewritten_query << " AND (" << query.where_expression << ")"; diff --git a/src/Interpreters/QueryAliasesVisitor.cpp b/src/Interpreters/QueryAliasesVisitor.cpp index 00c337920f4..d395bfc20e9 100644 --- a/src/Interpreters/QueryAliasesVisitor.cpp +++ b/src/Interpreters/QueryAliasesVisitor.cpp @@ -1,6 +1,3 @@ -#include -#include - #include #include #include diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 907cbaee243..8eeb939329b 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -342,11 +342,9 @@ void Set::checkColumnsNumber(size_t num_key_columns) const { if (data_types.size() != num_key_columns) { - std::stringstream message; - message.exceptions(std::ios::failbit); - message << "Number of columns in section IN doesn't match. " - << num_key_columns << " at left, " << data_types.size() << " at right."; - throw Exception(message.str(), ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns in section IN doesn't match. {} at left, {} at right.", + num_key_columns, data_types.size()); } } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 8d3cb123955..f6ff33112d0 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -552,8 +552,7 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select if (!unknown_required_source_columns.empty()) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString ss; ss << "Missing columns:"; for (const auto & name : unknown_required_source_columns) ss << " '" << name << "'"; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index cdb3d9b7d7b..9de788a94d4 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -778,11 +778,9 @@ static std::tuple executeQueryImpl( if (!internal && res.in) { - std::stringstream log_str; - log_str.exceptions(std::ios::failbit); - log_str << "Query pipeline:\n"; - res.in->dumpTree(log_str); - LOG_DEBUG(&Poco::Logger::get("executeQuery"), log_str.str()); + WriteBufferFromOwnString msg_buf; + res.in->dumpTree(msg_buf); + LOG_DEBUG(&Poco::Logger::get("executeQuery"), "Query pipeline:\n{}", msg_buf.str()); } } } diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index 241d3ff051a..30b001feeca 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index d51f37a0047..2717edd3b91 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include diff --git a/src/Parsers/ASTWithAlias.cpp b/src/Parsers/ASTWithAlias.cpp index 1feb89f4bdc..20f647fb575 100644 --- a/src/Parsers/ASTWithAlias.cpp +++ b/src/Parsers/ASTWithAlias.cpp @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/src/Parsers/DumpASTNode.h b/src/Parsers/DumpASTNode.h index 1208aeca2a9..27ea3b50d62 100644 --- a/src/Parsers/DumpASTNode.h +++ b/src/Parsers/DumpASTNode.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index 2a74c484187..bc00e2afe77 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 9428f312106..01ce4971c45 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -8,9 +8,7 @@ #include #include -#include #include -#include class SipHash; diff --git a/src/Parsers/formatAST.cpp b/src/Parsers/formatAST.cpp index 3a258df099e..fca8ea0aa35 100644 --- a/src/Parsers/formatAST.cpp +++ b/src/Parsers/formatAST.cpp @@ -1,6 +1,5 @@ #include -#include namespace DB { diff --git a/src/Parsers/formatAST.h b/src/Parsers/formatAST.h index bee89521812..15381b62028 100644 --- a/src/Parsers/formatAST.h +++ b/src/Parsers/formatAST.h @@ -16,16 +16,16 @@ void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite = true, bool one String serializeAST(const IAST & ast, bool one_line = true); -//inline std::ostream & operator<<(std::ostream & os, const IAST & ast) -//{ -// formatAST(ast, os, false, true); -// return os; -//} -// -//inline std::ostream & operator<<(std::ostream & os, const ASTPtr & ast) -//{ -// formatAST(*ast, os, false, true); -// return os; -//} +inline WriteBuffer & operator<<(WriteBuffer & buf, const IAST & ast) +{ + formatAST(ast, buf, false, true); + return buf; +} + +inline WriteBuffer & operator<<(WriteBuffer & buf, const ASTPtr & ast) +{ + formatAST(*ast, buf, false, true); + return buf; +} } diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp index 067c5cb0ab9..f40261b4561 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -2,8 +2,6 @@ #include #include #include -#include -#include namespace DB { diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp index 96a458eb49f..8bd4d36532d 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -163,15 +164,10 @@ void PrettyBlockOutputFormat::write(const Chunk & chunk, PortKind port_kind) ascii_grid_symbols; /// Create separators - std::stringstream top_separator; - std::stringstream middle_names_separator; - std::stringstream middle_values_separator; - std::stringstream bottom_separator; - - top_separator.exceptions(std::ios::failbit); - middle_names_separator.exceptions(std::ios::failbit); - middle_values_separator.exceptions(std::ios::failbit); - bottom_separator.exceptions(std::ios::failbit); + WriteBufferFromOwnString top_separator; + WriteBufferFromOwnString middle_names_separator; + WriteBufferFromOwnString middle_values_separator; + WriteBufferFromOwnString bottom_separator; top_separator << grid_symbols.bold_left_top_corner; middle_names_separator << grid_symbols.bold_left_separator; diff --git a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp index 9320b159836..cfa669ae8ad 100644 --- a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -132,22 +133,17 @@ void PrettyCompactBlockOutputFormat::writeBottom(const Widths & max_widths) const GridSymbols & grid_symbols = format_settings.pretty.charset == FormatSettings::Pretty::Charset::UTF8 ? utf8_grid_symbols : ascii_grid_symbols; - /// Create delimiters - std::stringstream bottom_separator; - bottom_separator.exceptions(std::ios::failbit); - - bottom_separator << grid_symbols.left_bottom_corner; + /// Write delimiters + out << grid_symbols.left_bottom_corner; for (size_t i = 0; i < max_widths.size(); ++i) { if (i != 0) - bottom_separator << grid_symbols.bottom_separator; + out << grid_symbols.bottom_separator; for (size_t j = 0; j < max_widths[i] + 2; ++j) - bottom_separator << grid_symbols.dash; + out << grid_symbols.dash; } - bottom_separator << grid_symbols.right_bottom_corner << "\n"; - - writeString(bottom_separator.str(), out); + out << grid_symbols.right_bottom_corner << "\n"; } void PrettyCompactBlockOutputFormat::writeRow( diff --git a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp index e7a7200ac34..8f6888b9a7a 100644 --- a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -41,8 +42,7 @@ void CollapsingSortedAlgorithm::reportIncorrectData() if (!log) return; - std::stringstream s; - s.exceptions(std::ios::failbit); + WriteBufferFromOwnString s; auto & sort_columns = *last_row.sort_columns; for (size_t i = 0, size = sort_columns.size(); i < size; ++i) { diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index bf7a3b8ab52..fc79ad9d134 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -36,8 +36,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request const MergeTreeSettings & settings = context.getReplicatedMergeTreeSettings(); bool ok = true; - std::stringstream message; - message.exceptions(std::ios::failbit); + WriteBufferFromOwnString message; auto databases = DatabaseCatalog::instance().getDatabases(); @@ -83,7 +82,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request } if (verbose) - response.send() << message.rdbuf(); + response.send() << message.str(); else { const char * data = "Ok.\n"; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 4dceb0aa905..f9d21b568b6 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -436,13 +436,9 @@ bool TCPHandler::readDataNext(const size_t & poll_interval, const int & receive_ double elapsed = watch.elapsedSeconds(); if (elapsed > receive_timeout) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << "Timeout exceeded while receiving data from client."; - ss << " Waited for " << static_cast(elapsed) << " seconds,"; - ss << " timeout is " << receive_timeout << " seconds."; - - throw Exception(ss.str(), ErrorCodes::SOCKET_TIMEOUT); + throw Exception(ErrorCodes::SOCKET_TIMEOUT, + "Timeout exceeded while receiving data from client. Waited for {} seconds, timeout is {} seconds.", + static_cast(elapsed), receive_timeout); } } diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index ef7c02163bb..093e5cea8ce 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -147,8 +147,7 @@ void DistributedBlockOutputStream::writeAsync(const Block & block) std::string DistributedBlockOutputStream::getCurrentStateDescription() { - std::stringstream buffer; - buffer.exceptions(std::ios::failbit); + WriteBufferFromOwnString buffer; const auto & addresses = cluster->getShardsAddresses(); buffer << "Insertion status:\n"; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 03eef37802f..effd2869be2 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -246,10 +246,7 @@ Names StorageKafka::parseTopics(String topic_list) String StorageKafka::getDefaultClientId(const StorageID & table_id_) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << VERSION_NAME << "-" << getFQDNOrHostName() << "-" << table_id_.database_name << "-" << table_id_.table_name; - return ss.str(); + return fmt::format("{}-{}-{}-{}", VERSION_NAME, getFQDNOrHostName(), table_id_.database_name, table_id_.table_name); } @@ -400,10 +397,7 @@ ConsumerBufferPtr StorageKafka::createReadBuffer(const size_t consumer_number) conf.set("group.id", group); if (num_consumers > 1) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << client_id << "-" << consumer_number; - conf.set("client.id", ss.str()); + conf.set("client.id", fmt::format("{}-{}", client_id, consumer_number)); } else { diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 265bc01be49..7c29b1582a6 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 58bdbcdcdcd..dd141a68248 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -442,12 +442,9 @@ void MinimalisticDataPartChecksums::checkEqualImpl(const MinimalisticDataPartChe { if (num_compressed_files != rhs.num_compressed_files || num_uncompressed_files != rhs.num_uncompressed_files) { - std::stringstream error_msg; - error_msg.exceptions(std::ios::failbit); - error_msg << "Different number of files: " << rhs.num_compressed_files << " compressed (expected " << num_compressed_files << ")" - << " and " << rhs.num_uncompressed_files << " uncompressed ones (expected " << num_uncompressed_files << ")"; - - throw Exception(error_msg.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, + "Different number of files: {} compressed (expected {}) and {} uncompressed ones (expected {})", + rhs.num_compressed_files, num_compressed_files, rhs.num_uncompressed_files, num_uncompressed_files); } Strings errors; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index f2b7882cbc9..5ebc0e4d423 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -228,14 +228,8 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( if (settings.force_primary_key && key_condition.alwaysUnknownOrTrue()) { - std::stringstream exception_message; - exception_message.exceptions(std::ios::failbit); - exception_message << "Primary key ("; - for (size_t i = 0, size = primary_key_columns.size(); i < size; ++i) - exception_message << (i == 0 ? "" : ", ") << primary_key_columns[i]; - exception_message << ") is not used and setting 'force_primary_key' is set."; - - throw Exception(exception_message.str(), ErrorCodes::INDEX_NOT_USED); + throw Exception(ErrorCodes::INDEX_NOT_USED, "Primary key ({}) is not used and setting 'force_primary_key' is set.", + boost::algorithm::join(primary_key_columns, ", ")); } std::optional minmax_idx_condition; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index aea740a3a26..9263dab638a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -941,8 +941,7 @@ size_t ReplicatedMergeTreeQueue::getConflictsCountForRange( if (out_description) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString ss; ss << "Can't execute command for range " << range.getPartName() << " (entry " << entry.znode_name << "). "; ss << "There are " << conflicts.size() << " currently executing entries blocking it: "; for (const auto & conflict : conflicts) diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index f8128e6223a..f9e3c0558d3 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -186,15 +186,10 @@ AMQP::ExchangeType StorageRabbitMQ::defineExchangeType(String exchange_type_) String StorageRabbitMQ::getTableBasedName(String name, const StorageID & table_id) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); - if (name.empty()) - ss << table_id.database_name << "_" << table_id.table_name; + return fmt::format("{}_{}", table_id.database_name, table_id.table_name); else - ss << name << "_" << table_id.database_name << "_" << table_id.table_name; - - return ss.str(); + return fmt::format("{}_{}_{}", name, table_id.database_name, table_id.table_name); } diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index ee08dd5a824..f81bcbc42d0 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include namespace DB @@ -32,12 +32,9 @@ namespace { if (names_and_types_set.find(column) == names_and_types_set.end()) { - std::string message = "Not found column "; - message += column.name + " " + column.type->getName(); - message += " in dictionary " + backQuote(dictionary_name) + ". "; - message += "There are only columns "; - message += StorageDictionary::generateNamesAndTypesDescription(dictionary_names_and_types); - throw Exception(message, ErrorCodes::THERE_IS_NO_COLUMN); + throw Exception(ErrorCodes::THERE_IS_NO_COLUMN, "Not found column {} {} in dictionary {}. There are only columns {}", + column.name, column.type->getName(), backQuote(dictionary_name), + StorageDictionary::generateNamesAndTypesDescription(dictionary_names_and_types)); } } } @@ -81,8 +78,7 @@ NamesAndTypesList StorageDictionary::getNamesAndTypes(const DictionaryStructure String StorageDictionary::generateNamesAndTypesDescription(const NamesAndTypesList & list) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString ss; bool first = true; for (const auto & name_and_type : list) { diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 5cc435f91fa..a4500e2aa7b 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -427,8 +428,7 @@ namespace String listOfColumns(const NamesAndTypesList & available_columns) { - std::stringstream ss; - ss.exceptions(std::ios::failbit); + WriteBufferFromOwnString ss; for (auto it = available_columns.begin(); it != available_columns.end(); ++it) { if (it != available_columns.begin()) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index fbeb188d649..1b47bd6c32d 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1013,17 +1013,13 @@ bool StorageMergeTree::optimize( { if (!merge(true, partition_id, true, deduplicate, &disable_reason)) { - std::stringstream message; - message.exceptions(std::ios::failbit); - message << "Cannot OPTIMIZE table"; - if (!disable_reason.empty()) - message << ": " << disable_reason; - else - message << " by some reason."; - LOG_INFO(log, message.str()); + constexpr const char * message = "Cannot OPTIMIZE table: {}"; + if (disable_reason.empty()) + disable_reason = "unknown reason"; + LOG_INFO(log, message, disable_reason); if (context.getSettingsRef().optimize_throw_if_noop) - throw Exception(message.str(), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); + throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason); return false; } } @@ -1036,17 +1032,13 @@ bool StorageMergeTree::optimize( if (!merge(true, partition_id, final, deduplicate, &disable_reason)) { - std::stringstream message; - message.exceptions(std::ios::failbit); - message << "Cannot OPTIMIZE table"; - if (!disable_reason.empty()) - message << ": " << disable_reason; - else - message << " by some reason."; - LOG_INFO(log, message.str()); + constexpr const char * message = "Cannot OPTIMIZE table: {}"; + if (disable_reason.empty()) + disable_reason = "unknown reason"; + LOG_INFO(log, message, disable_reason); if (context.getSettingsRef().optimize_throw_if_noop) - throw Exception(message.str(), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); + throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason); return false; } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 17de704ba40..c9e1b92bbbb 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -460,16 +460,12 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( if (!inactive_replicas.empty()) { - std::stringstream exception_message; - exception_message.exceptions(std::ios::failbit); - exception_message << "Mutation is not finished because"; + if (inactive_replicas.empty()) + throw Exception(ErrorCodes::UNFINISHED, "Mutation is not finished, it will be done asynchronously"); - if (!inactive_replicas.empty()) - exception_message << " some replicas are inactive right now: " << boost::algorithm::join(inactive_replicas, ", "); - - exception_message << ". Mutation will be done asynchronously"; - - throw Exception(exception_message.str(), ErrorCodes::UNFINISHED); + throw Exception(ErrorCodes::UNFINISHED, + "Mutation is not finished because some replicas are inactive right now: {}. Mutation will be done asynchronously", + boost::algorithm::join(inactive_replicas, ", ")); } } @@ -1018,13 +1014,6 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) for (const String & name : parts_to_fetch) parts_to_fetch_blocks += get_blocks_count_in_data_part(name); - std::stringstream sanity_report; - sanity_report.exceptions(std::ios::failbit); - sanity_report << "There are " - << unexpected_parts.size() << " unexpected parts with " << unexpected_parts_rows << " rows (" - << unexpected_parts_nonnew << " of them is not just-written with " << unexpected_parts_rows << " rows), " - << parts_to_fetch.size() << " missing parts (with " << parts_to_fetch_blocks << " blocks)."; - /** We can automatically synchronize data, * if the ratio of the total number of errors to the total number of parts (minimum - on the local filesystem or in ZK) * is no more than some threshold (for example 50%). @@ -1041,20 +1030,26 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) const auto storage_settings_ptr = getSettings(); bool insane = unexpected_parts_rows > total_rows_on_filesystem * storage_settings_ptr->replicated_max_ratio_of_wrong_parts; + constexpr const char * sanity_report_fmt = "The local set of parts of table {} doesn't look like the set of parts in ZooKeeper: " + "{} rows of {} total rows in filesystem are suspicious. " + "There are {} unexpected parts with {} rows ({} of them is not just-written with {} rows), " + "{} missing parts (with {} blocks)."; + if (insane && !skip_sanity_checks) { - std::stringstream why; - why.exceptions(std::ios::failbit); - why << "The local set of parts of table " << getStorageID().getNameForLogs() << " doesn't look like the set of parts " - << "in ZooKeeper: " - << formatReadableQuantity(unexpected_parts_rows) << " rows of " << formatReadableQuantity(total_rows_on_filesystem) - << " total rows in filesystem are suspicious."; - - throw Exception(why.str() + " " + sanity_report.str(), ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS); + throw Exception(ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS, sanity_report_fmt, getStorageID().getNameForLogs(), + formatReadableQuantity(unexpected_parts_rows), formatReadableQuantity(total_rows_on_filesystem), + unexpected_parts.size(), unexpected_parts_rows, unexpected_parts_nonnew, unexpected_parts_nonnew_rows, + parts_to_fetch.size(), parts_to_fetch_blocks); } if (unexpected_parts_nonnew_rows > 0) - LOG_WARNING(log, sanity_report.str()); + { + LOG_WARNING(log, sanity_report_fmt, getStorageID().getNameForLogs(), + formatReadableQuantity(unexpected_parts_rows), formatReadableQuantity(total_rows_on_filesystem), + unexpected_parts.size(), unexpected_parts_rows, unexpected_parts_nonnew, unexpected_parts_nonnew_rows, + parts_to_fetch.size(), parts_to_fetch_blocks); + } /// Add to the queue jobs to pick up the missing parts from other replicas and remove from ZK the information that we have them. std::vector> exists_futures; @@ -1343,15 +1338,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) { - // Log source part names just in case - { - std::stringstream source_parts_msg; - source_parts_msg.exceptions(std::ios::failbit); - for (auto i : ext::range(0, entry.source_parts.size())) - source_parts_msg << (i != 0 ? ", " : "") << entry.source_parts[i]; - - LOG_TRACE(log, "Executing log entry to merge parts {} to {}", source_parts_msg.str(), entry.new_part_name); - } + LOG_TRACE(log, "Executing log entry to merge parts {} to {}", boost::algorithm::join(entry.source_parts, ", "), entry.new_part_name); const auto storage_settings_ptr = getSettings(); @@ -3813,13 +3800,11 @@ bool StorageReplicatedMergeTree::optimize( if (!selected) { - std::stringstream message; - message.exceptions(std::ios::failbit); - message << "Cannot select parts for optimization"; - if (!disable_reason.empty()) - message << ": " << disable_reason; - LOG_INFO(log, message.str()); - return handle_noop(message.str()); + constexpr const char * message_fmt = "Cannot select parts for optimization: {}"; + if (disable_reason.empty()) + disable_reason = "unknown reason"; + LOG_INFO(log, message_fmt, disable_reason); + return handle_noop(fmt::format(message_fmt, disable_reason)); } ReplicatedMergeTreeLogEntryData merge_entry; diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 66c688f3195..e5c16dad958 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -35,8 +35,6 @@ #include #include -#include - namespace DB { diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index 81d3303e262..f35fb1c8a34 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 22a07a4d284..a031490b88b 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -241,13 +241,9 @@ TableFunctionRemote::TableFunctionRemote(const std::string & name_, bool secure_ : name{name_}, secure{secure_} { is_cluster_function = (name == "cluster" || name == "clusterAllReplicas"); - - std::stringstream ss; - ss.exceptions(std::ios::failbit); - ss << "Table function '" << name + "' requires from 2 to " << (is_cluster_function ? 3 : 5) << " parameters" - << ": , , " - << (is_cluster_function ? "" : ", [username, [password]]."); - help_message = ss.str(); + help_message = fmt::format("Table function '{}' requires from 2 to {} parameters: " + ", , {}", + name, is_cluster_function ? 3 : 5, is_cluster_function ? "" : ", [username, [password]]."); } diff --git a/utils/check-mysql-binlog/main.cpp b/utils/check-mysql-binlog/main.cpp index 4ec40ac41cc..0d831b84dce 100644 --- a/utils/check-mysql-binlog/main.cpp +++ b/utils/check-mysql-binlog/main.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include static DB::MySQLReplication::BinlogEventPtr parseSingleEventBody( @@ -126,18 +127,20 @@ static int checkBinLogFile(const std::string & bin_path, bool exist_checksum) } catch (...) { - std::cerr << "Unable to parse MySQL binlog event. Code: " << DB::getCurrentExceptionCode() << ", Exception message: " - << DB::getCurrentExceptionMessage(false) << std::endl << ", Previous event: " << std::endl; - last_event->dump(std::cerr); - std::cerr << std::endl << ", Event header: " << std::endl; - last_header->dump(std::cerr); - std::cerr << std::endl; + DB::WriteBufferFromOStream cerr(std::cerr); + cerr << "Unable to parse MySQL binlog event. Code: " << DB::getCurrentExceptionCode() << ", Exception message: " + << DB::getCurrentExceptionMessage(false) << '\n' << ", Previous event: " << '\n'; + last_event->dump(cerr); + cerr << '\n' << ", Event header: " << '\n'; + last_header->dump(cerr); + cerr << '\n'; return DB::getCurrentExceptionCode(); } - std::cout << "Check passed. " << std::endl << "No exception was thrown." << std::endl << "The last binlog event: " << std::endl; - last_event->dump(std::cout); - std::cout << std::endl; + DB::WriteBufferFromOStream cout(std::cout); + cout << "Check passed. " << '\n' << "No exception was thrown." << '\n' << "The last binlog event: " << '\n'; + last_event->dump(cout); + cout << '\n'; return 0; } diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 65ed4cec67a..56fd6843693 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -107,4 +107,4 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P ' $' | grep -P '.' && echo "^ Trailing whitespaces." # Forbid stringstream because it's easy to use them incorrectly and hard to debug possible issues -find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep 'std::ostringstream\|std::istringstream' | grep -v "STYLE_CHECK_ALLOW_STD_STRING_STREAM" && echo "Use WriteBufferFromString or ReadBufferFromString instead of std::ostringstream or std::istringstream" +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep 'std::[io]\?stringstream' | grep -v "STYLE_CHECK_ALLOW_STD_STRING_STREAM" && echo "Use WriteBufferFromOwnString or ReadBufferFromString instead of std::stringstream" diff --git a/utils/test-data-generator/ProtobufDelimitedMessagesSerializer.cpp b/utils/test-data-generator/ProtobufDelimitedMessagesSerializer.cpp index d16df83d12f..ad465913313 100644 --- a/utils/test-data-generator/ProtobufDelimitedMessagesSerializer.cpp +++ b/utils/test-data-generator/ProtobufDelimitedMessagesSerializer.cpp @@ -9,7 +9,7 @@ #include "00825_protobuf_format_syntax2.pb.h" -void writeInsertDataQueryForInputTest(std::stringstream & delimited_messages, const std::string & table_name, const std::string & format_schema, std::ostream & out) +void writeInsertDataQueryForInputTest(std::stringstream & delimited_messages, const std::string & table_name, const std::string & format_schema, std::ostream & out) // STYLE_CHECK_ALLOW_STD_STRING_STREAM { out << "echo -ne '"; std::string bytes = delimited_messages.str(); @@ -27,7 +27,7 @@ void writeInsertDataQueryForInputTest(std::stringstream & delimited_messages, co void writeInsertDataQueriesForInputTest(std::ostream & out) { - std::stringstream ss; + std::stringstream ss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM { Person person; person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49"); diff --git a/utils/zookeeper-cli/zookeeper-cli.cpp b/utils/zookeeper-cli/zookeeper-cli.cpp index 17a8c9f0da8..47423dcd202 100644 --- a/utils/zookeeper-cli/zookeeper-cli.cpp +++ b/utils/zookeeper-cli/zookeeper-cli.cpp @@ -80,7 +80,7 @@ int main(int argc, char ** argv) try { - std::stringstream ss(line); + std::stringstream ss(line); // STYLE_CHECK_ALLOW_STD_STRING_STREAM std::string cmd; ss >> cmd; From ac8b658a4ae4b5c12757cbe2b3d6499250e54495 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 Nov 2020 23:25:29 +0300 Subject: [PATCH 099/114] Try fix ci. --- src/AggregateFunctions/QuantileExact.h | 30 ++++++++++++++++++++++--- src/AggregateFunctions/QuantileTiming.h | 12 +++++++++- src/Columns/ColumnArray.cpp | 14 ++++++++++-- src/Columns/ColumnDecimal.cpp | 16 +++++++++++-- src/Columns/ColumnDecimal.h | 13 ++++++++++- src/Columns/ColumnFixedString.cpp | 20 ++++++++++++++++- src/Columns/ColumnLowCardinality.cpp | 9 +++++++- src/Columns/ColumnString.cpp | 14 ++++++++++-- src/Columns/ColumnTuple.cpp | 9 +++++++- src/Columns/ColumnVector.cpp | 20 ++++++++++++++++- 10 files changed, 142 insertions(+), 15 deletions(-) diff --git a/src/AggregateFunctions/QuantileExact.h b/src/AggregateFunctions/QuantileExact.h index de909c27565..f18ef75b1ba 100644 --- a/src/AggregateFunctions/QuantileExact.h +++ b/src/AggregateFunctions/QuantileExact.h @@ -8,7 +8,9 @@ #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif namespace DB { @@ -88,7 +90,11 @@ struct QuantileExact : QuantileExactBase> { size_t n = level < 1 ? level * array.size() : (array.size() - 1); +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_select(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm. +#else + std::nth_element(array.begin(), array.begin() + n, array.end()); /// NOTE You can think of the radix-select algorithm. +#endif return array[n]; } @@ -108,8 +114,11 @@ struct QuantileExact : QuantileExactBase> size_t n = level < 1 ? level * array.size() : (array.size() - 1); +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n, array.end()); - +#else + std::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); +#endif result[indices[i]] = array[n]; prev_n = n; } @@ -145,7 +154,11 @@ struct QuantileExactExclusive : public QuantileExact else if (n < 1) return static_cast(array[0]); +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_select(array.begin(), array.begin() + n - 1, array.end()); +#else + std::nth_element(array.begin(), array.begin() + n - 1, array.end()); +#endif auto nth_element = std::min_element(array.begin() + n, array.end()); return static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -174,7 +187,11 @@ struct QuantileExactExclusive : public QuantileExact result[indices[i]] = static_cast(array[0]); else { +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n - 1, array.end()); +#else + std::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); +#endif auto nth_element = std::min_element(array.begin() + n, array.end()); result[indices[i]] = static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -209,8 +226,11 @@ struct QuantileExactInclusive : public QuantileExact return static_cast(array[array.size() - 1]); else if (n < 1) return static_cast(array[0]); - +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_select(array.begin(), array.begin() + n - 1, array.end()); +#else + std::nth_element(array.begin(), array.begin() + n - 1, array.end()); +#endif auto nth_element = std::min_element(array.begin() + n, array.end()); return static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); @@ -237,7 +257,11 @@ struct QuantileExactInclusive : public QuantileExact result[indices[i]] = static_cast(array[0]); else { +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n - 1, array.end()); +#else + std::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); +#endif auto nth_element = std::min_element(array.begin() + n, array.end()); result[indices[i]] = static_cast(array[n - 1]) + (h - n) * static_cast(*nth_element - array[n - 1]); diff --git a/src/AggregateFunctions/QuantileTiming.h b/src/AggregateFunctions/QuantileTiming.h index 28bcde5c140..a1a8aabb5dc 100644 --- a/src/AggregateFunctions/QuantileTiming.h +++ b/src/AggregateFunctions/QuantileTiming.h @@ -7,7 +7,9 @@ #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif namespace DB { @@ -180,7 +182,11 @@ namespace detail /// Sorting an array will not be considered a violation of constancy. auto & array = elems; +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_select(array.begin(), array.begin() + n, array.end()); +#else + std::nth_element(array.begin(), array.begin() + n, array.end()); +#endif quantile = array[n]; } @@ -201,7 +207,11 @@ namespace detail ? level * elems.size() : (elems.size() - 1); +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_select(array.begin() + prev_n, array.begin() + n, array.end()); +#else + std::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); +#endif result[level_index] = array[n]; prev_n = n; diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 6dbe755f0ba..9bf0f929b8b 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -20,7 +20,10 @@ #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif + namespace DB { @@ -783,7 +786,11 @@ void ColumnArray::getPermutationImpl(size_t limit, Permutation & res, Comparator auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; }; if (limit) +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); +#else + std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); +#endif else std::sort(res.begin(), res.end(), less); } @@ -835,8 +842,11 @@ void ColumnArray::updatePermutationImpl(size_t limit, Permutation & res, EqualRa return; /// Since then we are working inside the interval. - +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); +#endif auto new_first = first; for (auto j = first + 1; j < limit; ++j) { diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 4285259a4f4..249dcfcf644 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -8,7 +8,10 @@ #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif + #include @@ -194,12 +197,21 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum /// Since then we are working inside the interval. if (reverse) +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, + [this](size_t a, size_t b) { return data[a] > data[b]; }); +#endif else +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); - +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, + [this](size_t a, size_t b) { return data[a] > data[b]; }); +#endif auto new_first = first; for (auto j = first + 1; j < limit; ++j) { diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 128b595803f..45fead7edad 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -7,7 +7,10 @@ #include #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif + namespace DB @@ -254,9 +257,17 @@ protected: sort_end = res.begin() + limit; if (reverse) +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; }); +#else + std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; }); +#endif else +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; }); +#else + std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; }); +#endif } }; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 41e46a7fa98..b2d48619af9 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -10,7 +10,9 @@ #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif #include @@ -158,9 +160,17 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir if (limit) { if (reverse) +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); +#else + std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); +#endif else +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); +#else + std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this)); +#endif } else { @@ -218,9 +228,17 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu /// Since then we are working inside the interval. if (reverse) +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); +#endif else +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); +#endif auto new_first = first; for (auto j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 9b87a409aaa..3514859842c 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -8,7 +8,10 @@ #include #include -#include + +#if !defined(ARCADIA_BUILD) + #include +#endif namespace DB { @@ -394,7 +397,11 @@ void ColumnLowCardinality::updatePermutationImpl(size_t limit, Permutation & res /// Since then we are working inside the interval. +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); +#endif auto new_first = first; for (auto j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 541863486a6..6ad13ec252b 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -10,7 +10,10 @@ #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif + namespace DB { @@ -314,7 +317,11 @@ void ColumnString::getPermutationImpl(size_t limit, Permutation & res, Comparato auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; }; if (limit) +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); +#else + std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); +#endif else std::sort(res.begin(), res.end(), less); } @@ -365,8 +372,11 @@ void ColumnString::updatePermutationImpl(size_t limit, Permutation & res, EqualR return; /// Since then we are working inside the interval. - +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less); +#endif size_t new_first = first; for (size_t j = first + 1; j < limit; ++j) diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index cbe5c7e11cd..e873f7d6530 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -9,7 +9,10 @@ #include #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif + namespace DB { @@ -352,7 +355,11 @@ void ColumnTuple::getPermutationImpl(size_t limit, Permutation & res, LessOperat if (limit) { +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less); +#else + std::partial_sort(res.begin(), res.begin() + limit, res.end(), less); +#endif } else { diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index e9af38d6984..99d92e9efa2 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -17,7 +17,9 @@ #include #include #include -#include +#if !defined(ARCADIA_BUILD) + #include +#endif #ifdef __SSE2__ #include @@ -156,9 +158,17 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi res[i] = i; if (reverse) +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint)); +#else + std::partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint)); +#endif else +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint)); +#else + std::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint)); +#endif } else { @@ -254,9 +264,17 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire /// Since then, we are working inside the interval. if (reverse) +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); +#endif else +#if !defined(ARCADIA_BUILD) miniselect::floyd_rivest_partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint)); +#else + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint)); +#endif size_t new_first = first; for (size_t j = first + 1; j < limit; ++j) From 0cca51f4aa4386c520f9a841ae796d0adb7d2236 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 11 Nov 2020 02:44:09 +0300 Subject: [PATCH 100/114] Check for wrong DWARF expressions for CFA --- contrib/libunwind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libunwind b/contrib/libunwind index 27026ef4a9c..198458b35f1 160000 --- a/contrib/libunwind +++ b/contrib/libunwind @@ -1 +1 @@ -Subproject commit 27026ef4a9c6c8cc956d1d131c4d794e24096981 +Subproject commit 198458b35f100da32bd3e74c2a3ce8d236db299b From d948bbd15c8f0b5d2a73680f8e8ccef223c06256 Mon Sep 17 00:00:00 2001 From: OuO Date: Wed, 11 Nov 2020 15:46:29 +0800 Subject: [PATCH 101/114] Update tips.md --- docs/zh/operations/tips.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/zh/operations/tips.md b/docs/zh/operations/tips.md index 4a8cf2efb96..511e8a22644 100644 --- a/docs/zh/operations/tips.md +++ b/docs/zh/operations/tips.md @@ -11,9 +11,9 @@ 不要禁用超线程。 它有助于某些查询，但不适用于其他查询。 -## 涡轮增压 {#turbo-boost} +## 超频 {#turbo-boost} -强烈推荐涡轮增压。 它显着提高了典型负载的性能。 +强烈推荐超频(turbo-boost)。 它显着提高了典型负载的性能。 您可以使用 `turbostat` 要查看负载下的CPU的实际时钟速率。 ## CPU缩放调控器 {#cpu-scaling-governor} @@ -39,18 +39,18 @@ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_gover 始终禁用交换文件。 不这样做的唯一原因是，如果您使用的ClickHouse在您的个人笔记本电脑。 -## 巨大的页面 {#huge-pages} +## 大页(Huge Pages) {#huge-pages} -始终禁用透明巨大的页面。 它会干扰内存分alloc，从而导致显着的性能下降。 +始终禁用透明大页(transparent huge pages)。 它会干扰内存分alloc，从而导致显着的性能下降。 ``` bash echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled ``` -使用 `perf top` 观看内核中用于内存管理的时间。 -永久巨大的页面也不需要被分配。 +使用 `perf top` 观察内核中用于内存管理的时间。 +永久大页(permanent huge pages)也不需要被分配。 -## 存储子系统 {#storage-subsystem} +## 存储系统 {#storage-subsystem} 如果您的预算允许您使用SSD，请使用SSD。 如果没有，请使用硬盘。 SATA硬盘7200转就行了。 @@ -100,27 +100,27 @@ XFS也是合适的，但它还没有经过ClickHouse的彻底测试。 如果可能的话，至少使用一个10GB的网络。 1Gb也可以工作，但对于使用数十tb的数据修补副本或处理具有大量中间数据的分布式查询，情况会更糟。 -## 动物园管理员 {#zookeeper} +## Zookeeper {#zookeeper} 您可能已经将ZooKeeper用于其他目的。 您可以使用相同的zookeeper安装，如果它还没有超载。 -It’s best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. +最好使用新版本的 Zookeeper – 3.4.9 或之后的版本. 稳定 Liunx 发行版中的 Zookeeper 版本可能是落后的。 -You should never use manually written scripts to transfer data between different ZooKeeper clusters, because the result will be incorrect for sequential nodes. Never use the «zkcopy» utility for the same reason: https://github.com/ksprojects/zkcopy/issues/15 +你永远不该使用自己手写的脚本在不同的 Zookeeper 集群之间转移数据, 这可能会导致序列节点的数据不正确。出于同样的原因，永远不要使用 zkcopy 工具: https://github.com/ksprojects/zkcopy/issues/15 如果要将现有ZooKeeper集群分为两个，正确的方法是增加其副本的数量，然后将其重新配置为两个独立的集群。 -不要在与ClickHouse相同的服务器上运行ZooKeeper。 由于ZooKeeper对延迟非常敏感，ClickHouse可能会利用所有可用的系统资源。 +不要在与ClickHouse相同的服务器上运行ZooKeeper。 因为ZooKeeper对延迟非常敏感，而ClickHouse可能会占用所有可用的系统资源。 -使用默认设置，ZooKeeper是一个定时炸弹: +默认设置下，ZooKeeper 就像是一个定时炸弹: -> 使用默认配置时，ZooKeeper服务器不会从旧快照和日志中删除文件（请参阅autopurge），这是操作员的责任。 +当使用默认配置时，ZooKeeper服务不会从旧快照和日志中删除文件（请参阅autopurge），这是操作员的责任。 -必须拆除炸弹 +必须拆除炸弹。 -下面的ZooKeeper（3.5.1）配置在Yandex中使用。梅地卡生产环境截至2017年5月20日: +下面的ZooKeeper（3.5.1）配置在 Yandex.Metrica 的生产环境中使用截至2017年5月20日: -动物园cfg: +zoo.cfg: ``` bash # http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html @@ -222,7 +222,7 @@ JAVA_OPTS="-Xms{{ '{{' }} cluster.get('xms','128M') {{ '}}' }} \ -XX:+CMSParallelRemarkEnabled" ``` -盐初始化: +Salt init: description "zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }} centralized coordination service" From 0410e383d9e562a53ab2c8a3904f88efc7dc46a9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 11 Nov 2020 12:46:36 +0300 Subject: [PATCH 102/114] Try fix check. --- src/AggregateFunctions/QuantileExact.h | 2 +- src/AggregateFunctions/QuantileTiming.h | 2 +- src/Columns/ColumnArray.cpp | 2 +- src/Columns/ColumnDecimal.cpp | 2 +- src/Columns/ColumnDecimal.h | 2 +- src/Columns/ColumnFixedString.cpp | 2 +- src/Columns/ColumnLowCardinality.cpp | 2 +- src/Columns/ColumnString.cpp | 2 +- src/Columns/ColumnTuple.cpp | 2 +- src/Columns/ColumnVector.cpp | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/AggregateFunctions/QuantileExact.h b/src/AggregateFunctions/QuantileExact.h index f18ef75b1ba..a2acde97f90 100644 --- a/src/AggregateFunctions/QuantileExact.h +++ b/src/AggregateFunctions/QuantileExact.h @@ -9,7 +9,7 @@ #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif namespace DB diff --git a/src/AggregateFunctions/QuantileTiming.h b/src/AggregateFunctions/QuantileTiming.h index a1a8aabb5dc..1a696088dd4 100644 --- a/src/AggregateFunctions/QuantileTiming.h +++ b/src/AggregateFunctions/QuantileTiming.h @@ -8,7 +8,7 @@ #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif namespace DB diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 9bf0f929b8b..82d954df334 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -21,7 +21,7 @@ #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 249dcfcf644..7c3af5fe095 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -9,7 +9,7 @@ #include #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 45fead7edad..cf9af32318e 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -8,7 +8,7 @@ #include #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index b2d48619af9..c1724b37fe2 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -11,7 +11,7 @@ #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif #include diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 3514859842c..f61062160b1 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -10,7 +10,7 @@ #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif namespace DB diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 6ad13ec252b..49180919abb 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -11,7 +11,7 @@ #include #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index e873f7d6530..9130f563735 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -10,7 +10,7 @@ #include #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 99d92e9efa2..c02963e4c5a 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -18,7 +18,7 @@ #include #include #if !defined(ARCADIA_BUILD) - #include + #include // Y_IGNORE #endif #ifdef __SSE2__ From 57c7ad1833595bdd2b0837f5a68863ebcbfece2a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 11 Nov 2020 12:47:48 +0300 Subject: [PATCH 103/114] Fix style. --- src/Columns/ColumnDecimal.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index cf9af32318e..abb49531e89 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -12,7 +12,6 @@ #endif - namespace DB { From fae862f2c374d320577e1dd43ae0f9469b1667c7 Mon Sep 17 00:00:00 2001 From: heng zhao Date: Wed, 11 Nov 2020 17:51:27 +0800 Subject: [PATCH 104/114] Update index.md fix mis-spellings --- docs/zh/engines/table-engines/log-family/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/engines/table-engines/log-family/index.md b/docs/zh/engines/table-engines/log-family/index.md index 64378a73daa..c45a68b2aae 100644 --- a/docs/zh/engines/table-engines/log-family/index.md +++ b/docs/zh/engines/table-engines/log-family/index.md @@ -46,6 +46,6 @@ toc_priority: 29 `Log` 引擎为表中的每一列使用不同的文件。`StripeLog` 将所有的数据存储在一个文件中。因此 `StripeLog` 引擎在操作系统中使用更少的描述符，但是 `Log` 引擎提供更高的读性能。 -`TingLog` 引擎是该系列中最简单的引擎并且提供了最少的功能和最低的性能。`TingLog` 引擎不支持并行读取和并发数据访问，并将每一列存储在不同的文件中。它比其余两种支持并行读取的引擎的读取速度更慢，并且使用了和 `Log` 引擎同样多的描述符。你可以在简单的低负载的情景下使用它。 +`TinyLog` 引擎是该系列中最简单的引擎并且提供了最少的功能和最低的性能。`TinyLog` 引擎不支持并行读取和并发数据访问，并将每一列存储在不同的文件中。它比其余两种支持并行读取的引擎的读取速度更慢，并且使用了和 `Log` 引擎同样多的描述符。你可以在简单的低负载的情景下使用它。 [来源文章](https://clickhouse.tech/docs/en/operations/table_engines/log_family/) From 66a45814858b8b915e9a67b0e9142f2829d8caf3 Mon Sep 17 00:00:00 2001 From: heng zhao Date: Wed, 11 Nov 2020 18:03:48 +0800 Subject: [PATCH 105/114] Update log.md engine name should not be translated. --- docs/zh/engines/table-engines/log-family/log.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/engines/table-engines/log-family/log.md b/docs/zh/engines/table-engines/log-family/log.md index 4a3855c4bfa..5d3ee38d745 100644 --- a/docs/zh/engines/table-engines/log-family/log.md +++ b/docs/zh/engines/table-engines/log-family/log.md @@ -1,5 +1,5 @@ -# 日志 {#log} +# Log {#log} -日志与 TinyLog 的不同之处在于，«标记» 的小文件与列文件存在一起。这些标记写在每个数据块上，并且包含偏移量，这些偏移量指示从哪里开始读取文件以便跳过指定的行数。这使得可以在多个线程中读取表数据。对于并发数据访问，可以同时执行读取操作，而写入操作则阻塞读取和其它写入。Log 引擎不支持索引。同样，如果写入表失败，则该表将被破坏，并且从该表读取将返回错误。Log 引擎适用于临时数据，write-once 表以及测试或演示目的。 +`Log` 与 `TinyLog` 的不同之处在于，«标记» 的小文件与列文件存在一起。这些标记写在每个数据块上，并且包含偏移量，这些偏移量指示从哪里开始读取文件以便跳过指定的行数。这使得可以在多个线程中读取表数据。对于并发数据访问，可以同时执行读取操作，而写入操作则阻塞读取和其它写入。`Log`引擎不支持索引。同样，如果写入表失败，则该表将被破坏，并且从该表读取将返回错误。`Log`引擎适用于临时数据，write-once 表以及测试或演示目的。 [原始文章](https://clickhouse.tech/docs/zh/operations/table_engines/log/) From 7da5e92754cda0e74c988472d45a4b579539ad0d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 11 Nov 2020 13:41:12 +0300 Subject: [PATCH 106/114] fix --- src/Storages/StorageReplicatedMergeTree.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c9e1b92bbbb..e27b9e8ec81 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -460,9 +460,6 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( if (!inactive_replicas.empty()) { - if (inactive_replicas.empty()) - throw Exception(ErrorCodes::UNFINISHED, "Mutation is not finished, it will be done asynchronously"); - throw Exception(ErrorCodes::UNFINISHED, "Mutation is not finished because some replicas are inactive right now: {}. Mutation will be done asynchronously", boost::algorithm::join(inactive_replicas, ", ")); From 072f481234dfdbf770f80529371811b4a052aeb2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 11 Nov 2020 15:01:16 +0300 Subject: [PATCH 107/114] Try fix tests. --- tests/queries/0_stateless/arcadia_skip_list.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6420eadfc09..25468835db4 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -165,4 +165,5 @@ 01548_query_log_query_execution_ms 01552_dict_fixedstring 01555_system_distribution_queue_mask -01557_max_parallel_replicas_no_sample.sql \ No newline at end of file +01557_max_parallel_replicas_no_sample.sql +01525_select_with_offset_fetch_clause From e311759c5d01052adf1a347fb1dea8786983063b Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 11 Nov 2020 15:15:16 +0300 Subject: [PATCH 108/114] Abort multipart upload if no data was written to WriteBufferFromS3. (#16840) --- src/IO/WriteBufferFromS3.cpp | 35 +++++++++++++++++------ tests/integration/test_storage_s3/test.py | 34 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 1ca8d8988d9..a32aa4acdc9 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -9,6 +9,7 @@ # include # include # include +# include # include # include # include @@ -167,6 +168,25 @@ void WriteBufferFromS3::complete() { if (is_multipart) { + if (part_tags.empty()) + { + LOG_DEBUG(log, "Multipart upload has no data. Aborting it. Bucket: {}, Key: {}, Upload_id: {}", bucket, key, upload_id); + + Aws::S3::Model::AbortMultipartUploadRequest req; + req.SetBucket(bucket); + req.SetKey(key); + req.SetUploadId(upload_id); + + auto outcome = client_ptr->AbortMultipartUpload(req); + + if (outcome.IsSuccess()) + LOG_DEBUG(log, "Aborting multipart upload completed. Upload_id: {}", upload_id); + else + throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); + + return; + } + LOG_DEBUG(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}", bucket, key, upload_id); Aws::S3::Model::CompleteMultipartUploadRequest req; @@ -174,18 +194,15 @@ void WriteBufferFromS3::complete() req.SetKey(key); req.SetUploadId(upload_id); - if (!part_tags.empty()) + Aws::S3::Model::CompletedMultipartUpload multipart_upload; + for (size_t i = 0; i < part_tags.size(); ++i) { - Aws::S3::Model::CompletedMultipartUpload multipart_upload; - for (size_t i = 0; i < part_tags.size(); ++i) - { - Aws::S3::Model::CompletedPart part; - multipart_upload.AddParts(part.WithETag(part_tags[i]).WithPartNumber(i + 1)); - } - - req.SetMultipartUpload(multipart_upload); + Aws::S3::Model::CompletedPart part; + multipart_upload.AddParts(part.WithETag(part_tags[i]).WithPartNumber(i + 1)); } + req.SetMultipartUpload(multipart_upload); + auto outcome = client_ptr->CompleteMultipartUpload(req); if (outcome.IsSuccess()) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 42dbeda4717..4a2cd77e233 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -136,6 +136,40 @@ def test_put(cluster, maybe_auth, positive): assert values_csv == get_s3_file_content(cluster, bucket, filename) +# Test put no data to S3. +@pytest.mark.parametrize("auth", [ + "'minio','minio123'," +]) +def test_empty_put(cluster, auth): + # type: (ClickHouseCluster) -> None + + bucket = cluster.minio_bucket + instance = cluster.instances["dummy"] # type: ClickHouseInstance + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" + + create_empty_table_query = """ + CREATE TABLE empty_table ( + {} + ) ENGINE = Null() + """.format(table_format) + + run_query(instance, create_empty_table_query) + + filename = "empty_put_test.csv" + put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') select * from empty_table".format( + cluster.minio_host, cluster.minio_port, bucket, filename, auth, table_format) + + run_query(instance, put_query) + + try: + run_query(instance, "select count(*) from s3('http://{}:{}/{}/{}', {}'CSV', '{}')".format( + cluster.minio_host, cluster.minio_port, bucket, filename, auth, table_format)) + + assert False, "Query should be failed." + except helpers.client.QueryRuntimeException as e: + assert str(e).find("The specified key does not exist") != 0 + + # Test put values in CSV format. @pytest.mark.parametrize("maybe_auth,positive", [ ("", True), From dbac7860b136d47342319cc28da28734d4c60d0e Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 11 Nov 2020 16:27:54 +0300 Subject: [PATCH 109/114] review suggestions --- programs/client/QueryFuzzer.cpp | 1 + src/Common/MemoryTracker.cpp | 10 ++++++---- src/IO/WriteHelpers.h | 5 ----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/programs/client/QueryFuzzer.cpp b/programs/client/QueryFuzzer.cpp index cb058f6db00..a8e32d87db5 100644 --- a/programs/client/QueryFuzzer.cpp +++ b/programs/client/QueryFuzzer.cpp @@ -422,6 +422,7 @@ void QueryFuzzer::fuzzMain(ASTPtr & ast) std::cout << std::endl; WriteBufferFromOStream ast_buf(std::cout, 4096); formatAST(*ast, ast_buf, false /*highlight*/); + ast_buf.next(); std::cout << std::endl << std::endl; } diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 1d324a00473..9f6db025886 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -136,8 +136,9 @@ void MemoryTracker::alloc(Int64 size) const auto * description = description_ptr.load(std::memory_order_relaxed); amount.fetch_sub(size, std::memory_order_relaxed); throw DB::Exception(DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, - "Memory tracker {}: fault injected. Would use {} (attempt to allocate chunk of {} bytes), maximum: {}", - description ? description : "", formatReadableSizeWithBinarySuffix(will_be), + "Memory tracker{}{}: fault injected. Would use {} (attempt to allocate chunk of {} bytes), maximum: {}", + description ? " " : "", description ? description : "", + formatReadableSizeWithBinarySuffix(will_be), size, formatReadableSizeWithBinarySuffix(current_hard_limit)); } @@ -164,8 +165,9 @@ void MemoryTracker::alloc(Int64 size) const auto * description = description_ptr.load(std::memory_order_relaxed); amount.fetch_sub(size, std::memory_order_relaxed); throw DB::Exception(DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, - "Memory limit {} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}", - description ? description : "", formatReadableSizeWithBinarySuffix(will_be), + "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}", + description ? " " : "", description ? description : "", + formatReadableSizeWithBinarySuffix(will_be), size, formatReadableSizeWithBinarySuffix(current_hard_limit)); } diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 4a117fefbee..dcf4e50e5af 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -271,11 +271,6 @@ inline void writeString(const StringRef & ref, WriteBuffer & buf) writeString(ref.data, ref.size, buf); } -//inline void writeString(const std::string_view & view, WriteBuffer & buf) -//{ -// writeString(view.data(), view.size(), buf); -//} - /** Writes a C-string without creating a temporary object. If the string is a literal, then `strlen` is executed at the compilation stage. * Use when the string is a literal. From bdbd41bd3db23e5b45a7a67dc646d4889f794e69 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Wed, 11 Nov 2020 18:29:36 +0300 Subject: [PATCH 110/114] Pytest test fixes (#16836) --- .../0_stateless/00039_inserts_through_http.sh | 2 + ...cated_merge_tree_alter_zookeeper.reference | 48 +- ..._replicated_merge_tree_alter_zookeeper.sql | 164 ++-- .../00070_insert_fewer_columns_http.sh | 1 + .../0_stateless/00386_long_in_pk.python | 2 + ...46_clear_column_in_partition_zookeeper.sql | 3 + ...ed_storage_definition_syntax_zookeeper.sql | 68 +- .../00643_cast_zookeeper.reference | 2 +- .../0_stateless/00643_cast_zookeeper.sql | 24 +- ..._create_temporary_table_with_in_clause.sql | 8 +- .../00719_insert_block_without_column.sh | 2 + ...fy_order_by_replicated_zookeeper.reference | 4 +- ...r_modify_order_by_replicated_zookeeper.sql | 46 +- ...dices_alter_replicated_zookeeper.reference | 24 +- ...836_indices_alter_replicated_zookeeper.sql | 114 +-- tests/queries/0_stateless/00900_orc_load.sh | 1 + ...om_compression_codecs_replicated.reference | 2 +- ...r_custom_compression_codecs_replicated.sql | 113 +-- .../00933_ttl_replicated_zookeeper.reference | 2 +- .../00933_ttl_replicated_zookeeper.sql | 31 +- ...nsert_into_distributed_different_types.sql | 3 + .../0_stateless/01056_create_table_as.sql | 4 + ..._materialize_clear_index_compact_parts.sql | 2 + ...1142_merge_join_lc_and_nullable_in_key.sql | 3 + ...01227_distributed_global_in_issue_2610.sql | 2 + ...ggregation_memory_efficient_mix_levels.sql | 5 + .../0_stateless/01231_operator_null_in.sql | 1 + tests/queries/0_stateless/01232_extremes.sql | 2 + ...ibuted_over_live_view_over_distributed.sql | 3 +- .../queries/0_stateless/01236_graphite_mt.sql | 10 +- .../0_stateless/01246_buffer_flush.sql | 2 + .../01246_finalize_aggregation_race.sql | 3 +- ...ted_group_by_sharding_key_dist_on_dist.sql | 1 + .../01268_data_numeric_parameters.sql | 4 + .../queries/0_stateless/01268_mv_scalars.sql | 2 + .../01272_totals_and_filter_bug.sql | 15 +- .../01273_lc_fixed_string_field.sql | 2 + .../queries/0_stateless/01275_parallel_mv.sql | 5 + .../0_stateless/01279_dist_group_by.sql | 2 + ..._max_threads_simple_query_optimization.sql | 2 + ...ence => 01283_strict_resize_bug.reference} | 0 ...ze_bug.sql => 01283_strict_resize_bug.sql} | 2 +- ...01285_data_skip_index_over_aggregation.sql | 2 + tests/queries/0_stateless/01291_geo_types.sql | 2 + .../01293_system_distribution_queue.sql | 3 + .../0_stateless/01296_pipeline_stuck.sql | 2 + tests/queries/0_stateless/01318_decrypt.sql | 2 + tests/queries/0_stateless/01318_encrypt.sql | 2 + .../0_stateless/01319_mv_constants_bug.sql | 1 + ...19_optimize_skip_unused_shards_nesting.sql | 4 + ...kip_unused_shards_no_non_deterministic.sql | 3 + ...num_partition_key_replicated_zookeeper.sql | 1 + .../0_stateless/01392_column_resolve.sql | 33 +- .../01400_join_get_with_multi_keys.sql | 1 + .../queries/0_stateless/01409_topK_merge.sql | 2 + .../01413_truncate_without_table_keyword.sql | 1 + .../01419_skip_index_compact_parts.sql | 4 +- .../01456_low_cardinality_sorting_bugfix.sql | 2 + ...457_create_as_table_function_structure.sql | 1 + .../01460_DistributedFilesToInsert.sql | 2 + .../01463_test_alter_live_view_refresh.sql | 3 + .../01487_distributed_in_not_default_db.sql | 1 + .../01511_prewhere_with_virtuals.sql | 4 +- ...3_optimize_aggregation_in_order_memory.sql | 2 + .../01514_empty_buffer_different_types.sql | 1 + .../01515_force_data_skipping_indices.sql | 2 + ...515_mv_and_array_join_optimisation_bag.sql | 10 +- .../01517_drop_mv_with_inner_table.sql | 7 + .../0_stateless/01526_initial_query_id.sh | 2 +- ...erministic_optimize_skip_unused_shards.sql | 2 + .../01530_drop_database_atomic_sync.sql | 8 +- .../01551_mergetree_read_in_order_spread.sql | 2 + tests/queries/conftest.py | 5 + tests/queries/query_test.py | 79 +- tests/queries/server.py | 842 ++++++++++++++++++ tests/queries/shell_config.sh | 3 + 76 files changed, 1403 insertions(+), 374 deletions(-) rename tests/queries/0_stateless/{001283_strict_resize_bug.reference => 01283_strict_resize_bug.reference} (100%) rename tests/queries/0_stateless/{001283_strict_resize_bug.sql => 01283_strict_resize_bug.sql} (90%) diff --git a/tests/queries/0_stateless/00039_inserts_through_http.sh b/tests/queries/0_stateless/00039_inserts_through_http.sh index 6b04437dcd6..35abcd166d7 100755 --- a/tests/queries/0_stateless/00039_inserts_through_http.sh +++ b/tests/queries/0_stateless/00039_inserts_through_http.sh @@ -10,3 +10,5 @@ for string_size in 1 10 100 1000 10000 100000 1000000; do LC_ALL=C perl -we 'for my $letter ("a" .. "z") { print(($letter x '$string_size') . "\n") }' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}&query=INSERT+INTO+long_insert+FORMAT+TabSeparated" --data-binary @- echo 'SELECT substring(a, 1, 1) AS c, length(a) AS l FROM long_insert ORDER BY c, l' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- done + +echo 'DROP TABLE long_insert' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- diff --git a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference index 69887db71c1..cb61ab3e9b9 100644 --- a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference +++ b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference @@ -1,22 +1,22 @@ d Date k UInt64 i32 Int32 -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 10 42 d Date k UInt64 i32 Int32 dt DateTime(\'UTC\') -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\')\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\')\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime(\'UTC\') -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\')\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\')\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 9 41 1992-01-01 08:00:00 2015-01-01 10 42 1970-01-01 00:00:00 d Date @@ -25,14 +25,14 @@ i32 Int32 dt DateTime(\'UTC\') n.ui8 Array(UInt8) n.s Array(String) -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime(\'UTC\') n.ui8 Array(UInt8) n.s Array(String) -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 2015-01-01 10 42 1970-01-01 00:00:00 [] [] @@ -43,7 +43,7 @@ dt DateTime(\'UTC\') n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -51,7 +51,7 @@ dt DateTime(\'UTC\') n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['1970-01-01','1970-01-01','1970-01-01'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] [] @@ -64,7 +64,7 @@ n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) s String DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date),\n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date),\n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -73,7 +73,7 @@ n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) s String DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date),\n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date),\n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 0 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['1970-01-01','1970-01-01','1970-01-01'] 0 @@ -86,7 +86,7 @@ dt DateTime(\'UTC\') n.ui8 Array(UInt8) n.s Array(String) s Int64 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -94,7 +94,7 @@ dt DateTime(\'UTC\') n.ui8 Array(UInt8) n.s Array(String) s Int64 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 @@ -108,7 +108,7 @@ n.ui8 Array(UInt8) n.s Array(String) s UInt32 DEFAULT \'0\' n.d Array(Date) -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\',\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\',\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -117,7 +117,7 @@ n.ui8 Array(UInt8) n.s Array(String) s UInt32 DEFAULT \'0\' n.d Array(Date) -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\',\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\',\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 ['1970-01-01','1970-01-01','1970-01-01'] 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 ['1970-01-01','1970-01-01','1970-01-01'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 ['1970-01-01','1970-01-01','1970-01-01'] @@ -129,14 +129,14 @@ i32 Int32 dt DateTime(\'UTC\') n.s Array(String) s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime(\'UTC\') n.s Array(String) s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 ['12','13','14'] 0 @@ -147,13 +147,13 @@ k UInt64 i32 Int32 dt DateTime(\'UTC\') s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime(\'UTC\') s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 @@ -166,7 +166,7 @@ dt DateTime(\'UTC\') s UInt32 DEFAULT \'0\' n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\',\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\',\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -174,7 +174,7 @@ dt DateTime(\'UTC\') s UInt32 DEFAULT \'0\' n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\',\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\',\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 [] [] 2015-01-01 7 39 2014-07-14 13:26:50 0 [] [] 2015-01-01 8 40 2012-12-12 12:12:12 0 [] [] @@ -185,13 +185,13 @@ k UInt64 i32 Int32 dt DateTime(\'UTC\') s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime(\'UTC\') s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime(\'UTC\'),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 @@ -202,13 +202,13 @@ k UInt64 i32 Int32 dt Date s DateTime(\'UTC\') DEFAULT \'1970-01-01 00:00:00\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` Date,\n `s` DateTime(\'UTC\') DEFAULT \'1970-01-01 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) +CREATE TABLE default.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` Date,\n `s` DateTime(\'UTC\') DEFAULT \'1970-01-01 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt Date s DateTime(\'UTC\') DEFAULT \'1970-01-01 00:00:00\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` Date,\n `s` DateTime(\'UTC\') DEFAULT \'1970-01-01 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) +CREATE TABLE default.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` Date,\n `s` DateTime(\'UTC\') DEFAULT \'1970-01-01 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00062/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 1970-01-02 03:55:00 2015-01-01 7 39 2014-07-14 1970-01-01 00:00:00 2015-01-01 8 40 2012-12-12 1970-01-01 00:00:00 diff --git a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.sql b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.sql index 2a933d21eeb..ac56b3416cd 100644 --- a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.sql +++ b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.sql @@ -1,110 +1,110 @@ -DROP TABLE IF EXISTS test.replicated_alter1; -DROP TABLE IF EXISTS test.replicated_alter2; +DROP TABLE IF EXISTS replicated_alter1; +DROP TABLE IF EXISTS replicated_alter2; SET replication_alter_partitions_sync = 2; -CREATE TABLE test.replicated_alter1 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_00062/alter', 'r1', d, k, 8192); -CREATE TABLE test.replicated_alter2 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_00062/alter', 'r2', d, k, 8192); +CREATE TABLE replicated_alter1 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_00062/alter', 'r1', d, k, 8192); +CREATE TABLE replicated_alter2 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_00062/alter', 'r2', d, k, 8192); -INSERT INTO test.replicated_alter1 VALUES ('2015-01-01', 10, 42); +INSERT INTO replicated_alter1 VALUES ('2015-01-01', 10, 42); -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 ADD COLUMN dt DateTime('UTC'); -INSERT INTO test.replicated_alter1 VALUES ('2015-01-01', 9, 41, '1992-01-01 08:00:00'); +ALTER TABLE replicated_alter1 ADD COLUMN dt DateTime('UTC'); +INSERT INTO replicated_alter1 VALUES ('2015-01-01', 9, 41, '1992-01-01 08:00:00'); -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 ADD COLUMN n Nested(ui8 UInt8, s String); -INSERT INTO test.replicated_alter1 VALUES ('2015-01-01', 8, 40, '2012-12-12 12:12:12', [1,2,3], ['12','13','14']); +ALTER TABLE replicated_alter1 ADD COLUMN n Nested(ui8 UInt8, s String); +INSERT INTO replicated_alter1 VALUES ('2015-01-01', 8, 40, '2012-12-12 12:12:12', [1,2,3], ['12','13','14']); -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 ADD COLUMN `n.d` Array(Date); -INSERT INTO test.replicated_alter1 VALUES ('2015-01-01', 7, 39, '2014-07-14 13:26:50', [10,20,30], ['120','130','140'],['2000-01-01','2000-01-01','2000-01-03']); +ALTER TABLE replicated_alter1 ADD COLUMN `n.d` Array(Date); +INSERT INTO replicated_alter1 VALUES ('2015-01-01', 7, 39, '2014-07-14 13:26:50', [10,20,30], ['120','130','140'],['2000-01-01','2000-01-01','2000-01-03']); -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 ADD COLUMN s String DEFAULT '0'; -INSERT INTO test.replicated_alter1 VALUES ('2015-01-01', 6,38,'2014-07-15 13:26:50',[10,20,30],['asd','qwe','qwe'],['2000-01-01','2000-01-01','2000-01-03'],'100500'); +ALTER TABLE replicated_alter1 ADD COLUMN s String DEFAULT '0'; +INSERT INTO replicated_alter1 VALUES ('2015-01-01', 6,38,'2014-07-15 13:26:50',[10,20,30],['asd','qwe','qwe'],['2000-01-01','2000-01-01','2000-01-03'],'100500'); -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 DROP COLUMN `n.d`, MODIFY COLUMN s Int64; +ALTER TABLE replicated_alter1 DROP COLUMN `n.d`, MODIFY COLUMN s Int64; -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 ADD COLUMN `n.d` Array(Date), MODIFY COLUMN s UInt32; +ALTER TABLE replicated_alter1 ADD COLUMN `n.d` Array(Date), MODIFY COLUMN s UInt32; -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 DROP COLUMN n.ui8, DROP COLUMN n.d; +ALTER TABLE replicated_alter1 DROP COLUMN n.ui8, DROP COLUMN n.d; -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 DROP COLUMN n.s; +ALTER TABLE replicated_alter1 DROP COLUMN n.s; -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 ADD COLUMN n.s Array(String), ADD COLUMN n.d Array(Date); +ALTER TABLE replicated_alter1 ADD COLUMN n.s Array(String), ADD COLUMN n.d Array(Date); -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 DROP COLUMN n; +ALTER TABLE replicated_alter1 DROP COLUMN n; -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -ALTER TABLE test.replicated_alter1 MODIFY COLUMN dt Date, MODIFY COLUMN s DateTime('UTC') DEFAULT '1970-01-01 00:00:00'; +ALTER TABLE replicated_alter1 MODIFY COLUMN dt Date, MODIFY COLUMN s DateTime('UTC') DEFAULT '1970-01-01 00:00:00'; -DESC TABLE test.replicated_alter1; -SHOW CREATE TABLE test.replicated_alter1; -DESC TABLE test.replicated_alter2; -SHOW CREATE TABLE test.replicated_alter2; -SELECT * FROM test.replicated_alter1 ORDER BY k; +DESC TABLE replicated_alter1; +SHOW CREATE TABLE replicated_alter1; +DESC TABLE replicated_alter2; +SHOW CREATE TABLE replicated_alter2; +SELECT * FROM replicated_alter1 ORDER BY k; -DROP TABLE test.replicated_alter1; -DROP TABLE test.replicated_alter2; +DROP TABLE replicated_alter1; +DROP TABLE replicated_alter2; diff --git a/tests/queries/0_stateless/00070_insert_fewer_columns_http.sh b/tests/queries/0_stateless/00070_insert_fewer_columns_http.sh index 853d4e161c9..0cf5f95d3d9 100755 --- a/tests/queries/0_stateless/00070_insert_fewer_columns_http.sh +++ b/tests/queries/0_stateless/00070_insert_fewer_columns_http.sh @@ -8,3 +8,4 @@ echo 'DROP TABLE IF EXISTS insert_fewer_columns' | ${ echo 'CREATE TABLE insert_fewer_columns (a UInt8, b UInt8) ENGINE = Memory' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- echo 'INSERT INTO insert_fewer_columns (a) VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- echo 'SELECT * FROM insert_fewer_columns' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- +echo 'DROP TABLE insert_fewer_columns' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @- diff --git a/tests/queries/0_stateless/00386_long_in_pk.python b/tests/queries/0_stateless/00386_long_in_pk.python index ab5fc50d8e3..e33bb254c60 100644 --- a/tests/queries/0_stateless/00386_long_in_pk.python +++ b/tests/queries/0_stateless/00386_long_in_pk.python @@ -50,6 +50,8 @@ def main(): print(resp.text) break + requests.post(url, data='drop table tab_00386') + if __name__ == "__main__": main() diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql index bd6c12ffce4..5d8c4de1c06 100644 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql @@ -71,3 +71,6 @@ OPTIMIZE TABLE clear_column1 PARTITION '200002'; ALTER TABLE clear_column1 CLEAR COLUMN s IN PARTITION '200012', CLEAR COLUMN i IN PARTITION '200012'; -- Drop empty partition also Ok ALTER TABLE clear_column1 DROP PARTITION '200012', DROP PARTITION '200011'; + +DROP TABLE clear_column1; +DROP TABLE clear_column2; diff --git a/tests/queries/0_stateless/00509_extended_storage_definition_syntax_zookeeper.sql b/tests/queries/0_stateless/00509_extended_storage_definition_syntax_zookeeper.sql index 4a3dd2981cd..4ed053f5953 100644 --- a/tests/queries/0_stateless/00509_extended_storage_definition_syntax_zookeeper.sql +++ b/tests/queries/0_stateless/00509_extended_storage_definition_syntax_zookeeper.sql @@ -1,81 +1,81 @@ SELECT '*** Replicated with sampling ***'; -DROP TABLE IF EXISTS test.replicated_with_sampling; +DROP TABLE IF EXISTS replicated_with_sampling; -CREATE TABLE test.replicated_with_sampling(x UInt8) +CREATE TABLE replicated_with_sampling(x UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00509/replicated_with_sampling', 'r1') ORDER BY x SAMPLE BY x; -INSERT INTO test.replicated_with_sampling VALUES (1), (128); -SELECT sum(x) FROM test.replicated_with_sampling SAMPLE 1/2; +INSERT INTO replicated_with_sampling VALUES (1), (128); +SELECT sum(x) FROM replicated_with_sampling SAMPLE 1/2; -DROP TABLE test.replicated_with_sampling; +DROP TABLE replicated_with_sampling; SELECT '*** Replacing with implicit version ***'; -DROP TABLE IF EXISTS test.replacing; +DROP TABLE IF EXISTS replacing; -CREATE TABLE test.replacing(d Date, x UInt32, s String) ENGINE = ReplacingMergeTree ORDER BY x PARTITION BY d; +CREATE TABLE replacing(d Date, x UInt32, s String) ENGINE = ReplacingMergeTree ORDER BY x PARTITION BY d; -INSERT INTO test.replacing VALUES ('2017-10-23', 1, 'a'); -INSERT INTO test.replacing VALUES ('2017-10-23', 1, 'b'); -INSERT INTO test.replacing VALUES ('2017-10-23', 1, 'c'); +INSERT INTO replacing VALUES ('2017-10-23', 1, 'a'); +INSERT INTO replacing VALUES ('2017-10-23', 1, 'b'); +INSERT INTO replacing VALUES ('2017-10-23', 1, 'c'); -OPTIMIZE TABLE test.replacing PARTITION '2017-10-23' FINAL; +OPTIMIZE TABLE replacing PARTITION '2017-10-23' FINAL; -SELECT * FROM test.replacing; +SELECT * FROM replacing; -DROP TABLE test.replacing; +DROP TABLE replacing; SELECT '*** Replicated Collapsing ***'; -DROP TABLE IF EXISTS test.replicated_collapsing; +DROP TABLE IF EXISTS replicated_collapsing; -CREATE TABLE test.replicated_collapsing(d Date, x UInt32, sign Int8) +CREATE TABLE replicated_collapsing(d Date, x UInt32, sign Int8) ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/test_00509/replicated_collapsing', 'r1', sign) PARTITION BY toYYYYMM(d) ORDER BY d; -INSERT INTO test.replicated_collapsing VALUES ('2017-10-23', 1, 1); -INSERT INTO test.replicated_collapsing VALUES ('2017-10-23', 1, -1), ('2017-10-23', 2, 1); +INSERT INTO replicated_collapsing VALUES ('2017-10-23', 1, 1); +INSERT INTO replicated_collapsing VALUES ('2017-10-23', 1, -1), ('2017-10-23', 2, 1); -SYSTEM SYNC REPLICA test.replicated_collapsing; -OPTIMIZE TABLE test.replicated_collapsing PARTITION 201710 FINAL; +SYSTEM SYNC REPLICA replicated_collapsing; +OPTIMIZE TABLE replicated_collapsing PARTITION 201710 FINAL; -SELECT * FROM test.replicated_collapsing; +SELECT * FROM replicated_collapsing; -DROP TABLE test.replicated_collapsing; +DROP TABLE replicated_collapsing; SELECT '*** Replicated VersionedCollapsing ***'; -DROP TABLE IF EXISTS test.replicated_versioned_collapsing; +DROP TABLE IF EXISTS replicated_versioned_collapsing; -CREATE TABLE test.replicated_versioned_collapsing(d Date, x UInt32, sign Int8, version UInt8) +CREATE TABLE replicated_versioned_collapsing(d Date, x UInt32, sign Int8, version UInt8) ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/test_00509/replicated_versioned_collapsing', 'r1', sign, version) PARTITION BY toYYYYMM(d) ORDER BY (d, version); -INSERT INTO test.replicated_versioned_collapsing VALUES ('2017-10-23', 1, 1, 0); -INSERT INTO test.replicated_versioned_collapsing VALUES ('2017-10-23', 1, -1, 0), ('2017-10-23', 2, 1, 0); -INSERT INTO test.replicated_versioned_collapsing VALUES ('2017-10-23', 1, -1, 1), ('2017-10-23', 2, 1, 2); +INSERT INTO replicated_versioned_collapsing VALUES ('2017-10-23', 1, 1, 0); +INSERT INTO replicated_versioned_collapsing VALUES ('2017-10-23', 1, -1, 0), ('2017-10-23', 2, 1, 0); +INSERT INTO replicated_versioned_collapsing VALUES ('2017-10-23', 1, -1, 1), ('2017-10-23', 2, 1, 2); -SYSTEM SYNC REPLICA test.replicated_versioned_collapsing; -OPTIMIZE TABLE test.replicated_versioned_collapsing PARTITION 201710 FINAL; +SYSTEM SYNC REPLICA replicated_versioned_collapsing; +OPTIMIZE TABLE replicated_versioned_collapsing PARTITION 201710 FINAL; -SELECT * FROM test.replicated_versioned_collapsing; +SELECT * FROM replicated_versioned_collapsing; -DROP TABLE test.replicated_versioned_collapsing; +DROP TABLE replicated_versioned_collapsing; SELECT '*** Table definition with SETTINGS ***'; -DROP TABLE IF EXISTS test.with_settings; +DROP TABLE IF EXISTS with_settings; -CREATE TABLE test.with_settings(x UInt32) +CREATE TABLE with_settings(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00509/with_settings', 'r1') ORDER BY x SETTINGS replicated_can_become_leader = 0; SELECT sleep(1); -- If replicated_can_become_leader were true, this replica would become the leader after 1 second. -SELECT is_leader FROM system.replicas WHERE database = 'test' AND table = 'with_settings'; +SELECT is_leader FROM system.replicas WHERE database = currentDatabase() AND table = 'with_settings'; -DROP TABLE test.with_settings; +DROP TABLE with_settings; diff --git a/tests/queries/0_stateless/00643_cast_zookeeper.reference b/tests/queries/0_stateless/00643_cast_zookeeper.reference index eee4b660bda..9123463de1a 100644 --- a/tests/queries/0_stateless/00643_cast_zookeeper.reference +++ b/tests/queries/0_stateless/00643_cast_zookeeper.reference @@ -1,4 +1,4 @@ -CREATE TABLE test.cast1 +CREATE TABLE default.cast1 ( `x` UInt8, `e` Enum8('hello' = 1, 'world' = 2) DEFAULT CAST(x, 'Enum8(\'hello\' = 1, \'world\' = 2)') diff --git a/tests/queries/0_stateless/00643_cast_zookeeper.sql b/tests/queries/0_stateless/00643_cast_zookeeper.sql index db439425eb8..c52d44bd88b 100644 --- a/tests/queries/0_stateless/00643_cast_zookeeper.sql +++ b/tests/queries/0_stateless/00643_cast_zookeeper.sql @@ -1,7 +1,7 @@ -DROP TABLE IF EXISTS test.cast1; -DROP TABLE IF EXISTS test.cast2; +DROP TABLE IF EXISTS cast1; +DROP TABLE IF EXISTS cast2; -CREATE TABLE test.cast1 +CREATE TABLE cast1 ( x UInt8, e Enum8 @@ -22,17 +22,17 @@ CREATE TABLE test.cast1 ) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00643/cast', 'r1') ORDER BY e; -SHOW CREATE TABLE test.cast1 FORMAT TSVRaw; -DESC TABLE test.cast1; +SHOW CREATE TABLE cast1 FORMAT TSVRaw; +DESC TABLE cast1; -INSERT INTO test.cast1 (x) VALUES (1); -SELECT * FROM test.cast1; +INSERT INTO cast1 (x) VALUES (1); +SELECT * FROM cast1; -CREATE TABLE test.cast2 AS test.cast1 ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00643/cast', 'r2') ORDER BY e; +CREATE TABLE cast2 AS cast1 ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00643/cast', 'r2') ORDER BY e; -SYSTEM SYNC REPLICA test.cast2; +SYSTEM SYNC REPLICA cast2; -SELECT * FROM test.cast2; +SELECT * FROM cast2; -DROP TABLE test.cast1; -DROP TABLE test.cast2; +DROP TABLE cast1; +DROP TABLE cast2; diff --git a/tests/queries/0_stateless/00714_create_temporary_table_with_in_clause.sql b/tests/queries/0_stateless/00714_create_temporary_table_with_in_clause.sql index 751f766d0a1..e56c3bd1d7b 100644 --- a/tests/queries/0_stateless/00714_create_temporary_table_with_in_clause.sql +++ b/tests/queries/0_stateless/00714_create_temporary_table_with_in_clause.sql @@ -1,10 +1,10 @@ DROP TEMPORARY TABLE IF EXISTS temporary_table; -DROP TEMPORARY TABLE IF EXISTS test_merge_1; -DROP TEMPORARY TABLE IF EXISTS test_merge_2; +DROP TABLE IF EXISTS test_merge_1; +DROP TABLE IF EXISTS test_merge_2; CREATE TABLE test_merge_1(id UInt64) ENGINE = Log; CREATE TABLE test_merge_2(id UInt64) ENGINE = Log; CREATE TEMPORARY TABLE temporary_table AS SELECT * FROM numbers(1) WHERE number NOT IN (SELECT id FROM merge(currentDatabase(), 'test_merge_1|test_merge_2')); SELECT * FROM temporary_table; DROP TEMPORARY TABLE IF EXISTS temporary_table; -DROP TEMPORARY TABLE IF EXISTS test_merge_1; -DROP TEMPORARY TABLE IF EXISTS test_merge_2; +DROP TABLE IF EXISTS test_merge_1; +DROP TABLE IF EXISTS test_merge_2; diff --git a/tests/queries/0_stateless/00719_insert_block_without_column.sh b/tests/queries/0_stateless/00719_insert_block_without_column.sh index 5b7619e7b85..384445b1ae6 100755 --- a/tests/queries/0_stateless/00719_insert_block_without_column.sh +++ b/tests/queries/0_stateless/00719_insert_block_without_column.sh @@ -19,3 +19,5 @@ ${CLICKHOUSE_CLIENT} --query "create table squashed_numbers (SomeID UInt64, Diff ${CLICKHOUSE_CURL} -sS --data-binary "@${CLICKHOUSE_TMP}/test_squashing_block_without_column.out" "${CLICKHOUSE_URL}&query=insert%20into%20squashed_numbers%20format%20Native" ${CLICKHOUSE_CLIENT} --query "select 'Still alive'" + +${CLICKHOUSE_CLIENT} --query "drop table squashed_numbers" diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference index 4a7d4e3ff00..48fe2d30bf3 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference @@ -9,6 +9,6 @@ 1 2 1 30 1 2 4 90 *** Check SHOW CREATE TABLE *** -CREATE TABLE test.summing_r2\n(\n `x` UInt32,\n `y` UInt32,\n `z` UInt32,\n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test_00754/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, -z)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.summing_r2\n(\n `x` UInt32,\n `y` UInt32,\n `z` UInt32,\n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test_00754/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, -z)\nSETTINGS index_granularity = 8192 *** Check SHOW CREATE TABLE after offline ALTER *** -CREATE TABLE test.summing_r2\n(\n `x` UInt32,\n `y` UInt32,\n `z` UInt32,\n `t` UInt32,\n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test_00754/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, t * t)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.summing_r2\n(\n `x` UInt32,\n `y` UInt32,\n `z` UInt32,\n `t` UInt32,\n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test_00754/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, t * t)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql index 0f861749537..4b150fb3826 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql @@ -5,48 +5,48 @@ CREATE TABLE old_style(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse ALTER TABLE old_style ADD COLUMN y UInt32, MODIFY ORDER BY (x, y); -- { serverError 36 } DROP TABLE old_style; -DROP TABLE IF EXISTS test.summing_r1; -DROP TABLE IF EXISTS test.summing_r2; -CREATE TABLE test.summing_r1(x UInt32, y UInt32, val UInt32) ENGINE ReplicatedSummingMergeTree('/clickhouse/tables/test_00754/summing', 'r1') ORDER BY (x, y); -CREATE TABLE test.summing_r2(x UInt32, y UInt32, val UInt32) ENGINE ReplicatedSummingMergeTree('/clickhouse/tables/test_00754/summing', 'r2') ORDER BY (x, y); +DROP TABLE IF EXISTS summing_r1; +DROP TABLE IF EXISTS summing_r2; +CREATE TABLE summing_r1(x UInt32, y UInt32, val UInt32) ENGINE ReplicatedSummingMergeTree('/clickhouse/tables/test_00754/summing', 'r1') ORDER BY (x, y); +CREATE TABLE summing_r2(x UInt32, y UInt32, val UInt32) ENGINE ReplicatedSummingMergeTree('/clickhouse/tables/test_00754/summing', 'r2') ORDER BY (x, y); /* Can't add an expression with existing column to ORDER BY. */ -ALTER TABLE test.summing_r1 MODIFY ORDER BY (x, y, -val); -- { serverError 36 } +ALTER TABLE summing_r1 MODIFY ORDER BY (x, y, -val); -- { serverError 36 } /* Can't add an expression with existing column to ORDER BY. */ -ALTER TABLE test.summing_r1 ADD COLUMN z UInt32 DEFAULT x + 1, MODIFY ORDER BY (x, y, -z); -- { serverError 36 } +ALTER TABLE summing_r1 ADD COLUMN z UInt32 DEFAULT x + 1, MODIFY ORDER BY (x, y, -z); -- { serverError 36 } /* Can't add nonexistent column to ORDER BY. */ -ALTER TABLE test.summing_r1 MODIFY ORDER BY (x, y, nonexistent); -- { serverError 47 } +ALTER TABLE summing_r1 MODIFY ORDER BY (x, y, nonexistent); -- { serverError 47 } /* Can't modyfy ORDER BY so that it is no longer a prefix of the PRIMARY KEY. */ -ALTER TABLE test.summing_r1 MODIFY ORDER BY x; -- { serverError 36 } +ALTER TABLE summing_r1 MODIFY ORDER BY x; -- { serverError 36 } -INSERT INTO test.summing_r1(x, y, val) VALUES (1, 2, 10), (1, 2, 20); -SYSTEM SYNC REPLICA test.summing_r2; +INSERT INTO summing_r1(x, y, val) VALUES (1, 2, 10), (1, 2, 20); +SYSTEM SYNC REPLICA summing_r2; -ALTER TABLE test.summing_r1 ADD COLUMN z UInt32 AFTER y, MODIFY ORDER BY (x, y, -z); +ALTER TABLE summing_r1 ADD COLUMN z UInt32 AFTER y, MODIFY ORDER BY (x, y, -z); -INSERT INTO test.summing_r1(x, y, z, val) values (1, 2, 1, 30), (1, 2, 2, 40), (1, 2, 2, 50); -SYSTEM SYNC REPLICA test.summing_r2; +INSERT INTO summing_r1(x, y, z, val) values (1, 2, 1, 30), (1, 2, 2, 40), (1, 2, 2, 50); +SYSTEM SYNC REPLICA summing_r2; SELECT '*** Check that the parts are sorted according to the new key. ***'; -SELECT * FROM test.summing_r2 ORDER BY _part; +SELECT * FROM summing_r2 ORDER BY _part; SELECT '*** Check that the rows are collapsed according to the new key. ***'; -SELECT * FROM test.summing_r2 FINAL ORDER BY x, y, z; +SELECT * FROM summing_r2 FINAL ORDER BY x, y, z; SELECT '*** Check SHOW CREATE TABLE ***'; -SHOW CREATE TABLE test.summing_r2; +SHOW CREATE TABLE summing_r2; -DETACH TABLE test.summing_r2; -ALTER TABLE test.summing_r1 ADD COLUMN t UInt32 AFTER z, MODIFY ORDER BY (x, y, t * t) SETTINGS replication_alter_partitions_sync = 2; -- { serverError 341 } -ATTACH TABLE test.summing_r2; +DETACH TABLE summing_r2; +ALTER TABLE summing_r1 ADD COLUMN t UInt32 AFTER z, MODIFY ORDER BY (x, y, t * t) SETTINGS replication_alter_partitions_sync = 2; -- { serverError 341 } +ATTACH TABLE summing_r2; -SYSTEM SYNC REPLICA test.summing_r2; +SYSTEM SYNC REPLICA summing_r2; SELECT '*** Check SHOW CREATE TABLE after offline ALTER ***'; -SHOW CREATE TABLE test.summing_r2; +SHOW CREATE TABLE summing_r2; -DROP TABLE test.summing_r1; -DROP TABLE test.summing_r2; +DROP TABLE summing_r1; +DROP TABLE summing_r2; diff --git a/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference b/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference index 4029f0666b7..838bd93ebaf 100644 --- a/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference @@ -1,5 +1,5 @@ -CREATE TABLE test.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 @@ -14,8 +14,8 @@ CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDE 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 4 1 5 @@ -28,10 +28,10 @@ CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDE 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 4 1 5 @@ -44,14 +44,14 @@ CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDE 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx2\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx2_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx2\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx2_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 3 1 2 1 3 -CREATE TABLE test.minmax_idx2\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx2_r\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx2\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx2_r\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00836/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 3 1 2 diff --git a/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.sql b/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.sql index 5773357c675..e038d2d425e 100644 --- a/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.sql +++ b/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.sql @@ -1,74 +1,74 @@ -DROP TABLE IF EXISTS test.minmax_idx; -DROP TABLE IF EXISTS test.minmax_idx_r; -DROP TABLE IF EXISTS test.minmax_idx2; -DROP TABLE IF EXISTS test.minmax_idx2_r; +DROP TABLE IF EXISTS minmax_idx; +DROP TABLE IF EXISTS minmax_idx_r; +DROP TABLE IF EXISTS minmax_idx2; +DROP TABLE IF EXISTS minmax_idx2_r; SET replication_alter_partitions_sync = 2; -CREATE TABLE test.minmax_idx +CREATE TABLE minmax_idx ( u64 UInt64, i32 Int32 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00836/indices_alter1', 'r1') ORDER BY u64; -CREATE TABLE test.minmax_idx_r +CREATE TABLE minmax_idx_r ( u64 UInt64, i32 Int32 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00836/indices_alter1', 'r2') ORDER BY u64; -INSERT INTO test.minmax_idx VALUES (1, 2); +INSERT INTO minmax_idx VALUES (1, 2); -SYSTEM SYNC REPLICA test.minmax_idx_r; +SYSTEM SYNC REPLICA minmax_idx_r; -ALTER TABLE test.minmax_idx ADD INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10; -ALTER TABLE test.minmax_idx_r ADD INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10; -ALTER TABLE test.minmax_idx ADD INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10 AFTER idx1; +ALTER TABLE minmax_idx ADD INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10; +ALTER TABLE minmax_idx_r ADD INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10; +ALTER TABLE minmax_idx ADD INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10 AFTER idx1; -SHOW CREATE TABLE test.minmax_idx; -SHOW CREATE TABLE test.minmax_idx_r; +SHOW CREATE TABLE minmax_idx; +SHOW CREATE TABLE minmax_idx_r; -SELECT * FROM test.minmax_idx WHERE u64 * i32 = 2 ORDER BY (u64, i32); -SELECT * FROM test.minmax_idx_r WHERE u64 * i32 = 2 ORDER BY (u64, i32); +SELECT * FROM minmax_idx WHERE u64 * i32 = 2 ORDER BY (u64, i32); +SELECT * FROM minmax_idx_r WHERE u64 * i32 = 2 ORDER BY (u64, i32); -INSERT INTO test.minmax_idx VALUES (1, 4); -INSERT INTO test.minmax_idx_r VALUES (3, 2); -INSERT INTO test.minmax_idx VALUES (1, 5); -INSERT INTO test.minmax_idx_r VALUES (65, 75); -INSERT INTO test.minmax_idx VALUES (19, 9); +INSERT INTO minmax_idx VALUES (1, 4); +INSERT INTO minmax_idx_r VALUES (3, 2); +INSERT INTO minmax_idx VALUES (1, 5); +INSERT INTO minmax_idx_r VALUES (65, 75); +INSERT INTO minmax_idx VALUES (19, 9); -SYSTEM SYNC REPLICA test.minmax_idx; -SYSTEM SYNC REPLICA test.minmax_idx_r; +SYSTEM SYNC REPLICA minmax_idx; +SYSTEM SYNC REPLICA minmax_idx_r; -SELECT * FROM test.minmax_idx WHERE u64 * i32 > 1 ORDER BY (u64, i32); -SELECT * FROM test.minmax_idx_r WHERE u64 * i32 > 1 ORDER BY (u64, i32); +SELECT * FROM minmax_idx WHERE u64 * i32 > 1 ORDER BY (u64, i32); +SELECT * FROM minmax_idx_r WHERE u64 * i32 > 1 ORDER BY (u64, i32); -ALTER TABLE test.minmax_idx DROP INDEX idx1; +ALTER TABLE minmax_idx DROP INDEX idx1; -SHOW CREATE TABLE test.minmax_idx; -SHOW CREATE TABLE test.minmax_idx_r; +SHOW CREATE TABLE minmax_idx; +SHOW CREATE TABLE minmax_idx_r; -SELECT * FROM test.minmax_idx WHERE u64 * i32 > 1 ORDER BY (u64, i32); -SELECT * FROM test.minmax_idx_r WHERE u64 * i32 > 1 ORDER BY (u64, i32); +SELECT * FROM minmax_idx WHERE u64 * i32 > 1 ORDER BY (u64, i32); +SELECT * FROM minmax_idx_r WHERE u64 * i32 > 1 ORDER BY (u64, i32); -ALTER TABLE test.minmax_idx DROP INDEX idx2; -ALTER TABLE test.minmax_idx_r DROP INDEX idx3; +ALTER TABLE minmax_idx DROP INDEX idx2; +ALTER TABLE minmax_idx_r DROP INDEX idx3; -SHOW CREATE TABLE test.minmax_idx; -SHOW CREATE TABLE test.minmax_idx_r; +SHOW CREATE TABLE minmax_idx; +SHOW CREATE TABLE minmax_idx_r; -ALTER TABLE test.minmax_idx ADD INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10; +ALTER TABLE minmax_idx ADD INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10; -SHOW CREATE TABLE test.minmax_idx; -SHOW CREATE TABLE test.minmax_idx_r; +SHOW CREATE TABLE minmax_idx; +SHOW CREATE TABLE minmax_idx_r; -SELECT * FROM test.minmax_idx WHERE u64 * i32 > 1 ORDER BY (u64, i32); -SELECT * FROM test.minmax_idx_r WHERE u64 * i32 > 1 ORDER BY (u64, i32); +SELECT * FROM minmax_idx WHERE u64 * i32 > 1 ORDER BY (u64, i32); +SELECT * FROM minmax_idx_r WHERE u64 * i32 > 1 ORDER BY (u64, i32); -CREATE TABLE test.minmax_idx2 +CREATE TABLE minmax_idx2 ( u64 UInt64, i32 Int32, @@ -77,7 +77,7 @@ CREATE TABLE test.minmax_idx2 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00836/indices_alter2', 'r1') ORDER BY u64; -CREATE TABLE test.minmax_idx2_r +CREATE TABLE minmax_idx2_r ( u64 UInt64, i32 Int32, @@ -87,27 +87,27 @@ CREATE TABLE test.minmax_idx2_r ORDER BY u64; -SHOW CREATE TABLE test.minmax_idx2; -SHOW CREATE TABLE test.minmax_idx2_r; +SHOW CREATE TABLE minmax_idx2; +SHOW CREATE TABLE minmax_idx2_r; -INSERT INTO test.minmax_idx2 VALUES (1, 2); -INSERT INTO test.minmax_idx2_r VALUES (1, 3); +INSERT INTO minmax_idx2 VALUES (1, 2); +INSERT INTO minmax_idx2_r VALUES (1, 3); -SYSTEM SYNC REPLICA test.minmax_idx2; -SYSTEM SYNC REPLICA test.minmax_idx2_r; +SYSTEM SYNC REPLICA minmax_idx2; +SYSTEM SYNC REPLICA minmax_idx2_r; -SELECT * FROM test.minmax_idx2 WHERE u64 * i32 >= 2 ORDER BY (u64, i32); -SELECT * FROM test.minmax_idx2_r WHERE u64 * i32 >= 2 ORDER BY (u64, i32); +SELECT * FROM minmax_idx2 WHERE u64 * i32 >= 2 ORDER BY (u64, i32); +SELECT * FROM minmax_idx2_r WHERE u64 * i32 >= 2 ORDER BY (u64, i32); -ALTER TABLE test.minmax_idx2_r DROP INDEX idx1, DROP INDEX idx2; +ALTER TABLE minmax_idx2_r DROP INDEX idx1, DROP INDEX idx2; -SHOW CREATE TABLE test.minmax_idx2; -SHOW CREATE TABLE test.minmax_idx2_r; +SHOW CREATE TABLE minmax_idx2; +SHOW CREATE TABLE minmax_idx2_r; -SELECT * FROM test.minmax_idx2 WHERE u64 * i32 >= 2 ORDER BY (u64, i32); -SELECT * FROM test.minmax_idx2_r WHERE u64 * i32 >= 2 ORDER BY (u64, i32); +SELECT * FROM minmax_idx2 WHERE u64 * i32 >= 2 ORDER BY (u64, i32); +SELECT * FROM minmax_idx2_r WHERE u64 * i32 >= 2 ORDER BY (u64, i32); -DROP TABLE test.minmax_idx; -DROP TABLE test.minmax_idx_r; -DROP TABLE test.minmax_idx2; -DROP TABLE test.minmax_idx2_r; +DROP TABLE minmax_idx; +DROP TABLE minmax_idx_r; +DROP TABLE minmax_idx2; +DROP TABLE minmax_idx2_r; diff --git a/tests/queries/0_stateless/00900_orc_load.sh b/tests/queries/0_stateless/00900_orc_load.sh index 5d2e45e2574..6e08b415397 100755 --- a/tests/queries/0_stateless/00900_orc_load.sh +++ b/tests/queries/0_stateless/00900_orc_load.sh @@ -10,3 +10,4 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (int Int32, smallint Int8, b cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" ${CLICKHOUSE_CLIENT} --query="select * from orc_load" +${CLICKHOUSE_CLIENT} --query="drop table orc_load" diff --git a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference index 328022414b8..29e7b23d3dd 100644 --- a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference +++ b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference @@ -20,7 +20,7 @@ 274972506.6 9175437371954010821 9175437371954010821 -CREATE TABLE test.compression_codec_multiple_more_types_replicated\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00910/compression_codec_multiple_more_types_replicated\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.compression_codec_multiple_more_types_replicated\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00910/compression_codec_multiple_more_types_replicated\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] 7.1000000000000 xxxxxxxxxxxx [127] ['Henry'] ! diff --git a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.sql b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.sql index e4c9f0ffc3b..52eb1d4e411 100644 --- a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.sql +++ b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.sql @@ -1,10 +1,10 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; -DROP TABLE IF EXISTS test.compression_codec_replicated1; -DROP TABLE IF EXISTS test.compression_codec_replicated2; +DROP TABLE IF EXISTS compression_codec_replicated1; +DROP TABLE IF EXISTS compression_codec_replicated2; -CREATE TABLE test.compression_codec_replicated1( +CREATE TABLE compression_codec_replicated1( id UInt64 CODEC(LZ4), data String CODEC(ZSTD), ddd Date CODEC(NONE), @@ -13,7 +13,7 @@ CREATE TABLE test.compression_codec_replicated1( othernum Int64 CODEC(Delta) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/compression_codec_replicated', '1') ORDER BY tuple(); -CREATE TABLE test.compression_codec_replicated2( +CREATE TABLE compression_codec_replicated2( id UInt64 CODEC(LZ4), data String CODEC(ZSTD), ddd Date CODEC(NONE), @@ -23,43 +23,43 @@ CREATE TABLE test.compression_codec_replicated2( ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/compression_codec_replicated', '2') ORDER BY tuple(); -INSERT INTO test.compression_codec_replicated1 VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); -INSERT INTO test.compression_codec_replicated1 VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); -INSERT INTO test.compression_codec_replicated1 VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); +INSERT INTO compression_codec_replicated1 VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); +INSERT INTO compression_codec_replicated1 VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); +INSERT INTO compression_codec_replicated1 VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); -SYSTEM SYNC REPLICA test.compression_codec_replicated2; +SYSTEM SYNC REPLICA compression_codec_replicated2; -SELECT * FROM test.compression_codec_replicated1 ORDER BY id; -SELECT * FROM test.compression_codec_replicated2 ORDER BY id; +SELECT * FROM compression_codec_replicated1 ORDER BY id; +SELECT * FROM compression_codec_replicated2 ORDER BY id; -OPTIMIZE TABLE test.compression_codec_replicated1 FINAL; +OPTIMIZE TABLE compression_codec_replicated1 FINAL; -INSERT INTO test.compression_codec_replicated1 VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); +INSERT INTO compression_codec_replicated1 VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); -SYSTEM SYNC REPLICA test.compression_codec_replicated2; +SYSTEM SYNC REPLICA compression_codec_replicated2; -DETACH TABLE test.compression_codec_replicated1; -ATTACH TABLE test.compression_codec_replicated1; +DETACH TABLE compression_codec_replicated1; +ATTACH TABLE compression_codec_replicated1; -SELECT count(*) FROM test.compression_codec_replicated1 WHERE id = 2 GROUP BY id; -SELECT count(*) FROM test.compression_codec_replicated2 WHERE id = 2 GROUP BY id; +SELECT count(*) FROM compression_codec_replicated1 WHERE id = 2 GROUP BY id; +SELECT count(*) FROM compression_codec_replicated2 WHERE id = 2 GROUP BY id; -DROP TABLE IF EXISTS test.compression_codec_replicated1; -DROP TABLE IF EXISTS test.compression_codec_replicated2; +DROP TABLE IF EXISTS compression_codec_replicated1; +DROP TABLE IF EXISTS compression_codec_replicated2; -DROP TABLE IF EXISTS test.compression_codec_multiple_replicated1; -DROP TABLE IF EXISTS test.compression_codec_multiple_replicated2; +DROP TABLE IF EXISTS compression_codec_multiple_replicated1; +DROP TABLE IF EXISTS compression_codec_multiple_replicated2; SET network_compression_method = 'lz4hc'; -CREATE TABLE test.compression_codec_multiple_replicated1 ( +CREATE TABLE compression_codec_multiple_replicated1 ( id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/compression_codec_multiple', '1') ORDER BY tuple(); -CREATE TABLE test.compression_codec_multiple_replicated2 ( +CREATE TABLE compression_codec_multiple_replicated2 ( id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), @@ -67,72 +67,73 @@ CREATE TABLE test.compression_codec_multiple_replicated2 ( ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/compression_codec_multiple', '2') ORDER BY tuple(); -INSERT INTO test.compression_codec_multiple_replicated2 VALUES (1, 'world', toDate('2018-10-05'), 1.1), (2, 'hello', toDate('2018-10-01'), 2.2), (3, 'buy', toDate('2018-10-11'), 3.3); +INSERT INTO compression_codec_multiple_replicated2 VALUES (1, 'world', toDate('2018-10-05'), 1.1), (2, 'hello', toDate('2018-10-01'), 2.2), (3, 'buy', toDate('2018-10-11'), 3.3); -SYSTEM SYNC REPLICA test.compression_codec_multiple_replicated1; +SYSTEM SYNC REPLICA compression_codec_multiple_replicated1; -SELECT * FROM test.compression_codec_multiple_replicated2 ORDER BY id; -SELECT * FROM test.compression_codec_multiple_replicated1 ORDER BY id; +SELECT * FROM compression_codec_multiple_replicated2 ORDER BY id; +SELECT * FROM compression_codec_multiple_replicated1 ORDER BY id; -INSERT INTO test.compression_codec_multiple_replicated1 select modulo(number, 100), toString(number), toDate('2018-12-01'), 5.5 * number FROM system.numbers limit 10000; +INSERT INTO compression_codec_multiple_replicated1 select modulo(number, 100), toString(number), toDate('2018-12-01'), 5.5 * number FROM system.numbers limit 10000; -SYSTEM SYNC REPLICA test.compression_codec_multiple_replicated2; +SYSTEM SYNC REPLICA compression_codec_multiple_replicated2; -SELECT count(*) FROM test.compression_codec_multiple_replicated1; -SELECT count(*) FROM test.compression_codec_multiple_replicated2; +SELECT count(*) FROM compression_codec_multiple_replicated1; +SELECT count(*) FROM compression_codec_multiple_replicated2; -SELECT count(distinct data) FROM test.compression_codec_multiple_replicated1; -SELECT count(distinct data) FROM test.compression_codec_multiple_replicated2; +SELECT count(distinct data) FROM compression_codec_multiple_replicated1; +SELECT count(distinct data) FROM compression_codec_multiple_replicated2; -SELECT floor(sum(somenum), 1) FROM test.compression_codec_multiple_replicated1; -SELECT floor(sum(somenum), 1) FROM test.compression_codec_multiple_replicated2; +SELECT floor(sum(somenum), 1) FROM compression_codec_multiple_replicated1; +SELECT floor(sum(somenum), 1) FROM compression_codec_multiple_replicated2; -TRUNCATE TABLE test.compression_codec_multiple_replicated1; -SYSTEM SYNC REPLICA test.compression_codec_multiple_replicated2; +TRUNCATE TABLE compression_codec_multiple_replicated1; +SYSTEM SYNC REPLICA compression_codec_multiple_replicated2; -INSERT INTO test.compression_codec_multiple_replicated1 select modulo(number, 100), toString(number), toDate('2018-12-01'), 5.5 * number FROM system.numbers limit 10000; +INSERT INTO compression_codec_multiple_replicated1 select modulo(number, 100), toString(number), toDate('2018-12-01'), 5.5 * number FROM system.numbers limit 10000; -SYSTEM SYNC REPLICA test.compression_codec_multiple_replicated2; +SYSTEM SYNC REPLICA compression_codec_multiple_replicated2; -SELECT sum(cityHash64(*)) FROM test.compression_codec_multiple_replicated2; -SELECT sum(cityHash64(*)) FROM test.compression_codec_multiple_replicated1; +SELECT sum(cityHash64(*)) FROM compression_codec_multiple_replicated2; +SELECT sum(cityHash64(*)) FROM compression_codec_multiple_replicated1; -DROP TABLE IF EXISTS test.compression_codec_multiple_replicated1; -DROP TABLE IF EXISTS test.compression_codec_multiple_replicated2; +DROP TABLE IF EXISTS compression_codec_multiple_replicated1; +DROP TABLE IF EXISTS compression_codec_multiple_replicated2; -DROP TABLE IF EXISTS test.compression_codec_multiple_more_types_replicated; +DROP TABLE IF EXISTS compression_codec_multiple_more_types_replicated; -CREATE TABLE test.compression_codec_multiple_more_types_replicated ( +CREATE TABLE compression_codec_multiple_more_types_replicated ( id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC), data FixedString(12) CODEC(ZSTD, ZSTD, Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC), ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8)) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/compression_codec_multiple_more_types_replicated', '1') ORDER BY tuple(); -SHOW CREATE TABLE test.compression_codec_multiple_more_types_replicated; +SHOW CREATE TABLE compression_codec_multiple_more_types_replicated; -INSERT INTO test.compression_codec_multiple_more_types_replicated VALUES(1.5555555555555, 'hello world!', [77], ['John']); -INSERT INTO test.compression_codec_multiple_more_types_replicated VALUES(7.1, 'xxxxxxxxxxxx', [127], ['Henry']); +INSERT INTO compression_codec_multiple_more_types_replicated VALUES(1.5555555555555, 'hello world!', [77], ['John']); +INSERT INTO compression_codec_multiple_more_types_replicated VALUES(7.1, 'xxxxxxxxxxxx', [127], ['Henry']); -SELECT * FROM test.compression_codec_multiple_more_types_replicated order by id; +SELECT * FROM compression_codec_multiple_more_types_replicated order by id; -DROP TABLE IF EXISTS test.compression_codec_multiple_with_key_replicated; +DROP TABLE IF EXISTS compression_codec_multiple_with_key_replicated; SET network_compression_method = 'zstd'; SET network_zstd_compression_level = 5; -CREATE TABLE test.compression_codec_multiple_with_key_replicated ( +CREATE TABLE compression_codec_multiple_with_key_replicated ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), Delta, Delta), id UInt64 CODEC(LZ4, ZSTD, Delta, NONE, LZ4HC, Delta), data String CODEC(ZSTD(2), Delta(1), LZ4HC, NONE, LZ4, LZ4) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/compression_codec_multiple_with_key_replicated', '1') PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; -INSERT INTO test.compression_codec_multiple_with_key_replicated VALUES(toDate('2018-10-12'), 100000, 'hello'), (toDate('2018-10-12'), 100002, 'world'), (toDate('2018-10-12'), 1111, '!'); +INSERT INTO compression_codec_multiple_with_key_replicated VALUES(toDate('2018-10-12'), 100000, 'hello'), (toDate('2018-10-12'), 100002, 'world'), (toDate('2018-10-12'), 1111, '!'); -SELECT data FROM test.compression_codec_multiple_with_key_replicated WHERE id BETWEEN 3 AND 1112; +SELECT data FROM compression_codec_multiple_with_key_replicated WHERE id BETWEEN 3 AND 1112; -INSERT INTO test.compression_codec_multiple_with_key_replicated SELECT toDate('2018-10-12'), number, toString(number) FROM system.numbers LIMIT 1000; +INSERT INTO compression_codec_multiple_with_key_replicated SELECT toDate('2018-10-12'), number, toString(number) FROM system.numbers LIMIT 1000; -SELECT COUNT(DISTINCT data) FROM test.compression_codec_multiple_with_key_replicated WHERE id < 222; +SELECT COUNT(DISTINCT data) FROM compression_codec_multiple_with_key_replicated WHERE id < 222; -DROP TABLE IF EXISTS test.compression_codec_multiple_with_key_replicated; +DROP TABLE IF EXISTS compression_codec_multiple_with_key_replicated; +DROP TABLE compression_codec_multiple_more_types_replicated; diff --git a/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference b/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference index 16818d9c7dc..e84c3beabee 100644 --- a/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference @@ -1,3 +1,3 @@ 200 400 -CREATE TABLE test.ttl_repl2\n(\n `d` Date,\n `x` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00933/ttl_repl\', \'2\')\nPARTITION BY toDayOfMonth(d)\nORDER BY x\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_repl2\n(\n `d` Date,\n `x` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_00933/ttl_repl\', \'2\')\nPARTITION BY toDayOfMonth(d)\nORDER BY x\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.sql b/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.sql index ac45f2903d5..dbbbe887e9f 100644 --- a/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.sql +++ b/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.sql @@ -1,25 +1,28 @@ -DROP TABLE IF EXISTS test.ttl_repl1; -DROP TABLE IF EXISTS test.ttl_repl2; +DROP TABLE IF EXISTS ttl_repl1; +DROP TABLE IF EXISTS ttl_repl2; -CREATE TABLE test.ttl_repl1(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00933/ttl_repl', '1') +CREATE TABLE ttl_repl1(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00933/ttl_repl', '1') PARTITION BY toDayOfMonth(d) ORDER BY x TTL d + INTERVAL 1 DAY; -CREATE TABLE test.ttl_repl2(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00933/ttl_repl', '2') +CREATE TABLE ttl_repl2(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00933/ttl_repl', '2') PARTITION BY toDayOfMonth(d) ORDER BY x TTL d + INTERVAL 1 DAY; -INSERT INTO TABLE test.ttl_repl1 VALUES (toDate('2000-10-10 00:00:00'), 100); -INSERT INTO TABLE test.ttl_repl1 VALUES (toDate('2100-10-10 00:00:00'), 200); +INSERT INTO TABLE ttl_repl1 VALUES (toDate('2000-10-10 00:00:00'), 100); +INSERT INTO TABLE ttl_repl1 VALUES (toDate('2100-10-10 00:00:00'), 200); -ALTER TABLE test.ttl_repl1 MODIFY TTL d + INTERVAL 1 DAY; -SYSTEM SYNC REPLICA test.ttl_repl2; +ALTER TABLE ttl_repl1 MODIFY TTL d + INTERVAL 1 DAY; +SYSTEM SYNC REPLICA ttl_repl2; -INSERT INTO TABLE test.ttl_repl1 VALUES (toDate('2000-10-10 00:00:00'), 300); -INSERT INTO TABLE test.ttl_repl1 VALUES (toDate('2100-10-10 00:00:00'), 400); +INSERT INTO TABLE ttl_repl1 VALUES (toDate('2000-10-10 00:00:00'), 300); +INSERT INTO TABLE ttl_repl1 VALUES (toDate('2100-10-10 00:00:00'), 400); -SYSTEM SYNC REPLICA test.ttl_repl2; +SYSTEM SYNC REPLICA ttl_repl2; SELECT sleep(1) format Null; -- wait for probable merges after inserts -OPTIMIZE TABLE test.ttl_repl2 FINAL; -SELECT x FROM test.ttl_repl2 ORDER BY x; +OPTIMIZE TABLE ttl_repl2 FINAL; +SELECT x FROM ttl_repl2 ORDER BY x; -SHOW CREATE TABLE test.ttl_repl2; +SHOW CREATE TABLE ttl_repl2; + +DROP TABLE ttl_repl1; +DROP TABLE ttl_repl2; diff --git a/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql b/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql index 33f16eb241c..455fab694cd 100644 --- a/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql +++ b/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql @@ -7,3 +7,6 @@ CREATE TABLE underlying_00967 (key Nullable(UInt64)) Engine=TinyLog(); INSERT INTO dist_00967 SELECT toUInt64(number) FROM system.numbers LIMIT 1; SELECT * FROM dist_00967; + +DROP TABLE dist_00967; +DROP TABLE underlying_00967; diff --git a/tests/queries/0_stateless/01056_create_table_as.sql b/tests/queries/0_stateless/01056_create_table_as.sql index 089daefe38d..32a39646170 100644 --- a/tests/queries/0_stateless/01056_create_table_as.sql +++ b/tests/queries/0_stateless/01056_create_table_as.sql @@ -45,3 +45,7 @@ CREATE TABLE t3 AS dict; -- { serverError 80; } DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t3; +DROP DICTIONARY dict; +DROP TABLE test_01056_dict_data.dict_data; + +DROP DATABASE test_01056_dict_data; diff --git a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql index 404922f36bb..831cb25d967 100644 --- a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql +++ b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql @@ -30,3 +30,5 @@ SELECT count() FROM minmax_compact WHERE i64 = 2; -- { serverError 158 } set max_rows_to_read = 10; SELECT count() FROM minmax_compact WHERE i64 = 2; + +DROP TABLE minmax_compact; diff --git a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql index 8a1601e3faa..dbc2d7c9f5d 100644 --- a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql +++ b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql @@ -46,3 +46,6 @@ SELECT '-'; SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +DROP TABLE nr; +DROP TABLE t; diff --git a/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.sql b/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.sql index a063e417e3a..781e9d6f8f9 100644 --- a/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.sql +++ b/tests/queries/0_stateless/01227_distributed_global_in_issue_2610.sql @@ -4,3 +4,5 @@ create table data_01227 (key Int) Engine=MergeTree() order by key; insert into data_01227 select * from numbers(10); select * from remote('127.1', currentDatabase(), data_01227) prewhere key global in (select key from data_01227 prewhere key = 2); select * from cluster('test_cluster_two_shards', currentDatabase(), data_01227) prewhere key global in (select key from data_01227 prewhere key = 2); + +drop table data_01227; diff --git a/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.sql b/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.sql index 31f09b35bf3..80f7625900a 100644 --- a/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.sql +++ b/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.sql @@ -23,3 +23,8 @@ select x, count() from ma_dist group by x order by x; drop table if exists shard_0.shard_01231_distributed_aggregation_memory_efficient; drop table if exists shard_1.shard_01231_distributed_aggregation_memory_efficient; + +drop table ma_dist; + +drop database shard_0; +drop database shard_1; diff --git a/tests/queries/0_stateless/01231_operator_null_in.sql b/tests/queries/0_stateless/01231_operator_null_in.sql index ddebaf23900..27ab0bbd838 100644 --- a/tests/queries/0_stateless/01231_operator_null_in.sql +++ b/tests/queries/0_stateless/01231_operator_null_in.sql @@ -141,3 +141,4 @@ SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [(2, NULL), (NULL, NULL)] FR SELECT arraySort(x -> (x.1, x.2), groupArray(t)) == [] FROM null_in_tuple WHERE t not in ((1, '1'), (1, NULL), (2, NULL), (NULL, '3'), (NULL, NULL)); DROP TABLE IF EXISTS null_in_tuple; +DROP TABLE test_set2; diff --git a/tests/queries/0_stateless/01232_extremes.sql b/tests/queries/0_stateless/01232_extremes.sql index e46a6602766..b0670f731c6 100644 --- a/tests/queries/0_stateless/01232_extremes.sql +++ b/tests/queries/0_stateless/01232_extremes.sql @@ -52,3 +52,5 @@ drop table if exists shard_1.num2_01232; drop table if exists distr; drop table if exists distr2; +drop database shard_0; +drop database shard_1; diff --git a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql index 618bdc1c5d2..522fc5dafbe 100644 --- a/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql +++ b/tests/queries/0_stateless/01236_distributed_over_live_view_over_distributed.sql @@ -7,7 +7,7 @@ DROP TABLE IF EXISTS visits_layer; CREATE TABLE visits (StartDate Date) ENGINE MergeTree ORDER BY(StartDate); CREATE TABLE visits_layer (StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'visits', rand()); -CREATE LIVE VIEW lv AS SELECT * FROM visits_layer ORDER BY StartDate; +CREATE LIVE VIEW lv AS SELECT * FROM visits_layer ORDER BY StartDate; CREATE TABLE visits_layer_lv (StartDate Date) ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'lv', rand()); @@ -20,3 +20,4 @@ DROP TABLE visits; DROP TABLE visits_layer; DROP TABLE lv; +DROP TABLE visits_layer_lv; diff --git a/tests/queries/0_stateless/01236_graphite_mt.sql b/tests/queries/0_stateless/01236_graphite_mt.sql index cee9b8c9fde..f3f1905b901 100644 --- a/tests/queries/0_stateless/01236_graphite_mt.sql +++ b/tests/queries/0_stateless/01236_graphite_mt.sql @@ -1,17 +1,17 @@ drop table if exists test_graphite; create table test_graphite (key UInt32, Path String, Time DateTime, Value Float64, Version UInt32, col UInt64) engine = GraphiteMergeTree('graphite_rollup') order by key settings index_granularity=10; -insert into test_graphite -select 1, 'sum_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +insert into test_graphite +select 1, 'sum_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all select 2, 'sum_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all select 1, 'sum_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all select 2, 'sum_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all -select 1, 'max_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 1, 'max_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all select 2, 'max_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all select 1, 'max_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all select 2, 'max_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300); -insert into test_graphite +insert into test_graphite select 1, 'sum_1', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all select 2, 'sum_1', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all select 1, 'sum_2', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all @@ -24,3 +24,5 @@ select 2, 'max_2', toDateTime(today() - 3) - number * 60 - 30, number, 1, number optimize table test_graphite; select key, Path, Value, Version, col from test_graphite order by key, Path, Time desc; + +drop table test_graphite; diff --git a/tests/queries/0_stateless/01246_buffer_flush.sql b/tests/queries/0_stateless/01246_buffer_flush.sql index efe0adf703a..47891a7f00e 100644 --- a/tests/queries/0_stateless/01246_buffer_flush.sql +++ b/tests/queries/0_stateless/01246_buffer_flush.sql @@ -42,3 +42,5 @@ select 'drop'; insert into buffer_01256 select * from system.numbers limit 10; drop table if exists buffer_01256; select count() from data_01256; + +drop table data_01256; diff --git a/tests/queries/0_stateless/01246_finalize_aggregation_race.sql b/tests/queries/0_stateless/01246_finalize_aggregation_race.sql index 336fe6bcfea..c4946bd9e83 100644 --- a/tests/queries/0_stateless/01246_finalize_aggregation_race.sql +++ b/tests/queries/0_stateless/01246_finalize_aggregation_race.sql @@ -2,7 +2,7 @@ drop table if exists test_quantile; create table test_quantile (x AggregateFunction(quantileTiming(0.2), UInt64)) engine = Memory; insert into test_quantile select medianTimingState(.2)(number) from (select * from numbers(1000) order by number desc); select y from ( -select finalizeAggregation(x) as y from test_quantile union all +select finalizeAggregation(x) as y from test_quantile union all select finalizeAggregation(x) as y from test_quantile union all select finalizeAggregation(x) as y from test_quantile union all select finalizeAggregation(x) as y from test_quantile union all @@ -21,3 +21,4 @@ select finalizeAggregation(x) as y from test_quantile union all select finalizeAggregation(x) as y from test_quantile union all select finalizeAggregation(x) as y from test_quantile) order by y; +drop table test_quantile; diff --git a/tests/queries/0_stateless/01247_optimize_distributed_group_by_sharding_key_dist_on_dist.sql b/tests/queries/0_stateless/01247_optimize_distributed_group_by_sharding_key_dist_on_dist.sql index 7654ba71cc9..1000e956583 100644 --- a/tests/queries/0_stateless/01247_optimize_distributed_group_by_sharding_key_dist_on_dist.sql +++ b/tests/queries/0_stateless/01247_optimize_distributed_group_by_sharding_key_dist_on_dist.sql @@ -39,3 +39,4 @@ select count(), * from dist_01247 group by number order by number limit 1; drop table dist_01247; drop table dist_layer_01247; +drop table data_01247; diff --git a/tests/queries/0_stateless/01268_data_numeric_parameters.sql b/tests/queries/0_stateless/01268_data_numeric_parameters.sql index eceba51e7f5..3450fef9a6f 100644 --- a/tests/queries/0_stateless/01268_data_numeric_parameters.sql +++ b/tests/queries/0_stateless/01268_data_numeric_parameters.sql @@ -40,3 +40,7 @@ CREATE TABLE strings ( INSERT INTO strings VALUES ('test', 'string'); SELECT toTypeName(a), toTypeName(b) FROM strings; + +DROP TABLE floats; +DROP TABLE ints; +DROP TABLE strings; diff --git a/tests/queries/0_stateless/01268_mv_scalars.sql b/tests/queries/0_stateless/01268_mv_scalars.sql index fc082aaf88c..f6621502ca0 100644 --- a/tests/queries/0_stateless/01268_mv_scalars.sql +++ b/tests/queries/0_stateless/01268_mv_scalars.sql @@ -31,3 +31,5 @@ drop table dest_table_mv; drop table left_table; drop table right_table; drop table dest_table; +drop table src_table; +drop view dst_mv; diff --git a/tests/queries/0_stateless/01272_totals_and_filter_bug.sql b/tests/queries/0_stateless/01272_totals_and_filter_bug.sql index de751eb73bd..11992ab052a 100644 --- a/tests/queries/0_stateless/01272_totals_and_filter_bug.sql +++ b/tests/queries/0_stateless/01272_totals_and_filter_bug.sql @@ -9,22 +9,22 @@ INSERT INTO foo VALUES ('2020-01-01', 'test1', 10), ('2020-01-01', 'test2', 20); INSERT INTO bar VALUES ('2020-01-01', 'test2', 30), ('2020-01-01', 'test3', 40); SELECT - dimension_1, - sum_metric_1, + dimension_1, + sum_metric_1, sum_metric_2 -FROM +FROM ( SELECT - dimension_1, + dimension_1, sum(metric_1) AS sum_metric_1 FROM foo GROUP BY dimension_1 WITH TOTALS ) AS subquery_1 -ALL FULL OUTER JOIN +ALL FULL OUTER JOIN ( SELECT - dimension_1, + dimension_1, sum(metric_2) AS sum_metric_2 FROM bar GROUP BY dimension_1 @@ -32,3 +32,6 @@ ALL FULL OUTER JOIN ) AS subquery_2 USING (dimension_1) WHERE sum_metric_2 < 20 ORDER BY dimension_1 ASC; + +DROP TABLE foo; +DROP TABLE bar; diff --git a/tests/queries/0_stateless/01273_lc_fixed_string_field.sql b/tests/queries/0_stateless/01273_lc_fixed_string_field.sql index 18ef71fd6dc..11f93e918c6 100644 --- a/tests/queries/0_stateless/01273_lc_fixed_string_field.sql +++ b/tests/queries/0_stateless/01273_lc_fixed_string_field.sql @@ -13,3 +13,5 @@ INSERT INTO t (d, s, c) VALUES ('2020-01-01', 'ABC', 2); OPTIMIZE TABLE t; SELECT * FROM t; + +DROP TABLE t; diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql b/tests/queries/0_stateless/01275_parallel_mv.sql index b67fbf02f8d..5d4dffae50a 100644 --- a/tests/queries/0_stateless/01275_parallel_mv.sql +++ b/tests/queries/0_stateless/01275_parallel_mv.sql @@ -16,3 +16,8 @@ select count() from testX; select count() from testXA; select count() from testXB; select count() from testXC; + +drop table testX; +drop view testXA; +drop view testXB; +drop view testXC; diff --git a/tests/queries/0_stateless/01279_dist_group_by.sql b/tests/queries/0_stateless/01279_dist_group_by.sql index 207b203f5d0..331efd4b687 100644 --- a/tests/queries/0_stateless/01279_dist_group_by.sql +++ b/tests/queries/0_stateless/01279_dist_group_by.sql @@ -7,3 +7,5 @@ set max_rows_to_group_by=10; set group_by_overflow_mode='any'; set group_by_two_level_threshold=100; select * from data_01279 group by key format Null; + +drop table data_01279; diff --git a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql index 8de0f40229c..a7e51047eba 100644 --- a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql +++ b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql @@ -19,3 +19,5 @@ FROM system.query_log WHERE type = 'QueryFinish' AND query LIKE '%data_01283 LIMIT 1%' GROUP BY thread_ids FORMAT Null; + +DROP TABLE data_01283; diff --git a/tests/queries/0_stateless/001283_strict_resize_bug.reference b/tests/queries/0_stateless/01283_strict_resize_bug.reference similarity index 100% rename from tests/queries/0_stateless/001283_strict_resize_bug.reference rename to tests/queries/0_stateless/01283_strict_resize_bug.reference diff --git a/tests/queries/0_stateless/001283_strict_resize_bug.sql b/tests/queries/0_stateless/01283_strict_resize_bug.sql similarity index 90% rename from tests/queries/0_stateless/001283_strict_resize_bug.sql rename to tests/queries/0_stateless/01283_strict_resize_bug.sql index f462f50c61f..ee1aef05cd2 100644 --- a/tests/queries/0_stateless/001283_strict_resize_bug.sql +++ b/tests/queries/0_stateless/01283_strict_resize_bug.sql @@ -4,4 +4,4 @@ insert into num_10m select * from numbers(10000000); select * from (select sum(number) from num_10m union all select sum(number) from num_10m) limit 1 settings max_block_size = 1024; -drop table if exists num_1m; +drop table if exists num_10m; diff --git a/tests/queries/0_stateless/01285_data_skip_index_over_aggregation.sql b/tests/queries/0_stateless/01285_data_skip_index_over_aggregation.sql index 110c5b65cab..575b4dfa4a5 100644 --- a/tests/queries/0_stateless/01285_data_skip_index_over_aggregation.sql +++ b/tests/queries/0_stateless/01285_data_skip_index_over_aggregation.sql @@ -32,3 +32,5 @@ OPTIMIZE TABLE data_01285 FINAL; SELECT * FROM data_01285; -- and this passes even without fix. SELECT * FROM data_01285 WHERE assumeNotNull(value) = 3; + +DROP TABLE data_01285; diff --git a/tests/queries/0_stateless/01291_geo_types.sql b/tests/queries/0_stateless/01291_geo_types.sql index 0d923f08ccd..6b686ddf520 100644 --- a/tests/queries/0_stateless/01291_geo_types.sql +++ b/tests/queries/0_stateless/01291_geo_types.sql @@ -7,3 +7,5 @@ CREATE TABLE geo (a Point, b Ring, c Polygon, d MultiPolygon) ENGINE=Memory(); INSERT INTO geo VALUES((0, 0), [(0, 0), (10, 0), (10, 10), (0, 10)], [[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]], [[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]); SELECT * from geo; + +DROP TABLE geo; diff --git a/tests/queries/0_stateless/01293_system_distribution_queue.sql b/tests/queries/0_stateless/01293_system_distribution_queue.sql index 4c9c690af09..8f84bbac41f 100644 --- a/tests/queries/0_stateless/01293_system_distribution_queue.sql +++ b/tests/queries/0_stateless/01293_system_distribution_queue.sql @@ -24,3 +24,6 @@ select is_blocked, error_count, data_files, data_compressed_bytes from system.di select 'UNBLOCK'; system start distributed sends dist_01293; select is_blocked, error_count, data_files, data_compressed_bytes from system.distribution_queue; + +drop table null_01293; +drop table dist_01293; diff --git a/tests/queries/0_stateless/01296_pipeline_stuck.sql b/tests/queries/0_stateless/01296_pipeline_stuck.sql index eeb67362634..2a23e6a9bf8 100644 --- a/tests/queries/0_stateless/01296_pipeline_stuck.sql +++ b/tests/queries/0_stateless/01296_pipeline_stuck.sql @@ -16,3 +16,5 @@ select 'INSERT SELECT max_insert_threads max_threads'; set max_insert_threads=2; insert into data_01295 select * from data_01295 final settings max_threads=2; -- no stuck for now select * from data_01295; + +drop table data_01295; diff --git a/tests/queries/0_stateless/01318_decrypt.sql b/tests/queries/0_stateless/01318_decrypt.sql index 796c42db1ab..fecca593272 100644 --- a/tests/queries/0_stateless/01318_decrypt.sql +++ b/tests/queries/0_stateless/01318_decrypt.sql @@ -150,3 +150,5 @@ WITH SELECT hex(decrypt('aes-256-gcm', concat(ciphertext, tag), key, iv, aad)) as plaintext_actual, plaintext_actual = hex(plaintext); + +DROP TABLE encryption_test; diff --git a/tests/queries/0_stateless/01318_encrypt.sql b/tests/queries/0_stateless/01318_encrypt.sql index 9766988764a..a9a60c89836 100644 --- a/tests/queries/0_stateless/01318_encrypt.sql +++ b/tests/queries/0_stateless/01318_encrypt.sql @@ -154,3 +154,5 @@ WITH SELECT hex(encrypt('aes-256-gcm', plaintext, key, iv, aad)) as ciphertext_actual, ciphertext_actual = concat(hex(ciphertext), hex(tag)); + +DROP TABLE encryption_test; diff --git a/tests/queries/0_stateless/01319_mv_constants_bug.sql b/tests/queries/0_stateless/01319_mv_constants_bug.sql index 191183ab286..4abb9d61b6c 100644 --- a/tests/queries/0_stateless/01319_mv_constants_bug.sql +++ b/tests/queries/0_stateless/01319_mv_constants_bug.sql @@ -22,3 +22,4 @@ DROP TABLE IF EXISTS distributed_table_1; DROP TABLE IF EXISTS distributed_table_2; DROP TABLE IF EXISTS local_table_1; DROP TABLE IF EXISTS local_table_2; +DROP TABLE local_table_merged; diff --git a/tests/queries/0_stateless/01319_optimize_skip_unused_shards_nesting.sql b/tests/queries/0_stateless/01319_optimize_skip_unused_shards_nesting.sql index b8a48c27e5f..4f7d6b81002 100644 --- a/tests/queries/0_stateless/01319_optimize_skip_unused_shards_nesting.sql +++ b/tests/queries/0_stateless/01319_optimize_skip_unused_shards_nesting.sql @@ -20,3 +20,7 @@ select * from dist_01319 where key = 1; set force_optimize_skip_unused_shards_nesting=2; set optimize_skip_unused_shards_nesting=1; select * from dist_01319 where key = 1; + +drop table data_01319; +drop table dist_01319; +drop table dist_layer_01319; diff --git a/tests/queries/0_stateless/01320_optimize_skip_unused_shards_no_non_deterministic.sql b/tests/queries/0_stateless/01320_optimize_skip_unused_shards_no_non_deterministic.sql index ca58f7be94c..f827bb733b4 100644 --- a/tests/queries/0_stateless/01320_optimize_skip_unused_shards_no_non_deterministic.sql +++ b/tests/queries/0_stateless/01320_optimize_skip_unused_shards_no_non_deterministic.sql @@ -8,3 +8,6 @@ create table dist_01320 as data_01320 Engine=Distributed(test_cluster_two_shards set optimize_skip_unused_shards=1; set force_optimize_skip_unused_shards=1; select * from dist_01320 where key = 0; -- { serverError 507 } + +drop table data_01320; +drop table dist_01320; diff --git a/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper.sql b/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper.sql index 0719282734a..d185973f564 100644 --- a/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper.sql +++ b/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper.sql @@ -47,3 +47,4 @@ ALTER TABLE test DROP COLUMN x; -- { serverError 47 } ALTER TABLE test DROP COLUMN y; -- { serverError 47 } DROP TABLE test; +DROP TABLE test2; diff --git a/tests/queries/0_stateless/01392_column_resolve.sql b/tests/queries/0_stateless/01392_column_resolve.sql index 93d84359e1f..728ccfd0728 100644 --- a/tests/queries/0_stateless/01392_column_resolve.sql +++ b/tests/queries/0_stateless/01392_column_resolve.sql @@ -1,15 +1,14 @@ -DROP TABLE IF EXISTS tableConversion; -DROP TABLE IF EXISTS tableClick; -DROP TABLE IF EXISTS leftjoin; +DROP DATABASE IF EXISTS test_01392; +CREATE DATABASE test_01392; -CREATE TABLE default.tableConversion (conversionId String, value Nullable(Double)) ENGINE = Log(); -CREATE TABLE default.tableClick (clickId String, conversionId String, value Nullable(Double)) ENGINE = Log(); -CREATE TABLE default.leftjoin (id String) ENGINE = Log(); +CREATE TABLE test_01392.tableConversion (conversionId String, value Nullable(Double)) ENGINE = Log(); +CREATE TABLE test_01392.tableClick (clickId String, conversionId String, value Nullable(Double)) ENGINE = Log(); +CREATE TABLE test_01392.leftjoin (id String) ENGINE = Log(); -INSERT INTO default.tableConversion(conversionId, value) VALUES ('Conversion 1', 1); -INSERT INTO default.tableClick(clickId, conversionId, value) VALUES ('Click 1', 'Conversion 1', 14); -INSERT INTO default.tableClick(clickId, conversionId, value) VALUES ('Click 2', 'Conversion 1', 15); -INSERT INTO default.tableClick(clickId, conversionId, value) VALUES ('Click 3', 'Conversion 1', 16); +INSERT INTO test_01392.tableConversion(conversionId, value) VALUES ('Conversion 1', 1); +INSERT INTO test_01392.tableClick(clickId, conversionId, value) VALUES ('Click 1', 'Conversion 1', 14); +INSERT INTO test_01392.tableClick(clickId, conversionId, value) VALUES ('Click 2', 'Conversion 1', 15); +INSERT INTO test_01392.tableClick(clickId, conversionId, value) VALUES ('Click 3', 'Conversion 1', 16); SELECT conversion.conversionId AS myConversionId, @@ -17,17 +16,19 @@ SELECT click.myValue AS myValue FROM ( SELECT conversionId, value as myValue - FROM default.tableConversion + FROM test_01392.tableConversion ) AS conversion INNER JOIN ( SELECT clickId, conversionId, value as myValue - FROM default.tableClick + FROM test_01392.tableClick ) AS click ON click.conversionId = conversion.conversionId LEFT JOIN ( - SELECT * FROM default.leftjoin + SELECT * FROM test_01392.leftjoin ) AS dummy ON (dummy.id = conversion.conversionId) ORDER BY myValue; -DROP TABLE IF EXISTS tableConversion; -DROP TABLE IF EXISTS tableClick; -DROP TABLE IF EXISTS leftjoin; +DROP TABLE test_01392.tableConversion; +DROP TABLE test_01392.tableClick; +DROP TABLE test_01392.leftjoin; + +DROP DATABASE test_01392; diff --git a/tests/queries/0_stateless/01400_join_get_with_multi_keys.sql b/tests/queries/0_stateless/01400_join_get_with_multi_keys.sql index 8a19865359b..8f83e1c15dd 100644 --- a/tests/queries/0_stateless/01400_join_get_with_multi_keys.sql +++ b/tests/queries/0_stateless/01400_join_get_with_multi_keys.sql @@ -13,3 +13,4 @@ INSERT INTO test_lc VALUES ('ab', '1', 0.1), ('ab', '2', 0.2), ('cd', '3', 0.3); SELECT joinGet(test_lc, 'c', 'ab', '1'); DROP TABLE test_joinGet; +DROP TABLE test_lc; diff --git a/tests/queries/0_stateless/01409_topK_merge.sql b/tests/queries/0_stateless/01409_topK_merge.sql index 5ac7c350093..d7659351540 100644 --- a/tests/queries/0_stateless/01409_topK_merge.sql +++ b/tests/queries/0_stateless/01409_topK_merge.sql @@ -11,3 +11,5 @@ select length(topKWeighted(20)(number, 1)) from remote('127.{1,1}', currentDatab select 'AggregateFunctionTopKGenericData'; select length(topK(20)((number, ''))) from remote('127.{1,1}', currentDatabase(), data_01409); select length(topKWeighted(20)((number, ''), 1)) from remote('127.{1,1}', currentDatabase(), data_01409); + +drop table data_01409; diff --git a/tests/queries/0_stateless/01413_truncate_without_table_keyword.sql b/tests/queries/0_stateless/01413_truncate_without_table_keyword.sql index 266c5b84be3..c6819f77e80 100644 --- a/tests/queries/0_stateless/01413_truncate_without_table_keyword.sql +++ b/tests/queries/0_stateless/01413_truncate_without_table_keyword.sql @@ -10,3 +10,4 @@ TRUNCATE truncate_test; SELECT * FROM truncate_test ORDER BY uint8; +DROP TABLE truncate_test; diff --git a/tests/queries/0_stateless/01419_skip_index_compact_parts.sql b/tests/queries/0_stateless/01419_skip_index_compact_parts.sql index 8b03f28df44..580fcf7f41e 100644 --- a/tests/queries/0_stateless/01419_skip_index_compact_parts.sql +++ b/tests/queries/0_stateless/01419_skip_index_compact_parts.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS index_compact; - + CREATE TABLE index_compact(a UInt32, b UInt32, index i1 b type minmax granularity 1) ENGINE = MergeTree ORDER BY a SETTINGS min_rows_for_wide_part = 1000, index_granularity = 128, merge_max_block_size = 100; @@ -10,3 +10,5 @@ INSERT INTO index_compact SELECT number, toString(number) FROM numbers(30); OPTIMIZE TABLE index_compact FINAL; SELECT count() FROM index_compact WHERE b < 10; + +DROP TABLE index_compact; diff --git a/tests/queries/0_stateless/01456_low_cardinality_sorting_bugfix.sql b/tests/queries/0_stateless/01456_low_cardinality_sorting_bugfix.sql index 507a798e7b6..06412d4b062 100644 --- a/tests/queries/0_stateless/01456_low_cardinality_sorting_bugfix.sql +++ b/tests/queries/0_stateless/01456_low_cardinality_sorting_bugfix.sql @@ -39,3 +39,5 @@ SELECT cast(color,'String') color, timestamp FROM order_test1 GROUP BY color, timestamp ORDER BY color ASC, timestamp DESC; + +DROP TABLE order_test1; diff --git a/tests/queries/0_stateless/01457_create_as_table_function_structure.sql b/tests/queries/0_stateless/01457_create_as_table_function_structure.sql index 1bb0d9c54ca..1c9c1e1ef44 100644 --- a/tests/queries/0_stateless/01457_create_as_table_function_structure.sql +++ b/tests/queries/0_stateless/01457_create_as_table_function_structure.sql @@ -32,3 +32,4 @@ SELECT (*,).1 AS c, toTypeName(c) FROM test_01457.tf_merge; DROP DATABASE test_01457; +DROP TABLE tmp; diff --git a/tests/queries/0_stateless/01460_DistributedFilesToInsert.sql b/tests/queries/0_stateless/01460_DistributedFilesToInsert.sql index 8ce10cf133f..34c0d55d573 100644 --- a/tests/queries/0_stateless/01460_DistributedFilesToInsert.sql +++ b/tests/queries/0_stateless/01460_DistributedFilesToInsert.sql @@ -40,3 +40,5 @@ select sleep(1) format Null; -- distributed_directory_monitor_sleep_time_ms select value from system.metrics where metric = 'DistributedFilesToInsert'; drop table dist_01460; select value from system.metrics where metric = 'DistributedFilesToInsert'; + +drop table data_01460; diff --git a/tests/queries/0_stateless/01463_test_alter_live_view_refresh.sql b/tests/queries/0_stateless/01463_test_alter_live_view_refresh.sql index ab316a377fd..04276087ece 100644 --- a/tests/queries/0_stateless/01463_test_alter_live_view_refresh.sql +++ b/tests/queries/0_stateless/01463_test_alter_live_view_refresh.sql @@ -8,3 +8,6 @@ CREATE LIVE VIEW live1 AS SELECT * FROM test0; select 'ALTER LIVE VIEW live1 REFRESH'; ALTER LIVE VIEW live1 REFRESH; -- success + +DROP TABLE test0; +DROP VIEW live1; diff --git a/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql b/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql index f6f7471711a..39e04804bfb 100644 --- a/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql +++ b/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql @@ -34,3 +34,4 @@ DROP TABLE IF EXISTS t; DROP DATABASE shard_0; DROP DATABASE shard_1; +DROP DATABASE main_01487; diff --git a/tests/queries/0_stateless/01511_prewhere_with_virtuals.sql b/tests/queries/0_stateless/01511_prewhere_with_virtuals.sql index 8b272c8cff1..43f003fc80f 100644 --- a/tests/queries/0_stateless/01511_prewhere_with_virtuals.sql +++ b/tests/queries/0_stateless/01511_prewhere_with_virtuals.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS test_not_found_column_nothing; -CREATE TABLE test_not_found_column_nothing +CREATE TABLE test_not_found_column_nothing ( col001 UInt8, col002 UInt8 @@ -10,3 +10,5 @@ INSERT INTO test_not_found_column_nothing(col001) SELECT number FROM numbers(11) SELECT _part, count() FROM test_not_found_column_nothing PREWHERE col001 % 3 != 0 GROUP BY _part ORDER BY _part; SELECT _part FROM test_not_found_column_nothing PREWHERE col001 = 0; + +DROP TABLE test_not_found_column_nothing; diff --git a/tests/queries/0_stateless/01513_optimize_aggregation_in_order_memory.sql b/tests/queries/0_stateless/01513_optimize_aggregation_in_order_memory.sql index 38920262fba..6aa38a914f7 100644 --- a/tests/queries/0_stateless/01513_optimize_aggregation_in_order_memory.sql +++ b/tests/queries/0_stateless/01513_optimize_aggregation_in_order_memory.sql @@ -14,3 +14,5 @@ select key, groupArray(repeat('a', 200)), count() from data_01513 group by key f select key, groupArray(repeat('a', 200)), count() from data_01513 group by key format Null settings optimize_aggregation_in_order=1; -- for WITH TOTALS previous groups should be kept. select key, groupArray(repeat('a', 200)), count() from data_01513 group by key with totals format Null settings optimize_aggregation_in_order=1; -- { serverError 241; } + +drop table data_01513; diff --git a/tests/queries/0_stateless/01514_empty_buffer_different_types.sql b/tests/queries/0_stateless/01514_empty_buffer_different_types.sql index 3afadbcd33f..a1debbf7e13 100644 --- a/tests/queries/0_stateless/01514_empty_buffer_different_types.sql +++ b/tests/queries/0_stateless/01514_empty_buffer_different_types.sql @@ -10,3 +10,4 @@ select s from buffer_table1 where x = 1; select s from buffer_table1 where x = 2; DROP TABLE IF EXISTS merge_tree_table1; +DROP TABLE buffer_table1; diff --git a/tests/queries/0_stateless/01515_force_data_skipping_indices.sql b/tests/queries/0_stateless/01515_force_data_skipping_indices.sql index 5e45018707d..53d3e5c736f 100644 --- a/tests/queries/0_stateless/01515_force_data_skipping_indices.sql +++ b/tests/queries/0_stateless/01515_force_data_skipping_indices.sql @@ -29,3 +29,5 @@ SELECT * FROM data_01515 WHERE d1 = 0 SETTINGS force_data_skipping_indices=' d1 SELECT * FROM data_01515 WHERE d1_null = 0 SETTINGS force_data_skipping_indices='d1_null_idx'; -- { serverError 277 } SELECT * FROM data_01515 WHERE assumeNotNull(d1_null) = 0 SETTINGS force_data_skipping_indices='d1_null_idx'; + +DROP TABLE data_01515; diff --git a/tests/queries/0_stateless/01515_mv_and_array_join_optimisation_bag.sql b/tests/queries/0_stateless/01515_mv_and_array_join_optimisation_bag.sql index 9cca5e85da7..ad762ea65fb 100644 --- a/tests/queries/0_stateless/01515_mv_and_array_join_optimisation_bag.sql +++ b/tests/queries/0_stateless/01515_mv_and_array_join_optimisation_bag.sql @@ -24,14 +24,14 @@ SELECT sumIfState(Sign, _uniq = 1) AS Visits, sumState(Sign) AS GoalReaches FROM visits -ARRAY JOIN +ARRAY JOIN GoalsID AS GoalID, arrayEnumerateUniq(GoalsID) AS _uniq -GROUP BY +GROUP BY CounterID, StartDate, GoalID -ORDER BY +ORDER BY CounterID ASC, StartDate ASC, GoalID ASC; @@ -46,3 +46,7 @@ CREATE TABLE goal ) ENGINE = AggregatingMergeTree PARTITION BY toStartOfMonth(StartDate) ORDER BY (CounterID, StartDate, GoalID) SETTINGS index_granularity = 256; INSERT INTO visits (`CounterID`,`StartDate`,`StartTime`,`Sign`,`GoalsID`) VALUES (1, toDate('2000-01-01'), toDateTime(toDate('2000-01-01')), 1, [1]); + +DROP TABLE goal; +DROP TABLE goal_view; +DROP TABLE visits; diff --git a/tests/queries/0_stateless/01517_drop_mv_with_inner_table.sql b/tests/queries/0_stateless/01517_drop_mv_with_inner_table.sql index bbf7f0d04be..b97480c2911 100644 --- a/tests/queries/0_stateless/01517_drop_mv_with_inner_table.sql +++ b/tests/queries/0_stateless/01517_drop_mv_with_inner_table.sql @@ -38,3 +38,10 @@ create materialized view db_01517_ordinary.mv engine=Null as select * from db_01 -- drops it and hangs with Atomic engine, due to recursive DROP drop table db_01517_ordinary.mv sync; show tables from db_01517_ordinary; + +drop table db_01517_atomic_sync.source; +drop table db_01517_ordinary.source; + +drop database db_01517_atomic; +drop database db_01517_atomic_sync; +drop database db_01517_ordinary; diff --git a/tests/queries/0_stateless/01526_initial_query_id.sh b/tests/queries/0_stateless/01526_initial_query_id.sh index c5459625023..e28f9ee1e40 100755 --- a/tests/queries/0_stateless/01526_initial_query_id.sh +++ b/tests/queries/0_stateless/01526_initial_query_id.sh @@ -10,7 +10,7 @@ ${CLICKHOUSE_CLIENT} -q "select 1 format Null" "--query_id=$query_id" ${CLICKHOUSE_CURL} \ --header "X-ClickHouse-Query-Id: $query_id" \ - "http://localhost:8123/" \ + $CLICKHOUSE_URL \ --get \ --data-urlencode "query=select 1 format Null" diff --git a/tests/queries/0_stateless/01528_allow_nondeterministic_optimize_skip_unused_shards.sql b/tests/queries/0_stateless/01528_allow_nondeterministic_optimize_skip_unused_shards.sql index b0bfb2aae3f..51970db73cb 100644 --- a/tests/queries/0_stateless/01528_allow_nondeterministic_optimize_skip_unused_shards.sql +++ b/tests/queries/0_stateless/01528_allow_nondeterministic_optimize_skip_unused_shards.sql @@ -5,3 +5,5 @@ set optimize_skip_unused_shards=1; set force_optimize_skip_unused_shards=1; select * from dist_01528 where dummy = 2; -- { serverError 507; } select * from dist_01528 where dummy = 2 settings allow_nondeterministic_optimize_skip_unused_shards=1; + +drop table dist_01528; diff --git a/tests/queries/0_stateless/01530_drop_database_atomic_sync.sql b/tests/queries/0_stateless/01530_drop_database_atomic_sync.sql index 010b8931448..d5fb25a9241 100644 --- a/tests/queries/0_stateless/01530_drop_database_atomic_sync.sql +++ b/tests/queries/0_stateless/01530_drop_database_atomic_sync.sql @@ -28,7 +28,7 @@ drop database db_01530_atomic; create database db_01530_atomic Engine=Atomic; create table db_01530_atomic.data (key Int) Engine=ReplicatedMergeTree('/clickhouse/tables/db_01530_atomic/data', 'test') order by key; -- { serverError 253; } --- TODO: SYSTEM FORCE DROP and uncomment the line below to cleanup the data after test --- (otherwise the test is not retriable...) --- --- drop database db_01530_atomic sync; + +set database_atomic_wait_for_drop_and_detach_synchronously=1; + +drop database db_01530_atomic sync; diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql index 831a7282861..a71c9f9a714 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql @@ -14,3 +14,5 @@ SET merge_tree_min_rows_for_concurrent_read=10000; SET optimize_aggregation_in_order=1; SET read_in_order_two_level_merge_threshold=1; EXPLAIN PIPELINE SELECT key FROM data_01551 GROUP BY key, key/2; + +DROP TABLE data_01551; diff --git a/tests/queries/conftest.py b/tests/queries/conftest.py index 85b7250af0c..2f19ae7c479 100644 --- a/tests/queries/conftest.py +++ b/tests/queries/conftest.py @@ -36,6 +36,11 @@ def sql_query(request): return os.path.join(QUERIES_PATH, os.path.splitext(request.param)[0]) +@pytest.fixture(scope='module', params=[f for f in os.listdir(QUERIES_PATH) if f.endswith('.sh')]) +def shell_query(request): + return os.path.join(QUERIES_PATH, os.path.splitext(request.param)[0]) + + @pytest.fixture def standalone_server(bin_prefix, tmp_path): server = ServerThread(bin_prefix, str(tmp_path)) diff --git a/tests/queries/query_test.py b/tests/queries/query_test.py index d0fc5759667..adaf9bc4e5e 100644 --- a/tests/queries/query_test.py +++ b/tests/queries/query_test.py @@ -8,18 +8,16 @@ import subprocess import sys -def run_client(bin_prefix, port, query, reference, replace_map={}): - client = subprocess.Popen([bin_prefix + '-client', '--port', str(port), '-m', '-n', '--testmode'], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - result, error = client.communicate(query.encode('utf-8')) - assert client.returncode is not None, "Client should exit after processing all queries" - +def check_result(result, error, return_code, reference, replace_map): for old, new in replace_map.items(): result = result.replace(old.encode('utf-8'), new.encode('utf-8')) - if client.returncode != 0: - print(error.decode('utf-8'), file=sys.stderr) - pytest.fail('Client died unexpectedly with code {code}'.format(code=client.returncode), pytrace=False) + if return_code != 0: + try: + print(error.decode('utf-8'), file=sys.stderr) + except UnicodeDecodeError: + print(error.decode('latin1'), file=sys.stderr) # encoding with 1 symbol per 1 byte, covering all values + pytest.fail('Client died unexpectedly with code {code}'.format(code=return_code), pytrace=False) elif result != reference: pytest.fail("Query output doesn't match reference:{eol}{diff}".format( eol=os.linesep, @@ -29,13 +27,38 @@ def run_client(bin_prefix, port, query, reference, replace_map={}): pytrace=False) +def run_client(bin_prefix, port, query, reference, replace_map={}): + # We can't use `text=True` since some tests may return binary data + client = subprocess.Popen([bin_prefix + '-client', '--port', str(port), '-m', '-n', '--testmode'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result, error = client.communicate(query.encode('utf-8')) + assert client.returncode is not None, "Client should exit after processing all queries" + + check_result(result, error, client.returncode, reference, replace_map) + + +def run_shell(bin_prefix, tmp_dir, tcp_port, http_port, inter_port, database, path, reference, replace_map={}): + env = { + 'CLICKHOUSE_BINARY': bin_prefix, + 'CLICKHOUSE_DATABASE': database, + 'CLICKHOUSE_PORT_TCP': str(tcp_port), + 'CLICKHOUSE_PORT_HTTP': str(http_port), + 'CLICKHOUSE_PORT_INTERSERVER': str(inter_port), + 'CLICKHOUSE_TMP': tmp_dir, + } + shell = subprocess.Popen([path], env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result, error = shell.communicate() + assert shell.returncode is not None, "Script should exit after executing all commands" + + check_result(result, error, shell.returncode, reference, replace_map) + + def random_str(length=10): alphabet = string.ascii_lowercase + string.digits return ''.join(random.choice(alphabet) for _ in range(length)) -@pytest.mark.timeout(timeout=30, method='signal') -def test_query(bin_prefix, sql_query, standalone_server): +def test_sql_query(bin_prefix, sql_query, standalone_server): tcp_port = standalone_server.tcp_port query_path = sql_query + ".sql" @@ -56,7 +79,39 @@ def test_query(bin_prefix, sql_query, standalone_server): query = "SELECT 'SHOW ORPHANED TABLES'; SELECT name FROM system.tables WHERE database != 'system' ORDER BY (database, name);" run_client(bin_prefix, tcp_port, query, b'SHOW ORPHANED TABLES\n') - run_client(bin_prefix, tcp_port, 'DROP DATABASE {random};'.format(random=random_name), b'') + query = 'DROP DATABASE {random};'.format(random=random_name) + run_client(bin_prefix, tcp_port, query, b'') + + query = "SELECT 'SHOW ORPHANED DATABASES'; SHOW DATABASES;" + run_client(bin_prefix, tcp_port, query, b'SHOW ORPHANED DATABASES\n_temporary_and_external_tables\ndefault\nsystem\n') + + +def test_shell_query(bin_prefix, shell_query, standalone_server): + tcp_port = standalone_server.tcp_port + http_port = standalone_server.http_port + inter_port = standalone_server.inter_port + tmp_path = standalone_server.tmp_dir + + shell_path = shell_query + ".sh" + reference_path = shell_query + ".reference" + + if not os.path.exists(reference_path): + pytest.skip('No .reference file found') + + with open(reference_path, 'rb') as file: + reference = file.read() + + random_name = 'test_{random}'.format(random=random_str()) + query = 'CREATE DATABASE {random};'.format(random=random_name) + run_client(bin_prefix, tcp_port, query, b'') + + run_shell(bin_prefix, tmp_path, tcp_port, http_port, inter_port, random_name, shell_path, reference, {random_name: 'default'}) + + query = "SELECT 'SHOW ORPHANED TABLES'; SELECT name FROM system.tables WHERE database != 'system' ORDER BY (database, name);" + run_client(bin_prefix, tcp_port, query, b'SHOW ORPHANED TABLES\n') + + query = 'DROP DATABASE {random};'.format(random=random_name) + run_client(bin_prefix, tcp_port, query, b'') query = "SELECT 'SHOW ORPHANED DATABASES'; SHOW DATABASES;" run_client(bin_prefix, tcp_port, query, b'SHOW ORPHANED DATABASES\n_temporary_and_external_tables\ndefault\nsystem\n') diff --git a/tests/queries/server.py b/tests/queries/server.py index c4f8968e08a..b3c3ece0b4c 100644 --- a/tests/queries/server.py +++ b/tests/queries/server.py @@ -23,6 +23,7 @@ class ServerThread(threading.Thread): self.etc_dir = os.path.join(tmp_dir, 'etc') self.server_config = os.path.join(self.etc_dir, 'server-config.xml') self.users_config = os.path.join(self.etc_dir, 'users.xml') + self.dicts_config = os.path.join(self.etc_dir, 'dictionaries.xml') os.makedirs(self.log_dir) os.makedirs(self.etc_dir) @@ -51,6 +52,9 @@ class ServerThread(threading.Thread): with open(self.users_config, 'w') as f: f.write(ServerThread.DEFAULT_USERS_CONFIG) + with open(self.dicts_config, 'w') as f: + f.write(ServerThread.DEFAULT_DICTIONARIES_CONFIG.format(tcp_port=self.tcp_port)) + def run(self): retries = ServerThread.DEFAULT_RETRIES @@ -122,10 +126,45 @@ ServerThread.DEFAULT_SERVER_CONFIG = \ {tmp_dir}/tmp/ {tmp_dir}/data/access/ users.xml + dictionaries.xml 5368709120 + 3 Europe/Moscow + + Hello, world! + s1 + r1 + + + + Version + + sum + sum + + 0 + 600 + + + 172800 + 6000 + + + + max + + 0 + 600 + + + 172800 + 6000 + + + + @@ -180,7 +219,78 @@ ServerThread.DEFAULT_SERVER_CONFIG = \ + + + + + shard_0 + localhost + {tcp_port} + + + + + shard_1 + localhost + {tcp_port} + + + + + + + true + + 127.0.0.1 + {tcp_port} + + + + true + + 127.0.0.2 + {tcp_port} + + + + + + + + memory + + + + + + testkeeper + + + + system + part_log
+ + + + system + query_log
+ + + + system + query_thread_log
+ + + + system + text_log
+ + + + system + trace_log
+ """ @@ -242,3 +352,735 @@ ServerThread.DEFAULT_USERS_CONFIG = \ """ + + +ServerThread.DEFAULT_DICTIONARIES_CONFIG = \ +"""\ + + + flat_ints + + + localhost + {tcp_port} + default + + system + ints
+ + + 0 + + + + + + key + + + i8 + Int8 + 0 + + + i16 + Int16 + 0 + + + i32 + Int32 + 0 + + + i64 + Int64 + 0 + + + u8 + UInt8 + 0 + + + u16 + UInt16 + 0 + + + u32 + UInt32 + 0 + + + u64 + UInt64 + 0 + + + + + + hashed_ints + + + localhost + {tcp_port} + default + + system + ints
+ + + 0 + + + + + + key + + + i8 + Int8 + 0 + + + i16 + Int16 + 0 + + + i32 + Int32 + 0 + + + i64 + Int64 + 0 + + + u8 + UInt8 + 0 + + + u16 + UInt16 + 0 + + + u32 + UInt32 + 0 + + + u64 + UInt64 + 0 + + + + + + hashed_sparse_ints + + + localhost + {tcp_port} + default + + system + ints
+ + + 0 + + + + + + key + + + i8 + Int8 + 0 + + + i16 + Int16 + 0 + + + i32 + Int32 + 0 + + + i64 + Int64 + 0 + + + u8 + UInt8 + 0 + + + u16 + UInt16 + 0 + + + u32 + UInt32 + 0 + + + u64 + UInt64 + 0 + + + + + + cache_ints + + + localhost + {tcp_port} + default + + system + ints
+ + + 0 + + 1000 + + + + key + + + i8 + Int8 + 0 + + + i16 + Int16 + 0 + + + i32 + Int32 + 0 + + + i64 + Int64 + 0 + + + u8 + UInt8 + 0 + + + u16 + UInt16 + 0 + + + u32 + UInt32 + 0 + + + u64 + UInt64 + 0 + + + + + + complex_hashed_ints + + + localhost + {tcp_port} + default + + system + ints
+ + + 0 + + + + + + + key + UInt64 + + + + i8 + Int8 + 0 + + + i16 + Int16 + 0 + + + i32 + Int32 + 0 + + + i64 + Int64 + 0 + + + u8 + UInt8 + 0 + + + u16 + UInt16 + 0 + + + u32 + UInt32 + 0 + + + u64 + UInt64 + 0 + + + + + + complex_cache_ints + + + localhost + {tcp_port} + default + + system + ints
+ + + 0 + + 1000 + + + + + key + UInt64 + + + + i8 + Int8 + 0 + + + i16 + Int16 + 0 + + + i32 + Int32 + 0 + + + i64 + Int64 + 0 + + + u8 + UInt8 + 0 + + + u16 + UInt16 + 0 + + + u32 + UInt32 + 0 + + + u64 + UInt64 + 0 + + + + + + flat_strings + + + localhost + {tcp_port} + default + + system + strings
+ + + 0 + + + + + + key + + + str + String + + + + + + + hashed_strings + + + localhost + {tcp_port} + default + + system + strings
+ + + 0 + + + + + + key + + + str + String + + + + + + + cache_strings + + + localhost + {tcp_port} + default + + system + strings
+ + + 0 + + 1000 + + + + key + + + str + String + + + + + + + complex_hashed_strings + + + localhost + {tcp_port} + default + + system + strings
+ + + 0 + + + + + + + key + UInt64 + + + + str + String + + + + + + + complex_cache_strings + + + localhost + {tcp_port} + default + + system + strings
+ + + 0 + + 1000 + + + + + key + UInt64 + + + + str + String + + + + + + + flat_decimals + + + localhost + {tcp_port} + default + + system + decimals
+ + + 0 + + + + + + key + + + d32 + Decimal32(4) + 0 + + + d64 + Decimal64(6) + 0 + + + d128 + Decimal128(1) + 0 + + + + + + hashed_decimals + + + localhost + {tcp_port} + default + + system + decimals
+ + + 0 + + + + + + key + + + d32 + Decimal32(4) + 0 + + + d64 + Decimal64(6) + 0 + + + d128 + Decimal128(1) + 0 + + + + + + cache_decimals + + + localhost + {tcp_port} + default + + system + decimals
+ + + 0 + + 1000 + + + + key + + + d32 + Decimal32(4) + 0 + + + d64 + Decimal64(6) + 0 + + + d128 + Decimal128(1) + 0 + + + + + + complex_hashed_decimals + + + localhost + {tcp_port} + default + + system + decimals
+ + + 0 + + + + + + + key + UInt64 + + + + d32 + Decimal32(4) + 0 + + + d64 + Decimal64(6) + 0 + + + d128 + Decimal128(1) + 0 + + + + + + complex_cache_decimals + + + localhost + {tcp_port} + default + + system + decimals
+ + + 0 + + 1000 + + + + + key + UInt64 + + + + d32 + Decimal32(4) + 0 + + + d64 + Decimal64(6) + 0 + + + d128 + Decimal128(1) + 0 + + + + +""" diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 0b5b0940cd7..9cfec2c2a28 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Don't check for ODR violation, since we may test shared build with ASAN +export ASAN_OPTIONS=detect_odr_violation=0 + export CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:="test"} export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:="warning"} [ -v CLICKHOUSE_CONFIG_CLIENT ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " From f1d721fa74ac0b4ae54dd231c5c6be69812106a1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 11 Nov 2020 19:58:31 +0300 Subject: [PATCH 111/114] Marked some perf test queries as short --- tests/performance/push_down_limit.xml | 2 +- tests/performance/quantile_merge.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 6ae63b54ec6..0611216410d 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -4,5 +4,5 @@ select number from (select number from numbers(1500000000) order by -number) limit 10 select number from (select number from numbers_mt(1500000000) order by -number) limit 10 - select number from numbers_view limit 100 + select number from numbers_view limit 100 diff --git a/tests/performance/quantile_merge.xml b/tests/performance/quantile_merge.xml index 0ddb688d8eb..ad0f0571682 100644 --- a/tests/performance/quantile_merge.xml +++ b/tests/performance/quantile_merge.xml @@ -1,3 +1,3 @@ - SELECT quantileMerge(arrayJoin(arrayMap(x -> state, range(500000)))) FROM (SELECT quantileState(rand()) AS state FROM numbers(10000)) + SELECT quantileMerge(arrayJoin(arrayMap(x -> state, range(500000)))) FROM (SELECT quantileState(rand()) AS state FROM numbers(10000)) From d7e2276a9e4a18fbd7e62341d3b6d39c21b68ec6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2020 20:39:16 +0300 Subject: [PATCH 112/114] Update version_date.tsv after release 20.11.2.1 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0e55de496b7..47f6aefe406 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v20.11.2.1-stable 2020-11-11 v20.10.3.30-stable 2020-10-29 v20.10.2.20-stable 2020-10-23 v20.9.4.76-stable 2020-10-29 From f6b7bd88468ff55810cb1ac50a0045de7e86cde8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 12 Nov 2020 01:51:33 +0300 Subject: [PATCH 113/114] Add changelog for 20.11 --- CHANGELOG.md | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 457346aff9a..e552fe06962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,122 @@ +### ClickHouse release v20.11.2.1, 2020-11-11 + +#### Backward Incompatible Change + +* If some `profile` was specified in `distributed_ddl` config section, then this profile could overwrite settings of `default` profile on server startup. It's fixed, now settings of distributed DDL queries should not affect global server settings. [#16635](https://github.com/ClickHouse/ClickHouse/pull/16635) ([tavplubix](https://github.com/tavplubix)). +* Restrict to use of non-comparable data types (like `AggregateFunction`) in keys (Sorting key, Primary key, Partition key, and so on). [#16601](https://github.com/ClickHouse/ClickHouse/pull/16601) ([alesapin](https://github.com/alesapin)). +* Remove `ANALYZE` and `AST` queries, and make the setting `enable_debug_queries` obsolete since now it is the part of full featured `EXPLAIN` query. [#16536](https://github.com/ClickHouse/ClickHouse/pull/16536) ([Ivan](https://github.com/abyss7)). +* Aggregate functions `boundingRatio`, `rankCorr`, `retention`, `timeSeriesGroupSum`, `timeSeriesGroupRateSum`, `windowFunnel` were erroneously made case-insensitive. Now their names are made case sensitive as designed. Only functions that are specified in SQL standard or made for compatibility with other DBMS or functions similar to those should be case-insensitive. [#16407](https://github.com/ClickHouse/ClickHouse/pull/16407) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Make `rankCorr` function return nan on insufficient data https://github.com/ClickHouse/ClickHouse/issues/16124. [#16135](https://github.com/ClickHouse/ClickHouse/pull/16135) ([hexiaoting](https://github.com/hexiaoting)). + +#### New Feature + +* Added support of LDAP as a user directory for locally non-existent users. [#12736](https://github.com/ClickHouse/ClickHouse/pull/12736) ([Denis Glazachev](https://github.com/traceon)). +* Add `system.replicated_fetches` table which shows currently running background fetches. [#16428](https://github.com/ClickHouse/ClickHouse/pull/16428) ([alesapin](https://github.com/alesapin)). +* Added setting `date_time_output_format`. [#15845](https://github.com/ClickHouse/ClickHouse/pull/15845) ([Maksim Kita](https://github.com/kitaisreal)). +* Added minimal web UI to ClickHouse. [#16158](https://github.com/ClickHouse/ClickHouse/pull/16158) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allows to read/write Single protobuf message at once (w/o length-delimiters). [#15199](https://github.com/ClickHouse/ClickHouse/pull/15199) ([filimonov](https://github.com/filimonov)). +* Added initial OpenTelemetry support. ClickHouse now accepts OpenTelemetry traceparent headers over Native and HTTP protocols, and passes them downstream in some cases. The trace spans for executed queries are saved into the `system.opentelemetry_span_log` table. [#14195](https://github.com/ClickHouse/ClickHouse/pull/14195) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow specify primary key in column list of `CREATE TABLE` query. This is needed for compatibility with other SQL dialects. [#15823](https://github.com/ClickHouse/ClickHouse/pull/15823) ([Maksim Kita](https://github.com/kitaisreal)). +* Implement `OFFSET offset_row_count {ROW | ROWS} FETCH {FIRST | NEXT} fetch_row_count {ROW | ROWS} {ONLY | WITH TIES}` in SELECT query with ORDER BY. This is the SQL-standard way to specify `LIMIT`. [#15855](https://github.com/ClickHouse/ClickHouse/pull/15855) ([hexiaoting](https://github.com/hexiaoting)). +* `errorCodeToName` function - return variable name of the error (useful for analyzing query_log and similar). `system.errors` table - shows how many times errors has been happened (respects `system_events_show_zero_values`). [#16438](https://github.com/ClickHouse/ClickHouse/pull/16438) ([Azat Khuzhin](https://github.com/azat)). +* Added function `untuple` which is a special function which can introduce new columns to the SELECT list by expanding a named tuple. [#16242](https://github.com/ClickHouse/ClickHouse/pull/16242) ([Nikolai Kochetov](https://github.com/KochetovNicolai), [Amos Bird](https://github.com/amosbird)). +* Now we can provide identifiers via query parameters. And these parameters can be used as table objects or columns. [#16594](https://github.com/ClickHouse/ClickHouse/pull/16594) ([Amos Bird](https://github.com/amosbird)). +* Added big integers (UInt256, Int128, Int256) and UUID data types support for MergeTree BloomFilter index. Big integers is an experimental feature. [#16642](https://github.com/ClickHouse/ClickHouse/pull/16642) ([Maksim Kita](https://github.com/kitaisreal)). +* Add `farmFingerprint64` function (non-cryptographic string hashing). [#16570](https://github.com/ClickHouse/ClickHouse/pull/16570) ([Jacob Hayes](https://github.com/JacobHayes)). +* Add `log_queries_min_query_duration_ms`, only queries slower then the value of this setting will go to `query_log`/`query_thread_log` (i.e. something like `slow_query_log` in mysql). [#16529](https://github.com/ClickHouse/ClickHouse/pull/16529) ([Azat Khuzhin](https://github.com/azat)). +* Ability to create a docker image on the top of `Alpine`. Uses precompiled binary and glibc components from ubuntu 20.04. [#16479](https://github.com/ClickHouse/ClickHouse/pull/16479) ([filimonov](https://github.com/filimonov)). +* Added `toUUIDOrNull`, `toUUIDOrZero` cast functions. [#16337](https://github.com/ClickHouse/ClickHouse/pull/16337) ([Maksim Kita](https://github.com/kitaisreal)). +* Add `max_concurrent_queries_for_all_users` setting, see [#6636](https://github.com/ClickHouse/ClickHouse/issues/6636) for use cases. [#16154](https://github.com/ClickHouse/ClickHouse/pull/16154) ([nvartolomei](https://github.com/nvartolomei)). +* Add a new option `print_query_id` to clickhouse-client. It helps generate arbitrary strings with the current query id generated by the client. Also print query id in clickhouse-client by default. [#15809](https://github.com/ClickHouse/ClickHouse/pull/15809) ([Amos Bird](https://github.com/amosbird)). +* Add `tid` and `logTrace` functions. This closes [#9434](https://github.com/ClickHouse/ClickHouse/issues/9434). [#15803](https://github.com/ClickHouse/ClickHouse/pull/15803) ([flynn](https://github.com/ucasFL)). +* Add function `formatReadableTimeDelta` that format time delta to human readable string ... [#15497](https://github.com/ClickHouse/ClickHouse/pull/15497) ([Filipe Caixeta](https://github.com/filipecaixeta)). +* Added `disable_merges` option for volumes in multi-disk configuration. [#13956](https://github.com/ClickHouse/ClickHouse/pull/13956) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### Experimental Feature + +* New functions `encrypt`, `aes_encrypt_mysql`, `decrypt`, `aes_decrypt_mysql`. These functions are working slowly, so we consider it as an experimental feature. [#11844](https://github.com/ClickHouse/ClickHouse/pull/11844) ([Vasily Nemkov](https://github.com/Enmk)). + +#### Bug Fix + +* Mask password in data_path in the `system.distribution_queue`. [#16727](https://github.com/ClickHouse/ClickHouse/pull/16727) ([Azat Khuzhin](https://github.com/azat)). +* Fix `IN` operator over several columns and tuples with enabled `transform_null_in` setting. Fixes [#15310](https://github.com/ClickHouse/ClickHouse/issues/15310). [#16722](https://github.com/ClickHouse/ClickHouse/pull/16722) ([Anton Popov](https://github.com/CurtizJ)). +* The setting `max_parallel_replicas` worked incorrectly if the queried table has no sampling. This fixes [#5733](https://github.com/ClickHouse/ClickHouse/issues/5733). [#16675](https://github.com/ClickHouse/ClickHouse/pull/16675) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix optimize_read_in_order/optimize_aggregation_in_order with max_threads > 0 and expression in ORDER BY. [#16637](https://github.com/ClickHouse/ClickHouse/pull/16637) ([Azat Khuzhin](https://github.com/azat)). +* Calculation of `DEFAULT` expressions was involving possible name collisions (that was very unlikely to encounter). This fixes [#9359](https://github.com/ClickHouse/ClickHouse/issues/9359). [#16612](https://github.com/ClickHouse/ClickHouse/pull/16612) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix `query_thread_log.query_duration_ms` unit. [#16563](https://github.com/ClickHouse/ClickHouse/pull/16563) ([Azat Khuzhin](https://github.com/azat)). +* Fix a bug when using MySQL Master -> MySQL Slave -> ClickHouse MaterializeMySQL Engine. `MaterializeMySQL` is an experimental feature. [#16504](https://github.com/ClickHouse/ClickHouse/pull/16504) ([TCeason](https://github.com/TCeason)). +* Specifically crafted argument of `round` function with `Decimal` was leading to integer division by zero. This fixes [#13338](https://github.com/ClickHouse/ClickHouse/issues/13338). [#16451](https://github.com/ClickHouse/ClickHouse/pull/16451) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix DROP TABLE for Distributed (racy with INSERT). [#16409](https://github.com/ClickHouse/ClickHouse/pull/16409) ([Azat Khuzhin](https://github.com/azat)). +* Fix processing of very large entries in replication queue. Very large entries may appear in ALTER queries if table structure is extremely large (near 1 MB). This fixes [#16307](https://github.com/ClickHouse/ClickHouse/issues/16307). [#16332](https://github.com/ClickHouse/ClickHouse/pull/16332) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed the inconsistent behaviour when a part of return data could be dropped because the set for its filtration wasn't created. [#16308](https://github.com/ClickHouse/ClickHouse/pull/16308) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix dictGet in sharding_key (and similar places, i.e. when the function context is stored permanently). [#16205](https://github.com/ClickHouse/ClickHouse/pull/16205) ([Azat Khuzhin](https://github.com/azat)). +* Fix the exception thrown in `clickhouse-local` when trying to execute `OPTIMIZE` command. Fixes [#16076](https://github.com/ClickHouse/ClickHouse/issues/16076). [#16192](https://github.com/ClickHouse/ClickHouse/pull/16192) ([filimonov](https://github.com/filimonov)). +* Fixes [#15780](https://github.com/ClickHouse/ClickHouse/issues/15780) regression, e.g. `indexOf([1, 2, 3], toLowCardinality(1))` now is prohibited but it should not be. [#16038](https://github.com/ClickHouse/ClickHouse/pull/16038) ([Mike](https://github.com/myrrc)). +* Fix bug with MySQL database. When MySQL server used as database engine is down some queries raise Exception, because they try to get tables from disabled server, while it's unnecessary. For example, query `SELECT ... FROM system.parts` should work only with MergeTree tables and don't touch MySQL database at all. [#16032](https://github.com/ClickHouse/ClickHouse/pull/16032) ([Kruglov Pavel](https://github.com/Avogar)). +* Now exception will be thrown when `ALTER MODIFY COLUMN ... DEFAULT ...` has incompatible default with column type. Fixes [#15854](https://github.com/ClickHouse/ClickHouse/issues/15854). [#15858](https://github.com/ClickHouse/ClickHouse/pull/15858) ([alesapin](https://github.com/alesapin)). +* Fixed IPv4CIDRToRange/IPv6CIDRToRange functions to accept const IP-column values. [#15856](https://github.com/ClickHouse/ClickHouse/pull/15856) ([vladimir-golovchenko](https://github.com/vladimir-golovchenko)). + +#### Improvement + +* Treat `INTERVAL '1 hour'` as equivalent to `INTERVAL 1 HOUR`, to be compatible with Postgres and similar. This fixes [#15637](https://github.com/ClickHouse/ClickHouse/issues/15637). [#15978](https://github.com/ClickHouse/ClickHouse/pull/15978) ([flynn](https://github.com/ucasFL)). +* Enable parsing enum values by their numeric ids for CSV, TSV and JSON input formats. [#15685](https://github.com/ClickHouse/ClickHouse/pull/15685) ([vivarum](https://github.com/vivarum)). +* Better read task scheduling for JBOD architecture and `MergeTree` storage. New setting `read_backoff_min_concurrency` which serves as the lower limit to the number of reading threads. [#16423](https://github.com/ClickHouse/ClickHouse/pull/16423) ([Amos Bird](https://github.com/amosbird)). +* Add missing support for `LowCardinality` in `Avro` format. [#16521](https://github.com/ClickHouse/ClickHouse/pull/16521) ([Mike](https://github.com/myrrc)). +* Workaround for use `S3` with nginx server as proxy. Nginx currenty does not accept urls with empty path like `http://domain.com?delete`, but vanilla aws-sdk-cpp produces this kind of urls. This commit uses patched aws-sdk-cpp version, which makes urls with "/" as path in this cases, like http://domain.com/?delete. [#16814](https://github.com/ClickHouse/ClickHouse/pull/16814) ([ianton-ru](https://github.com/ianton-ru)). +* Better diagnostics on parse errors in input data. Provide row number on `Cannot read all data` errors. [#16644](https://github.com/ClickHouse/ClickHouse/pull/16644) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Make the behaviour of `minMap` and `maxMap` more desireable. It will not skip zero values in the result. Fixes [#16087](https://github.com/ClickHouse/ClickHouse/issues/16087). [#16631](https://github.com/ClickHouse/ClickHouse/pull/16631) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Better update of ZooKeeper configuration in runtime. [#16630](https://github.com/ClickHouse/ClickHouse/pull/16630) ([sundyli](https://github.com/sundy-li)). +* Apply SETTINGS clause as early as possible. It allows to modify more settings in the query. This closes [#3178](https://github.com/ClickHouse/ClickHouse/issues/3178). [#16619](https://github.com/ClickHouse/ClickHouse/pull/16619) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Now `event_time_microseconds` field stores in Decimal64, not UInt64. [#16617](https://github.com/ClickHouse/ClickHouse/pull/16617) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Now paratmeterized functions can be used in `APPLY` column transformer. [#16589](https://github.com/ClickHouse/ClickHouse/pull/16589) ([Amos Bird](https://github.com/amosbird)). +* Improve scheduling of background task which removes data of dropped tables in `Atomic` databases. `Atomic` databases do not create broken symlink to table data directory if table actually has no data directory. [#16584](https://github.com/ClickHouse/ClickHouse/pull/16584) ([tavplubix](https://github.com/tavplubix)). +* Subqueries in `WITH` section (CTE) can reference previous subqueries in `WITH` section by their name. [#16575](https://github.com/ClickHouse/ClickHouse/pull/16575) ([Amos Bird](https://github.com/amosbird)). +* Add current_database into `system.query_thread_log`. [#16558](https://github.com/ClickHouse/ClickHouse/pull/16558) ([Azat Khuzhin](https://github.com/azat)). +* Allow to fetch parts that are already committed or outdated in the current instance into the detached directory. It's useful when migrating tables from another cluster and having N to 1 shards mapping. It's also consistent with the current fetchPartition implementation. [#16538](https://github.com/ClickHouse/ClickHouse/pull/16538) ([Amos Bird](https://github.com/amosbird)). +* Multiple improvements for `RabbitMQ`: Fixed bug for [#16263](https://github.com/ClickHouse/ClickHouse/issues/16263). Also minimized event loop lifetime. Added more efficient queues setup. [#16426](https://github.com/ClickHouse/ClickHouse/pull/16426) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix debug assertion in `quantileDeterministic` function. In previous version it may also transfer up to two times more data over the network. Although no bug existed. This fixes [#15683](https://github.com/ClickHouse/ClickHouse/issues/15683). [#16410](https://github.com/ClickHouse/ClickHouse/pull/16410) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add `TablesToDropQueueSize` metric. It's equal to number of dropped tables, that are waiting for background data removal. [#16364](https://github.com/ClickHouse/ClickHouse/pull/16364) ([tavplubix](https://github.com/tavplubix)). +* Better diagnostics when client has dropped connection. In previous versions, `Attempt to read after EOF` and `Broken pipe` exceptions were logged in server. In new version, it's information message `Client has dropped the connection, cancel the query.`. [#16329](https://github.com/ClickHouse/ClickHouse/pull/16329) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add total_rows/total_bytes (from system.tables) support for Set/Join table engines. [#16306](https://github.com/ClickHouse/ClickHouse/pull/16306) ([Azat Khuzhin](https://github.com/azat)). +* Now it's possible to specify `PRIMARY KEY` without `ORDER BY` for MergeTree table engines family. Closes [#15591](https://github.com/ClickHouse/ClickHouse/issues/15591). [#16284](https://github.com/ClickHouse/ClickHouse/pull/16284) ([alesapin](https://github.com/alesapin)). +* If there is no tmp folder in the system (chroot, misconfigutation etc) `clickhouse-local` will create temporary subfolder in the current directory. [#16280](https://github.com/ClickHouse/ClickHouse/pull/16280) ([filimonov](https://github.com/filimonov)). +* Add support for nested data types (like named tuple) as sub-types. Fixes [#15587](https://github.com/ClickHouse/ClickHouse/issues/15587). [#16262](https://github.com/ClickHouse/ClickHouse/pull/16262) ([Ivan](https://github.com/abyss7)). +* Support for `database_atomic_wait_for_drop_and_detach_synchronously`/`NO DELAY`/`SYNC` for `DROP DATABASE`. [#16127](https://github.com/ClickHouse/ClickHouse/pull/16127) ([Azat Khuzhin](https://github.com/azat)). +* Add `allow_nondeterministic_optimize_skip_unused_shards` (to allow non deterministic like `rand()` or `dictGet()` in sharding key). [#16105](https://github.com/ClickHouse/ClickHouse/pull/16105) ([Azat Khuzhin](https://github.com/azat)). +* Fix `memory_profiler_step`/`max_untracked_memory` for queries via HTTP (test included). Fix the issue that adjusting this value globally in xml config does not help either, since those settings are not applied anyway, only default (4MB) value is [used](https://github.com/ClickHouse/ClickHouse/blob/17731245336d8c84f75e4c0894c5797ed7732190/src/Common/ThreadStatus.h#L104). Fix `query_id` for the most root ThreadStatus of the http query (by initializing QueryScope after reading query_id). [#16101](https://github.com/ClickHouse/ClickHouse/pull/16101) ([Azat Khuzhin](https://github.com/azat)). +* Now it's allowed to execute `ALTER ... ON CLUSTER` queries regardless of the `` setting in cluster config. [#16075](https://github.com/ClickHouse/ClickHouse/pull/16075) ([alesapin](https://github.com/alesapin)). +* Fix rare issue when `clickhouse-client` may abort on exit due to loading of suggestions. This fixes [#16035](https://github.com/ClickHouse/ClickHouse/issues/16035). [#16047](https://github.com/ClickHouse/ClickHouse/pull/16047) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add support of `cache` layout for `Redis` dictionaries with complex key. [#15985](https://github.com/ClickHouse/ClickHouse/pull/15985) ([Anton Popov](https://github.com/CurtizJ)). +* Fix query hang (endless loop) in case of misconfiguration (`connections_with_failover_max_tries` set to 0). [#15876](https://github.com/ClickHouse/ClickHouse/pull/15876) ([Azat Khuzhin](https://github.com/azat)). +* Change level of some log messages from information to debug, so information messages will not appear for every query. This closes [#5293](https://github.com/ClickHouse/ClickHouse/issues/5293). [#15816](https://github.com/ClickHouse/ClickHouse/pull/15816) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Remove `MemoryTrackingInBackground*` metrics to avoid potentially misleading results. This fixes [#15684](https://github.com/ClickHouse/ClickHouse/issues/15684). [#15813](https://github.com/ClickHouse/ClickHouse/pull/15813) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add reconnects to `zookeeper-dump-tree` tool. [#15711](https://github.com/ClickHouse/ClickHouse/pull/15711) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow explicitly specify columns list in `CREATE TABLE table AS table_function(...)` query. Fixes [#9249](https://github.com/ClickHouse/ClickHouse/issues/9249) Fixes [#14214](https://github.com/ClickHouse/ClickHouse/issues/14214). [#14295](https://github.com/ClickHouse/ClickHouse/pull/14295) ([tavplubix](https://github.com/tavplubix)). + +#### Performance Improvement + +* Do not merge parts across partitions in SELECT FINAL. [#15938](https://github.com/ClickHouse/ClickHouse/pull/15938) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve performance of `-OrNull` and `-OrDefault` aggregate functions. [#16661](https://github.com/ClickHouse/ClickHouse/pull/16661) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improve performance of `quantileMerge`. In previous versions it was obnoxiously slow. This closes [#1463](https://github.com/ClickHouse/ClickHouse/issues/1463). [#16643](https://github.com/ClickHouse/ClickHouse/pull/16643) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improve performance of logical functions a little. [#16347](https://github.com/ClickHouse/ClickHouse/pull/16347) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improved performance of merges assignment in MergeTree table engines. Shouldn't be visible for the user. [#16191](https://github.com/ClickHouse/ClickHouse/pull/16191) ([alesapin](https://github.com/alesapin)). +* Speedup hashed/sparse_hashed dictionary loading by preallocating the hash table. [#15454](https://github.com/ClickHouse/ClickHouse/pull/15454) ([Azat Khuzhin](https://github.com/azat)). +* Now trivial count optimization becomes slightly non-trivial. Predicates that contain exact partition expr can be optimized too. This also fixes [#11092](https://github.com/ClickHouse/ClickHouse/issues/11092) which returns wrong count when `max_parallel_replicas > 1`. [#15074](https://github.com/ClickHouse/ClickHouse/pull/15074) ([Amos Bird](https://github.com/amosbird)). + +#### Build/Testing/Packaging Improvement + +* Add flaky check for stateless tests. It will detect potentially flaky functional tests in advance, before they are merged. [#16238](https://github.com/ClickHouse/ClickHouse/pull/16238) ([alesapin](https://github.com/alesapin)). +* Use proper version for `croaring` instead of amalgamation. [#16285](https://github.com/ClickHouse/ClickHouse/pull/16285) ([sundyli](https://github.com/sundy-li)). +* Improve generation of build files for `ya.make` build system (Arcadia). [#16700](https://github.com/ClickHouse/ClickHouse/pull/16700) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add MySQL BinLog file check tool for `MaterializeMySQL` database engine. `MaterializeMySQL` is an experimental feature. [#16223](https://github.com/ClickHouse/ClickHouse/pull/16223) ([Winter Zhang](https://github.com/zhang2014)). +* Check for executable bit on non-executable files. People often accidentially commit executable files from Windows. [#15843](https://github.com/ClickHouse/ClickHouse/pull/15843) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Check for `#pragma once` in headers. [#15818](https://github.com/ClickHouse/ClickHouse/pull/15818) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix illegal code style `&vector[idx]` in libhdfs3. This fixes libcxx debug build. See also https://github.com/ClickHouse-Extras/libhdfs3/pull/8 . [#15815](https://github.com/ClickHouse/ClickHouse/pull/15815) ([Amos Bird](https://github.com/amosbird)). +* Fix build of one miscellaneous example tool on Mac OS. Note that we don't build examples on Mac OS in our CI (we build only ClickHouse binary), so there is zero chance it will not break again. This fixes [#15804](https://github.com/ClickHouse/ClickHouse/issues/15804). [#15808](https://github.com/ClickHouse/ClickHouse/pull/15808) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Simplify Sys/V init script. [#14135](https://github.com/ClickHouse/ClickHouse/pull/14135) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Added `boost::program_options` to `db_generator` in order to increase its usability. This closes [#15940](https://github.com/ClickHouse/ClickHouse/issues/15940). [#15973](https://github.com/ClickHouse/ClickHouse/pull/15973) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + + ## ClickHouse release 20.10 ### ClickHouse release v20.10.3.30, 2020-10-28 From 0b9f0db62074e92aa0c7fd7341ebe204f6403fd5 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 12 Nov 2020 01:58:49 +0300 Subject: [PATCH 114/114] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e552fe06962..4474675e9ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,7 @@ * Enable parsing enum values by their numeric ids for CSV, TSV and JSON input formats. [#15685](https://github.com/ClickHouse/ClickHouse/pull/15685) ([vivarum](https://github.com/vivarum)). * Better read task scheduling for JBOD architecture and `MergeTree` storage. New setting `read_backoff_min_concurrency` which serves as the lower limit to the number of reading threads. [#16423](https://github.com/ClickHouse/ClickHouse/pull/16423) ([Amos Bird](https://github.com/amosbird)). * Add missing support for `LowCardinality` in `Avro` format. [#16521](https://github.com/ClickHouse/ClickHouse/pull/16521) ([Mike](https://github.com/myrrc)). -* Workaround for use `S3` with nginx server as proxy. Nginx currenty does not accept urls with empty path like `http://domain.com?delete`, but vanilla aws-sdk-cpp produces this kind of urls. This commit uses patched aws-sdk-cpp version, which makes urls with "/" as path in this cases, like http://domain.com/?delete. [#16814](https://github.com/ClickHouse/ClickHouse/pull/16814) ([ianton-ru](https://github.com/ianton-ru)). +* Workaround for use `S3` with nginx server as proxy. Nginx currenty does not accept urls with empty path like `http://domain.com?delete`, but vanilla aws-sdk-cpp produces this kind of urls. This commit uses patched aws-sdk-cpp version, which makes urls with "/" as path in this cases, like `http://domain.com/?delete`. [#16814](https://github.com/ClickHouse/ClickHouse/pull/16814) ([ianton-ru](https://github.com/ianton-ru)). * Better diagnostics on parse errors in input data. Provide row number on `Cannot read all data` errors. [#16644](https://github.com/ClickHouse/ClickHouse/pull/16644) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Make the behaviour of `minMap` and `maxMap` more desireable. It will not skip zero values in the result. Fixes [#16087](https://github.com/ClickHouse/ClickHouse/issues/16087). [#16631](https://github.com/ClickHouse/ClickHouse/pull/16631) ([Ildus Kurbangaliev](https://github.com/ildus)). * Better update of ZooKeeper configuration in runtime. [#16630](https://github.com/ClickHouse/ClickHouse/pull/16630) ([sundyli](https://github.com/sundy-li)).